From afd4f72016c7ddfaff21e0a88c8a0c5bb2a4fd18 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Thu, 9 Apr 2026 14:05:10 -0400 Subject: [PATCH 001/250] ALP decoding plugin (#7370) Follow up to #7314 this time for ALP --------- Signed-off-by: Andrew Duffy --- encodings/alp/src/alp/mod.rs | 3 + encodings/alp/src/alp/plugin.rs | 226 ++++++++++++++++++++++++ encodings/alp/src/lib.rs | 9 +- encodings/fastlanes/public-api.lock | 2 - encodings/fastlanes/src/lib.rs | 14 +- vortex-array/public-api.lock | 2 + vortex-array/src/arrays/patched/mod.rs | 14 ++ vortex-btrblocks/src/schemes/float.rs | 31 +++- vortex-btrblocks/src/schemes/integer.rs | 2 +- vortex-file/src/strategy.rs | 3 +- 10 files changed, 284 insertions(+), 22 deletions(-) create mode 100644 encodings/alp/src/alp/plugin.rs diff --git a/encodings/alp/src/alp/mod.rs b/encodings/alp/src/alp/mod.rs index 7d5049b59e3..88375fe2ae6 100644 --- a/encodings/alp/src/alp/mod.rs +++ b/encodings/alp/src/alp/mod.rs @@ -18,8 +18,11 @@ mod compress; pub(crate) mod compute; mod decompress; mod ops; +mod plugin; mod rules; +pub(crate) use plugin::ALPPatchedPlugin; + #[cfg(test)] mod tests { use prost::Message; diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs new file mode 100644 index 00000000000..87f777ee9a2 --- /dev/null +++ b/encodings/alp/src/alp/plugin.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! A custom [`ArrayPlugin`] that lets you load in and deserialize an `ALP` array with interior +//! patches as a `PatchedArray` that wraps a patchless `ALP` array. +//! +//! This enables zero-cost backward compatibility with previously written datasets. + +use vortex_array::ArrayId; +use vortex_array::ArrayPlugin; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Patched; +use vortex_array::buffer::BufferHandle; +use vortex_array::dtype::DType; +use vortex_array::serde::ArrayChildren; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use vortex_session::VortexSession; + +use crate::ALP; +use crate::ALPArrayExt; +use crate::ALPArrayOwnedExt; + +/// Custom deserialization plugin that converts an ALP array with interior +/// patches into a PatchedArray holding an ALP array. +#[derive(Debug, Clone)] +pub(crate) struct ALPPatchedPlugin; + +impl ArrayPlugin for ALPPatchedPlugin { + fn id(&self) -> ArrayId { + // We reuse the existing `ALP` ID so that we can take over its + // deserialization pathway. + ALP::ID + } + + fn serialize( + &self, + array: &ArrayRef, + session: &VortexSession, + ) -> VortexResult>> { + // Delegate to ALP's metadata serde + ALP.serialize(array, session) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + buffers: &[BufferHandle], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult { + let alp_array = ALP + .deserialize(dtype, len, metadata, buffers, children, session)? + .try_downcast::() + .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?; + + // Check if there are interior patches to externalize. + let Some(patches) = alp_array.patches() else { + return Ok(alp_array.into_array()); + }; + + // Extract components and create a new ALP array without patches. + let (encoded, exponents, _) = alp_array.into_parts(); + + let alp_without_patches = ALP::try_new(encoded, exponents, None)?.into_array(); + + let patched = Patched::from_array_and_patches( + alp_without_patches, + &patches, + &mut session.create_execution_ctx(), + )?; + + Ok(patched.into_array()) + } +} + +#[cfg(test)] +mod tests { + use std::f64::consts::PI; + use std::sync::LazyLock; + + use vortex_array::ArrayPlugin; + use vortex_array::IntoArray; + use vortex_array::arrays::PatchedArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::patched::PatchedArraySlotsExt; + use vortex_array::buffer::BufferHandle; + use vortex_array::session::ArraySession; + use vortex_array::session::ArraySessionExt; + use vortex_error::VortexResult; + use vortex_error::vortex_err; + use vortex_session::VortexSession; + + use super::ALPPatchedPlugin; + use crate::ALP; + use crate::ALPArray; + use crate::ALPArrayExt; + use crate::alp_encode; + + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + session.arrays().register(ALPPatchedPlugin); + session + }); + + #[test] + fn test_decode_alp_patches() -> VortexResult<()> { + // Create values where some don't encode cleanly with ALP, causing patches. + // PI doesn't encode cleanly. + let values: Vec = (0..100) + .map(|i| if i % 4 == 3 { PI } else { i as f64 }) + .collect(); + + let parray = PrimitiveArray::from_iter(values); + let alp_encoded = alp_encode(&parray, None)?; + + assert!( + alp_encoded.patches().is_some(), + "Expected ALP array to have patches" + ); + + let array = alp_encoded.as_array(); + + let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let children = array.children(); + let buffers = array + .buffers() + .into_iter() + .map(BufferHandle::new_host) + .collect::>(); + + let deserialized = ALPPatchedPlugin.deserialize( + array.dtype(), + array.len(), + &metadata, + &buffers, + &children, + &SESSION, + )?; + + let patched: PatchedArray = deserialized + .try_downcast() + .map_err(|a| vortex_err!("Expected Patched, got {}", a.encoding_id()))?; + + let inner_alp: ALPArray = patched + .inner() + .clone() + .try_downcast() + .map_err(|a| vortex_err!("Expected inner ALP, got {}", a.encoding_id()))?; + + assert!( + inner_alp.patches().is_none(), + "Inner ALP should NOT have patches" + ); + + Ok(()) + } + + #[test] + fn alp_without_patches_stays_alp() -> VortexResult<()> { + // Values that encode cleanly without patches. + let values: Vec = (0..100).map(|i| i as f64).collect(); + let parray = PrimitiveArray::from_iter(values); + let alp_encoded = alp_encode(&parray, None)?; + + assert!( + alp_encoded.patches().is_none(), + "Expected ALP array without patches" + ); + + let array = alp_encoded.as_array(); + + let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let children = array.children(); + let buffers = array + .buffers() + .into_iter() + .map(BufferHandle::new_host) + .collect::>(); + + let deserialized = ALPPatchedPlugin.deserialize( + array.dtype(), + array.len(), + &metadata, + &buffers, + &children, + &SESSION, + )?; + + let result = deserialized + .try_downcast::() + .map_err(|a| vortex_err!("Expected deserialized ALP, got {}", a.encoding_id()))?; + + assert!(result.patches().is_none(), "Result should not have patches"); + + Ok(()) + } + + #[test] + #[should_panic(expected = "index out of bounds")] + fn primitive_array_returns_error() { + let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array(); + + let metadata = array.metadata(&SESSION).unwrap().unwrap_or_default(); + let children = array.children(); + let buffers = array + .buffers() + .into_iter() + .map(BufferHandle::new_host) + .collect::>(); + + // This panics because PrimitiveArray has no children and ALP requires encoded child. + let _result = ALPPatchedPlugin.deserialize( + array.dtype(), + array.len(), + &metadata, + &buffers, + &children, + &SESSION, + ); + } +} diff --git a/encodings/alp/src/lib.rs b/encodings/alp/src/lib.rs index 3a955f368af..35a7534d7aa 100644 --- a/encodings/alp/src/lib.rs +++ b/encodings/alp/src/lib.rs @@ -21,6 +21,7 @@ pub use alp_rd::*; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::nan_count::NanCount; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; +use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; @@ -29,7 +30,13 @@ mod alp_rd; /// Initialize ALP encoding in the given session. pub fn initialize(session: &VortexSession) { - session.arrays().register(ALP); + // If we're using the experimental Patched encoding, register a shim + // for ALP with interior patches to decode as Patched array. + if *USE_EXPERIMENTAL_PATCHES { + session.arrays().register(ALPPatchedPlugin); + } else { + session.arrays().register(ALP); + } session.arrays().register(ALPRD); // Register the ALP-specific NaN count aggregate kernel. diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 15cc889bf9f..e65c373b460 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -570,8 +570,6 @@ impl vortex_array::hash::ArrayHash for vortex_fastlanes::RLEData pub fn vortex_fastlanes::RLEData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) -pub static vortex_fastlanes::USE_EXPERIMENTAL_PATCHES: std::sync::lazy_lock::LazyLock - pub trait vortex_fastlanes::BitPackedArrayExt: vortex_fastlanes::BitPackedArraySlotsExt pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 9a9f37d9af8..a00e13b53fa 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -3,9 +3,6 @@ #![allow(clippy::cast_possible_truncation)] -use std::env; -use std::sync::LazyLock; - pub use bitpacking::*; pub use delta::*; pub use r#for::*; @@ -31,19 +28,10 @@ use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; +use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; -/// Flag indicating if experimental patched array support is enabled. -/// -/// This is set using the environment variable `VORTEX_EXPERIMENTAL_PATCHED_ARRAY`. -/// -/// When this is true, any BitPacked array with interior patches will be read as a `Patched` -/// array, and the builtin compressor will use Patched array with BitPacked instead of -/// BitPacked array with interior patches. -pub static USE_EXPERIMENTAL_PATCHES: LazyLock = - LazyLock::new(|| env::var("VORTEX_EXPERIMENTAL_PATCHED_ARRAY").is_ok()); - /// Initialize fastlanes encodings in the given session. pub fn initialize(session: &VortexSession) { // If we're using the experimental Patched encoding, register a shim diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 19dc56a37f2..de53b12a037 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3538,6 +3538,8 @@ pub fn vortex_array::arrays::patched::PatchedSlotsView<'a>::fmt(&self, f: &mut c impl<'a> core::marker::Copy for vortex_array::arrays::patched::PatchedSlotsView<'a> +pub static vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES: std::sync::lazy_lock::LazyLock + pub trait vortex_array::arrays::patched::PatchedArrayExt: vortex_array::arrays::patched::PatchedArraySlotsExt pub fn vortex_array::arrays::patched::PatchedArrayExt::lane_range(&self, chunk: usize, lane: usize) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/patched/mod.rs b/vortex-array/src/arrays/patched/mod.rs index 56024f50d8e..b0453c68677 100644 --- a/vortex-array/src/arrays/patched/mod.rs +++ b/vortex-array/src/arrays/patched/mod.rs @@ -71,6 +71,9 @@ mod array; mod compute; mod vtable; +use std::env; +use std::sync::LazyLock; + pub use array::*; use vortex_buffer::ByteBuffer; pub use vtable::*; @@ -96,3 +99,14 @@ const fn patch_lanes() -> usize { // from shared to global memory. if size_of::() < 8 { 32 } else { 16 } } + +/// Flag indicating if experimental patched array support is enabled. +/// +/// This is set using the environment variable `VORTEX_EXPERIMENTAL_PATCHED_ARRAY`. +/// +/// When this is true, any arrays with interior `Patches` will be read as a `Patched` +/// array, and eliminate the interior patches. +/// +/// The builtin compressor will also generate Patched arrays. +pub static USE_EXPERIMENTAL_PATCHES: LazyLock = + LazyLock::new(|| env::var("VORTEX_EXPERIMENTAL_PATCHED_ARRAY").is_ok()); diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index e74b130bbd0..6c3dae9ab64 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -13,7 +13,11 @@ use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Patched; +use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; use vortex_compressor::estimate::CompressionEstimate; @@ -107,11 +111,30 @@ impl Scheme for ALPScheme { let compressed_alp_ints = compressor.compress_child(alp_encoded.encoded(), &ctx, self.id(), 0)?; - // Patches are not compressed. They should be infrequent, and if they are not then we want - // to keep them linear for easy indexing. - let patches = alp_encoded.patches().map(compress_patches).transpose()?; + let alp_stats = alp_encoded.as_array().statistics().to_owned(); + let exponents = alp_encoded.exponents(); + + if *USE_EXPERIMENTAL_PATCHES { + let patches = alp_encoded.patches(); + + // Create ALP array without interior patches. + let alp_array = ALP::new(compressed_alp_ints, exponents, None).into_array(); + + match patches { + None => Ok(alp_array), + Some(p) => Ok(Patched::from_array_and_patches( + alp_array, + &p, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .with_stats_set(alp_stats) + .into_array()), + } + } else { + let patches = alp_encoded.patches().map(compress_patches).transpose()?; - Ok(ALP::new(compressed_alp_ints, alp_encoded.exponents(), patches).into_array()) + Ok(ALP::new(compressed_alp_ints, exponents, patches).into_array()) + } } } diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index f56b8d478f0..1fd75c61910 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -11,6 +11,7 @@ use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Patched; +use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::scalar::Scalar; use vortex_compressor::builtins::FloatDictScheme; @@ -32,7 +33,6 @@ use vortex_fastlanes::FoR; use vortex_fastlanes::FoRArrayExt; use vortex_fastlanes::RLE; use vortex_fastlanes::RLEArrayExt; -use vortex_fastlanes::USE_EXPERIMENTAL_PATCHES; use vortex_fastlanes::bitpack_compress::bit_width_histogram; use vortex_fastlanes::bitpack_compress::bitpack_encode; use vortex_fastlanes::bitpack_compress::find_best_bit_width; diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 3253d998bf5..a30d28ab886 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -26,6 +26,7 @@ use vortex_array::arrays::Primitive; use vortex_array::arrays::Struct; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinView; +use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::dtype::FieldPath; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; @@ -91,7 +92,7 @@ pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { allowed.insert(Masked.id()); allowed.insert(Dict.id()); - if *vortex_fastlanes::USE_EXPERIMENTAL_PATCHES { + if *USE_EXPERIMENTAL_PATCHES { allowed.insert(Patched.id()); } From 256a0299f993205fe05ed2e5d6e31627d690293c Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 9 Apr 2026 19:06:44 +0100 Subject: [PATCH 002/250] fix(cuda): handle validity in GPU kernels (#7372) All GPU kernels should now handle validity or fall back to the CPU otherwise in cases they don't. Dynamic dispatch: - Propagate root array validity through FusedPlan to output. - Reject Dict with nullable codes and RunEnd with nullable ends (out-of-bounds risk). - Short-circuit AllInvalid (skip kernel) and empty arrays. Standalone kernels: - RunEnd/Zstd: replace unreachable!()/unimplemented!() with vortex_bail!() for unsupported nullable cases, enabling graceful CPU fallback. - ALP: remove stale TODO; patch validity scatter is unnecessary since the encoder already strips null positions from the exception list. Signed-off-by: Alexander Droste --- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 1 + vortex-cuda/src/dynamic_dispatch/mod.rs | 227 +++++++++++++++++- .../src/dynamic_dispatch/plan_builder.rs | 23 +- vortex-cuda/src/kernel/encodings/alp.rs | 76 +++++- vortex-cuda/src/kernel/encodings/runend.rs | 38 ++- vortex-cuda/src/kernel/encodings/zstd.rs | 39 ++- 6 files changed, 395 insertions(+), 9 deletions(-) diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 9cff91f9856..e346119d35e 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -135,6 +135,7 @@ impl BenchRunner { dispatch_plan, device_buffers, shared_mem_bytes, + .. } = plan.materialize(cuda_ctx).vortex_expect("materialize plan"); let device_plan = Arc::new( diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 192438d0329..e82cce668cb 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -24,14 +24,18 @@ use cudarc::driver::DevicePtr; use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use vortex::array::Canonical; +use vortex::array::IntoArray; +use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::buffer::BufferHandle; use vortex::array::buffer::DeviceBufferExt; use vortex::array::match_each_unsigned_integer_ptype; +use vortex::array::scalar::Scalar; use vortex::array::validity::Validity; use vortex::buffer::Alignment; use vortex::buffer::ByteBuffer; use vortex::buffer::ByteBufferMut; +use vortex::dtype::DType; use vortex::dtype::Nullability; use vortex::dtype::PType; use vortex::error::VortexResult; @@ -408,6 +412,15 @@ impl ScalarOp { impl MaterializedPlan { pub fn execute(self, len: usize, ctx: &mut CudaExecutionCtx) -> VortexResult { let output_ptype = self.dispatch_plan.output_ptype(); + + // All values are null — no need to touch the GPU. + if matches!(self.validity, Validity::AllInvalid) { + let dtype = DType::Primitive(output_ptype, Nullability::Nullable); + return ConstantArray::new(Scalar::null(dtype), len) + .into_array() + .to_canonical(); + } + // The CUDA kernels are instantiated for unsigned integer types only; // map signed/float ptypes to their same-width unsigned counterpart. let unsigned_ptype = match output_ptype { @@ -431,9 +444,11 @@ impl MaterializedPlan { where T: cudarc::driver::DeviceRepr + vortex::dtype::NativePType, { + let nullability = self.validity.nullability(); + if len == 0 { return Ok(Canonical::Primitive(PrimitiveArray::empty::( - Nullability::NonNullable, + nullability, ))); } @@ -467,7 +482,7 @@ impl MaterializedPlan { Ok(Canonical::Primitive(PrimitiveArray::from_buffer_handle( BufferHandle::new_device(output_buf.slice_typed::(0..len)), output_ptype, - Validity::NonNullable, + self.validity, ))) } } @@ -485,6 +500,7 @@ mod tests { use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::scalar::Scalar; + use vortex::array::validity::Validity; use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; use vortex::dtype::PType; @@ -509,6 +525,7 @@ mod tests { use crate::CudaBufferExt; use crate::CudaDeviceBuffer; use crate::CudaExecutionCtx; + use crate::executor::CudaArrayExt; use crate::hybrid_dispatch::try_gpu_dispatch; use crate::session::CudaSession; @@ -1817,4 +1834,210 @@ mod tests { Ok(()) } + + // ═══════════════════════════════════════════════════════════════════ + // Validity propagation tests + // ═══════════════════════════════════════════════════════════════════ + + /// Nullable Primitive array — LOAD source with validity propagated. + #[crate::test] + async fn test_nullable_primitive() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let array = PrimitiveArray::from_option_iter( + (0..2048u32).map(|i| if i % 3 == 0 { None } else { Some(i) }), + ); + let cpu = array.to_canonical()?.into_array(); + + let gpu = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// Nullable FoR(BitPacked) — validity from the root propagated through + /// the fused plan. The standard encoding flow is: subtract FoR reference + /// to get residuals, then bitpack. BitPacked::encode preserves input + /// validity, so this produces a real nullable FoR(BitPacked) tree. + #[crate::test] + async fn test_nullable_for_bitpacked() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let len = 2048; + let reference = 1000u32; + + // Original values in [reference, reference+63], every 5th null. + let values: Vec> = (0..len) + .map(|i| { + if i % 5 == 0 { + None + } else { + Some((i as u32 % 64) + reference) + } + }) + .collect(); + let prim = PrimitiveArray::from_option_iter(values.iter().copied()); + let cpu = prim.to_canonical()?.into_array(); + + // FoR encoding: subtract reference to get residuals [0..63]. + // Null positions get 0 (from from_option_iter), which is fine — + // after subtracting reference it wraps, but validity masks it. + let residuals = + PrimitiveArray::from_option_iter(values.iter().map(|v| v.map(|x| x - reference))); + + // BitPacked::encode preserves nullable validity from the input. + let bp = BitPacked::encode(&residuals.into_array(), 6)?; + let for_arr = FoR::try_new(bp.into_array(), reference.into())?; + + // Verify the plan actually fuses (not just a LOAD). + assert!( + matches!( + DispatchPlan::new(&for_arr.clone().into_array())?, + DispatchPlan::Fused(_) + ), + "FoR(BitPacked) with nullable validity should produce a Fused plan" + ); + + let gpu = try_gpu_dispatch(&for_arr.into_array(), &mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// AllInvalid array — kernel should be skipped entirely. + #[crate::test] + async fn test_all_invalid_skips_kernel() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let array = PrimitiveArray::new(Buffer::from(vec![0u32; 2048]), Validity::AllInvalid); + + let result = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx) + .await? + .into_host() + .await?; + + let prim = result.into_primitive(); + assert_eq!(prim.len(), 2048); + assert!(matches!(prim.validity()?, Validity::AllInvalid)); + Ok(()) + } + + /// AllValid nullable array — should fuse and produce AllValid output. + #[crate::test] + async fn test_all_valid_nullable() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let values: Vec = (0..2048).collect(); + let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::AllValid); + + let cpu = array.to_canonical()?.into_array(); + let gpu = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// Dict with nullable codes must fall back to Unfused (not fused). + #[crate::test] + fn test_dict_nullable_codes_rejected() -> VortexResult<()> { + use vortex::buffer::buffer; + + let codes = PrimitiveArray::from_option_iter([Some(0u32), None, Some(1), None, Some(2)]); + let values = PrimitiveArray::new(buffer![10u32, 20, 30], NonNullable); + let dict = DictArray::try_new(codes.into_array(), values.into_array())?; + + let plan = DispatchPlan::new(&dict.into_array())?; + assert!( + matches!(plan, DispatchPlan::Unfused), + "Dict with nullable codes should fall back to Unfused" + ); + Ok(()) + } + + /// Dict with non-nullable codes but nullable values should still fuse. + #[crate::test] + async fn test_dict_nullable_values_fuses() -> VortexResult<()> { + use vortex::buffer::buffer; + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let codes = PrimitiveArray::new(buffer![0u32, 1, 2, 2, 1, 0], NonNullable); + let values = PrimitiveArray::from_option_iter([Some(10u32), None, Some(30)]); + let dict = DictArray::try_new(codes.into_array(), values.into_array())?; + + let cpu = dict.to_canonical()?.into_array(); + let gpu = dict + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// Nullable FoR(BitPacked) through CUB filter — the original bug scenario. + /// Validity must survive through fused dispatch and into the filter. + #[crate::test] + async fn test_nullable_fused_then_filter() -> VortexResult<()> { + use vortex::array::arrays::FilterArray; + use vortex::mask::Mask; + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let len = 2048usize; + let values: Vec> = (0..len) + .map(|i| { + if i % 7 == 0 { + None + } else { + Some((i % 64) as u32) + } + }) + .collect(); + let prim = PrimitiveArray::from_option_iter(values.iter().copied()); + + // Keep every other element. + let mask = Mask::from_iter((0..len).map(|i| i % 2 == 0)); + let filter_array = FilterArray::try_new(prim.into_array(), mask)?; + + let cpu = filter_array.to_canonical()?.into_array(); + let gpu = filter_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// Empty nullable array should preserve nullability. + #[crate::test] + async fn test_empty_nullable_array() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let array = PrimitiveArray::new(Buffer::::empty(), Validity::AllValid); + let result = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx).await?; + let prim = result.into_primitive(); + assert_eq!(prim.len(), 0); + assert_eq!(prim.validity()?.nullability(), Nullability::Nullable); + Ok(()) + } } diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index c44b7224b7e..ebeb94263b1 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -15,6 +15,7 @@ use vortex::array::arrays::Slice; use vortex::array::arrays::dict::DictArraySlotsExt; use vortex::array::arrays::slice::SliceArrayExt; use vortex::array::buffer::BufferHandle; +use vortex::array::validity::Validity; use vortex::dtype::PType; use vortex::encodings::alp::ALP; use vortex::encodings::alp::ALPArrayExt; @@ -52,6 +53,8 @@ pub struct MaterializedPlan { pub device_buffers: Vec, /// Dynamic shared memory bytes needed to launch this plan. pub shared_mem_bytes: u32, + /// Validity of the root array, propagated to the output. + pub validity: Validity, } /// Checks whether the encoding of an array can be fused into a dynamic-dispatch plan. @@ -72,6 +75,11 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { } if id == Dict::ID { let arr = array.as_::(); + // Nullable codes could hold garbage values at null positions, causing + // out-of-bounds shared memory reads in the DICT gather scalar op. + if arr.codes().dtype().is_nullable() { + return false; + } // Dict codes and values may have different byte widths. // The kernel handles mixed widths via widening input stages, // but only when codes are no wider than values (the output type). @@ -85,6 +93,12 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { } if id == RunEnd::ID { let arr = array.as_::(); + // Nullable ends could hold garbage values at null positions, causing + // unpredictable binary search / forward-scan behavior in the RUNEND + // source op. + if arr.ends().dtype().is_nullable() { + return false; + } // RunEnd ends and values may have different byte widths. // The kernel handles mixed widths via widening input stages, // but only when ends are no wider than values (the output type). @@ -213,6 +227,8 @@ pub struct FusedPlan { output_elem_bytes: u32, /// PType of the root (output) array, as a C ABI tag. output_ptype: PTypeTag, + /// Validity of the root array, propagated to the output. + validity: Validity, } impl DispatchPlan { @@ -220,7 +236,9 @@ impl DispatchPlan { /// /// # Limitations /// - /// - Validity bitmaps are ignored; only `NonNullable`/`AllValid` is supported. + /// - Validity is propagated from the root array to the output. Nullable + /// arrays are supported, but Dict with nullable codes and RunEnd with + /// nullable ends are rejected to guard against out-of-bounds access. /// - `BitPackedArray` and `ALPArray` with patches are not supported. /// - Only f32 ALP is supported (kernel stores multipliers as `float`). pub fn new(array: &ArrayRef) -> VortexResult { @@ -276,6 +294,7 @@ impl FusedPlan { } let output_elem_bytes = output_ptype_rust.byte_width() as u32; let output_ptype = ptype_to_tag(output_ptype_rust); + let validity = array.validity()?; let mut pending_subtrees: Vec = Vec::new(); let mut plan = Self { @@ -284,6 +303,7 @@ impl FusedPlan { source_buffers: Vec::new(), output_elem_bytes, output_ptype, + validity, }; let len = array.len() as u32; @@ -365,6 +385,7 @@ impl FusedPlan { dispatch_plan: CudaDispatchPlan::new(stages, self.output_ptype), device_buffers, shared_mem_bytes, + validity: self.validity, }) } diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 6661e0fed5e..c163b92203f 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -98,7 +98,10 @@ where .arg(&array_len_u64); })?; - // Check if there are any patches to decode here + // Check if there are any patches to decode here. Patch validity does not + // need to be scattered: the ALP encoder strips null positions from the + // exception list, so patches only exist at valid positions. execute_patches + // additionally guards against nullable patch values at runtime. let output_buf = if let Some(patches) = array.patches() { match_each_unsigned_integer_ptype!(patches.indices_ptype()?, |I| { execute_patches::(patches.clone(), output_buf, ctx).await? @@ -107,9 +110,6 @@ where output_buf }; - // TODO(aduffy): scatter patch values validity. There are several places we'll need to start - // handling validity. - let output_handle = BufferHandle::new_device(Arc::new(output_buf)); Ok(Canonical::Primitive(PrimitiveArray::from_buffer_handle( output_handle, @@ -129,11 +129,13 @@ mod tests { use vortex::buffer::buffer; use vortex::encodings::alp::ALP; use vortex::encodings::alp::Exponents; + use vortex::encodings::alp::alp_encode; use vortex::error::VortexExpect; use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; + use crate::executor::CudaArrayExt; use crate::session::CudaSession; #[crate::test] @@ -179,4 +181,70 @@ mod tests { Ok(()) } + + /// ALP with nullable encoded data and patches — the encoder strips null + /// positions from the exception list, so patch validity doesn't need + /// scattering. This test verifies that the encoded child's validity is + /// preserved through the standalone ALP GPU executor. + #[crate::test] + async fn test_cuda_alp_nullable_with_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Values that will produce ALP exceptions at non-null positions. + // Nulls at positions 1 and 3; the exception at position 4 (1.23456) + // can't be encoded losslessly by ALP. + let values: Vec> = vec![ + Some(1.0), + None, + Some(2.0), + None, + Some(1.23456), + Some(3.0), + Some(4.0), + Some(5.0), + ]; + let prim = PrimitiveArray::from_option_iter(values); + let alp_array = alp_encode(&prim, None)?; + + let cpu_result = alp_array.to_canonical()?.into_array(); + + let gpu_result = alp_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } + + /// ALP with all-valid nullable data — the dtype is nullable but no + /// elements are actually null. + #[crate::test] + async fn test_cuda_alp_all_valid_nullable() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + let values = PrimitiveArray::new( + Buffer::from(vec![1.0f32, 2.0, 3.0, 4.0, 5.0]), + Validity::AllValid, + ); + let alp_array = alp_encode(&values, None)?; + + let cpu_result = alp_array.to_canonical()?.into_array(); + + let gpu_result = alp_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } } diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 3f3bce2579b..428dbfea71e 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -148,7 +148,9 @@ async fn decode_runend_typed { - unreachable!("Array validity not yet supported for run-end decoding on GPU"); + vortex_bail!( + "RunEnd GPU decoding does not yet support per-element validity in values; falling back to CPU" + ); } }; @@ -163,6 +165,7 @@ async fn decode_runend_typed(ends: Vec, values: Vec) -> RunEndArray @@ -294,4 +298,36 @@ mod tests { Ok(()) } + + #[crate::test] + async fn test_cuda_runend_nullable_values_falls_back_to_cpu() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Build a RunEnd array whose values have Validity::Array (some nulls). + let ends_array = + PrimitiveArray::new(Buffer::from(vec![3u32, 6, 10]), Validity::NonNullable) + .into_array(); + let validity = + Validity::Array(BoolArray::from_iter([true, false, true].into_iter()).into_array()); + let values_array = + PrimitiveArray::new(Buffer::from(vec![10i32, 0, 30]), validity).into_array(); + let runend_array = RunEnd::new(ends_array, values_array); + + let cpu_result = runend_array.to_canonical()?.into_array(); + + // execute_cuda should fall back to CPU and still produce the correct result. + let gpu_result = runend_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await + .vortex_expect("GPU/CPU fallback should succeed") + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + + Ok(()) + } } diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index 77ade13a6b0..4f09f418e4b 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -31,6 +31,7 @@ use vortex::encodings::zstd::ZstdDataParts; use vortex::encodings::zstd::ZstdMetadata; use vortex::error::VortexExpect; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_err; use vortex::mask::AllOr; use vortex_nvcomp::sys::nvcompStatus_t; @@ -342,7 +343,7 @@ async fn decode_zstd(array: ZstdArray, ctx: &mut CudaExecutionCtx) -> VortexResu })) } _ => { - unimplemented!("CUDA ZSTD decompression does not yet support arrays with nulls") + vortex_bail!("CUDA ZSTD decompression does not yet support arrays with nulls") } } } @@ -357,6 +358,8 @@ mod tests { use vortex::session::VortexSession; use super::*; + use crate::CanonicalCudaExt; + use crate::executor::CudaArrayExt; use crate::session::CudaSession; #[crate::test] @@ -450,4 +453,38 @@ mod tests { assert_arrays_eq!(cpu_result.into_array(), gpu_result.into_array()); Ok(()) } + + /// Zstd with nullable data — the GPU kernel does not yet support nulls, + /// so `execute_cuda` should gracefully fall back to CPU and produce + /// correct results instead of panicking. + #[crate::test] + async fn test_cuda_zstd_nullable_falls_back_to_cpu() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + let strings = VarBinViewArray::from_iter_nullable_str([ + Some("hello"), + None, + Some("world"), + None, + Some("testing nullable zstd"), + Some("another string"), + ]); + + let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; + + let cpu_result = zstd_array.to_canonical()?.into_array(); + + // execute_cuda should fall back to CPU and still produce the correct result. + let gpu_result = zstd_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } } From 50d64a4bc6eea0cc02a1e6d8893812ff1170a834 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:00:13 -0400 Subject: [PATCH 003/250] Fix `AGENTS.md` sccache (#7377) ## Summary The agents keep running cargo commands with `RUSTC_WRAPPER=` even locally. Also, this capitalizes the list items because it was annoying me. ## Testing N/A Signed-off-by: Connor Tsui --- AGENTS.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f4cc4620f00..3d8fcc84eaa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,26 +2,27 @@ ## Development Guidelines -- project is a monorepo Rust workspace, java bindings in `/java`, python bindings in +- Project is a monorepo Rust workspace, java bindings in `/java`, python bindings in `/vortex-python` -- run `cargo build -p` to build a specific crate -- use `cargo clippy --all-targets --all-features` to make sure a project is free of lint issues. +- Run `cargo build -p` to build a specific crate +- Use `cargo clippy --all-targets --all-features` to make sure a project is free of lint issues. Please do this every time you reach a stopping point or think you've finished work. -- run `cargo +nightly fmt --all` to format Rust source files. Please do this every time you reach a +- Run `cargo +nightly fmt --all` to format Rust source files. Please do this every time you reach a stopping point or think you've finished work. -- run `./scripts/public-api.sh` to re-generate the public API lock files. Please do this every time +- Run `./scripts/public-api.sh` to re-generate the public API lock files. Please do this every time you reach a stopping point or think you've finished work. -- you can try running +- You can try running `cargo fix --lib --allow-dirty --allow-staged && cargo clippy --fix --lib --allow-dirty --allow-staged` to automatically many fix minor errors. -- when iterating on CI failures, fetch only failed job logs first +- When iterating on CI failures, fetch only failed job logs first (`gh run view --job --log-failed`) and run narrow local repro commands for the affected crate/tests before running workspace-wide checks. -- if `gh` commands fail with `error connecting to api.github.com` in sandbox, immediately rerun with +- If `gh` commands fail with `error connecting to api.github.com` in sandbox, immediately rerun with escalated network permissions instead of retrying in sandbox. -- if cargo fails with `sccache: error: Operation not permitted`, rerun the command with - `RUSTC_WRAPPER=` so rustc runs directly. -- run docs doctests from the docs directory (`make -C docs doctest`) so the correct Sphinx Makefile +- If cargo fails with `sccache: error: Operation not permitted`, rerun the command with + `RUSTC_WRAPPER=` so rustc runs directly. You should ONLY do this if you get this exact error, as + this is only a concern when you are running on our CI. +- Run docs doctests from the docs directory (`make -C docs doctest`) so the correct Sphinx Makefile target is used. ## Architecture From 7618c23b5348ebe3150f92ec4669bd5dc462a021 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 9 Apr 2026 16:32:52 -0400 Subject: [PATCH 004/250] Replace `&PrimitiveArray` with `ArrayView` in a bunch of places (#7378) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 Originally I just wanted the compressor stats to return `ArrayView` instead of owned things, but I decided to just do it everywhere that it touches instead of a bunch of `into_owned()` all over the place. This should be a purely cosmetic change as we only really save `Arc` copies internally. ## Testing N/A Signed-off-by: Connor Tsui --- encodings/alp/benches/alp_compress.rs | 8 +- encodings/alp/public-api.lock | 4 +- encodings/alp/src/alp/array.rs | 16 ++-- encodings/alp/src/alp/compress.rs | 59 ++++++------ encodings/alp/src/alp/compute/between.rs | 2 +- encodings/alp/src/alp/compute/cast.rs | 8 +- encodings/alp/src/alp/compute/compare.rs | 16 ++-- encodings/alp/src/alp/compute/filter.rs | 2 +- encodings/alp/src/alp/compute/mask.rs | 6 +- encodings/alp/src/alp/compute/mod.rs | 24 ++--- encodings/alp/src/alp/compute/take.rs | 2 +- encodings/alp/src/alp/plugin.rs | 4 +- encodings/alp/src/alp_rd/array.rs | 2 +- encodings/alp/src/alp_rd/compute/cast.rs | 14 +-- encodings/alp/src/alp_rd/compute/filter.rs | 15 ++- encodings/alp/src/alp_rd/compute/mask.rs | 13 +-- encodings/alp/src/alp_rd/compute/mod.rs | 22 ++--- encodings/alp/src/alp_rd/compute/take.rs | 17 ++-- encodings/alp/src/alp_rd/mod.rs | 6 +- encodings/alp/src/alp_rd/ops.rs | 6 +- .../fastlanes/benches/compute_between.rs | 2 +- encodings/fastlanes/public-api.lock | 6 +- .../src/bitpacking/array/bitpack_compress.rs | 12 ++- encodings/fastlanes/src/rle/array/mod.rs | 10 +- .../fastlanes/src/rle/array/rle_compress.rs | 54 ++++++----- encodings/fastlanes/src/rle/compute/cast.rs | 2 +- encodings/fastlanes/src/rle/vtable/mod.rs | 4 +- .../fastlanes/src/rle/vtable/operations.rs | 4 +- encodings/pco/public-api.lock | 4 +- encodings/pco/src/array.rs | 16 ++-- encodings/pco/src/compute/cast.rs | 10 +- encodings/pco/src/compute/mod.rs | 16 ++-- encodings/pco/src/test.rs | 12 +-- encodings/sequence/public-api.lock | 2 +- encodings/sequence/src/compress.rs | 18 ++-- encodings/zigzag/public-api.lock | 2 +- encodings/zigzag/src/array.rs | 2 +- encodings/zigzag/src/compress.rs | 13 ++- encodings/zigzag/src/compute/cast.rs | 16 ++-- encodings/zigzag/src/compute/mod.rs | 94 +++++++++---------- vortex-btrblocks/src/schemes/float.rs | 6 +- vortex-btrblocks/src/schemes/integer.rs | 16 ++-- vortex-btrblocks/src/schemes/string.rs | 4 +- vortex-compressor/benches/dict_encode.rs | 2 +- vortex-compressor/public-api.lock | 8 +- vortex-compressor/src/builtins/dict/float.rs | 9 +- .../src/builtins/dict/integer.rs | 9 +- vortex-compressor/src/stats/cache.rs | 15 +-- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 2 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 2 +- vortex-cuda/src/kernel/encodings/alp.rs | 4 +- vortex-python/src/arrays/compressed.rs | 2 +- .../arrays/synthetic/encodings/alp.rs | 22 ++--- .../arrays/synthetic/encodings/alprd.rs | 26 +++-- .../arrays/synthetic/encodings/pco.rs | 16 ++-- .../arrays/synthetic/encodings/rle.rs | 16 ++-- .../arrays/synthetic/encodings/zigzag.rs | 22 ++--- .../common_encoding_tree_throughput.rs | 2 +- vortex/benches/single_encoding_throughput.rs | 18 ++-- 59 files changed, 387 insertions(+), 359 deletions(-) diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index 239541533f8..f4706c30bb2 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -68,7 +68,7 @@ fn compress_alp(bencher: Bencher, args: (usize, f64, bencher .with_inputs(|| &array) - .bench_values(|array| alp_encode(array, None).unwrap()) + .bench_values(|array| alp_encode(array.as_view(), None).unwrap()) } #[divan::bench(types = [f32, f64], args = BENCH_ARGS)] @@ -93,7 +93,7 @@ fn decompress_alp(bencher: Bencher, args: (usize, f64 .with_inputs(|| { ( alp_encode( - &PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()), + PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()).as_view(), None, ) .unwrap(), @@ -136,7 +136,7 @@ fn compress_rd(bencher: Bencher, args: (usize, f64) bencher .with_inputs(|| (&primitive, &encoder)) - .bench_refs(|(primitive, encoder)| encoder.encode(primitive)) + .bench_refs(|(primitive, encoder)| encoder.encode(primitive.as_view())) } #[divan::bench(types = [f32, f64], args = RD_BENCH_ARGS)] @@ -144,7 +144,7 @@ fn decompress_rd(bencher: Bencher, args: (usize, f6 let (n, fraction_patch) = args; let primitive = make_rd_array::(n, fraction_patch); let encoder = RDEncoder::new(primitive.as_slice::()); - let encoded = encoder.encode(&primitive); + let encoded = encoder.encode(primitive.as_view()); bencher .with_inputs(|| &encoded) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index 0c5987dbae5..0f3259d6d6f 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -386,7 +386,7 @@ pub struct vortex_alp::RDEncoder impl vortex_alp::RDEncoder -pub fn vortex_alp::RDEncoder::encode(&self, array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_alp::ALPRDArray +pub fn vortex_alp::RDEncoder::encode(&self, array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_alp::ALPRDArray pub fn vortex_alp::RDEncoder::from_parts(right_bit_width: u8, codes: alloc::vec::Vec) -> Self @@ -642,7 +642,7 @@ pub fn f64::to_bits(value: Self) -> Self::UINT pub fn f64::to_u16(bits: Self::UINT) -> u16 -pub fn vortex_alp::alp_encode(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, exponents: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_alp::alp_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, exponents: core::option::Option) -> vortex_error::VortexResult pub fn vortex_alp::alp_rd_decode(left_parts: vortex_buffer::buffer_mut::BufferMut, left_parts_dict: &[u16], right_bit_width: u8, right_parts: vortex_buffer::buffer_mut::BufferMut<::UINT>, left_parts_patches: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 73852549965..2c3d2d70923 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -555,7 +555,7 @@ mod tests { #[case(2049)] fn test_execute_f32(#[case] size: usize) { let values = PrimitiveArray::from_iter((0..size).map(|i| i as f32)); - let encoded = alp_encode(&values, None).unwrap(); + let encoded = alp_encode(values.as_view(), None).unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -584,7 +584,7 @@ mod tests { #[case(2049)] fn test_execute_f64(#[case] size: usize) { let values = PrimitiveArray::from_iter((0..size).map(|i| i as f64)); - let encoded = alp_encode(&values, None).unwrap(); + let encoded = alp_encode(values.as_view(), None).unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -618,7 +618,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_iter(values); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -652,7 +652,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -687,7 +687,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -723,7 +723,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -774,7 +774,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -815,7 +815,7 @@ mod tests { let original = PrimitiveArray::from_iter(values); // First encode normally to get a properly formed ALPArray with patches. - let normally_encoded = alp_encode(&original, None).unwrap(); + let normally_encoded = alp_encode(original.as_view(), None).unwrap(); assert!( normally_encoded.patches().is_some(), "Test requires patches to be present" diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index b5c259aeef1..56050b05c6f 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -3,7 +3,9 @@ use itertools::Itertools; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::PType; use vortex_array::patches::Patches; @@ -39,7 +41,10 @@ macro_rules! match_each_alp_float_ptype { }}; } -pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> VortexResult { +pub fn alp_encode( + parray: ArrayView<'_, Primitive>, + exponents: Option, +) -> VortexResult { let (exponents, encoded, patches) = match parray.ptype() { PType::F32 => alp_encode_components_typed::(parray, exponents)?, PType::F64 => alp_encode_components_typed::(parray, exponents)?, @@ -55,7 +60,7 @@ pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> Vort reason = "u64 index cast to usize is safe for reasonable array sizes" )] fn alp_encode_components_typed( - values: &PrimitiveArray, + values: ArrayView<'_, Primitive>, exponents: Option, ) -> VortexResult<(Exponents, ArrayRef, Option)> where @@ -68,7 +73,7 @@ where let encoded_array = PrimitiveArray::new(encoded, values.validity()?).into_array(); - let validity = values.validity_mask()?; + let validity = values.array().validity_mask()?; // exceptional_positions may contain exceptions at invalid positions (which contain garbage // data). We remove null exceptions in order to keep the Patches small. let (valid_exceptional_positions, valid_exceptional_values): (Buffer, Buffer) = @@ -141,7 +146,7 @@ mod tests { #[test] fn test_compress() { let array = PrimitiveArray::new(buffer![1.234f32; 1025], Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i32; 1025]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -155,7 +160,7 @@ mod tests { #[test] fn test_nullable_compress() { let array = PrimitiveArray::from_option_iter([None, Some(1.234f32), None]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter([None, Some(1234i32), None]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -172,7 +177,7 @@ mod tests { fn test_patched_compress() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i64, 2718, 1234, 4000]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -189,7 +194,7 @@ mod tests { fn test_compress_ignores_invalid_exceptional_values() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values, Validity::from_iter([true, true, false, true])); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter(buffer![Some(1234i64), Some(2718), None, Some(4000)]); @@ -211,7 +216,7 @@ mod tests { Some(4.0), None, ]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); assert_eq!(encoded.exponents(), Exponents { e: 16, f: 13 }); @@ -225,7 +230,7 @@ mod tests { #[test] fn roundtrips_close_fractional() { let original = PrimitiveArray::from_iter([195.26274f32, 195.27837, -48.815685]); - let alp_arr = alp_encode(&original, None).unwrap(); + let alp_arr = alp_encode(original.as_view(), None).unwrap(); assert_arrays_eq!(alp_arr, original); } @@ -233,7 +238,7 @@ mod tests { fn roundtrips_all_null() { let original = PrimitiveArray::new(buffer![195.26274f64, PI, -48.815685], Validity::AllInvalid); - let alp_arr = alp_encode(&original, None).unwrap(); + let alp_arr = alp_encode(original.as_view(), None).unwrap(); let decompressed = alp_arr.into_array().to_primitive(); assert_eq!( @@ -251,7 +256,7 @@ mod tests { buffer![0.0f32, -0.0, f32::NAN, f32::NEG_INFINITY, f32::INFINITY], Validity::NonNullable, ); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let decoded = encoded.as_array().to_primitive(); for idx in 0..original.len() { let decoded_val = decoded.as_slice::()[idx]; @@ -272,7 +277,7 @@ mod tests { values[1025] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -295,7 +300,7 @@ mod tests { values[2048] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -317,7 +322,7 @@ mod tests { values[0] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -340,7 +345,7 @@ mod tests { values[100] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -361,7 +366,7 @@ mod tests { // Create 1024 elements, encode, slice to first 512, then decode let values = vec![1.234f32; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -373,7 +378,7 @@ mod tests { fn test_slice_half_chunk_f64_roundtrip() { let values = vec![5.678f64; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -389,7 +394,7 @@ mod tests { values[600] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -409,7 +414,7 @@ mod tests { values[1023] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let sliced_alp = encoded.slice(1023..1025).unwrap(); @@ -425,7 +430,7 @@ mod tests { .collect::>(); let original = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); let decoded = sliced_alp.to_primitive(); @@ -438,7 +443,7 @@ mod tests { fn test_large_f32_array_uniform_values() { let size = 10_000; let array = PrimitiveArray::new(buffer![42.125f32; size], Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -450,7 +455,7 @@ mod tests { fn test_large_f64_array_uniform_values() { let size = 50_000; let array = PrimitiveArray::new(buffer![123.456789f64; size], Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -468,7 +473,7 @@ mod tests { values[4500] = f32::INFINITY; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -490,7 +495,7 @@ mod tests { values[7000] = 999.999999999; let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::NonNullable); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -520,7 +525,7 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -541,7 +546,7 @@ mod tests { let validity = Validity::from_iter((0..size).map(|i| !matches!(i, 500 | 2500))); let array = PrimitiveArray::new(Buffer::from(values), validity); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -572,7 +577,7 @@ mod tests { values[2900] = PI; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(&original, None).unwrap(); + let encoded = alp_encode(original.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); // Slice ending mid-chunk-2 (element 2500 is inside chunk 2 = 2048..3072). diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index c1922a719f9..b9281140c1d 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -123,7 +123,7 @@ mod tests { fn comparison_range() { let value = 0.0605_f32; let array = PrimitiveArray::from_iter([value; 1]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); assert_between( diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index 83ee609f886..b663b861c79 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -78,7 +78,7 @@ mod tests { #[test] fn issue_5766_test_cast_alp_with_patches_to_nullable() -> VortexResult<()> { let values = buffer![1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456].into_array(); - let alp = alp_encode(&values.to_primitive(), None)?; + let alp = alp_encode(values.to_primitive().as_view(), None)?; assert!( alp.patches().is_some(), @@ -98,7 +98,7 @@ mod tests { #[test] fn test_cast_alp_f32_to_f64() -> VortexResult<()> { let values = buffer![1.5f32, 2.5, 3.5, 4.5].into_array(); - let alp = alp_encode(&values.to_primitive(), None)?; + let alp = alp_encode(values.to_primitive().as_view(), None)?; let casted = alp .into_array() @@ -120,7 +120,7 @@ mod tests { #[test] fn test_cast_alp_to_int() -> VortexResult<()> { let values = buffer![1.0f32, 2.0, 3.0, 4.0].into_array(); - let alp = alp_encode(&values.to_primitive(), None)?; + let alp = alp_encode(values.to_primitive().as_view(), None)?; let casted = alp .into_array() @@ -143,7 +143,7 @@ mod tests { #[case(buffer![42.42f64].into_array())] #[case(buffer![0.0f32, -1.5, 2.5, -3.5, 4.5].into_array())] fn test_cast_alp_conformance(#[case] array: vortex_array::ArrayRef) -> VortexResult<()> { - let alp = alp_encode(&array.to_primitive(), None).vortex_expect("cannot fail"); + let alp = alp_encode(array.to_primitive().as_view(), None).vortex_expect("cannot fail"); test_cast_conformance(&alp.into_array()); Ok(()) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index 245715c2a6e..8c4e272bcaf 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -183,7 +183,7 @@ mod tests { #[test] fn basic_comparison_test() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -206,7 +206,7 @@ mod tests { #[test] fn comparison_with_unencodable_value() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -231,7 +231,7 @@ mod tests { #[test] fn comparison_range() { let array = PrimitiveArray::from_iter([0.0605_f32; 10]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -270,7 +270,7 @@ mod tests { #[test] fn comparison_zeroes() { let array = PrimitiveArray::from_iter([0.0_f32; 10]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -312,7 +312,7 @@ mod tests { fn compare_with_patches() { let array = PrimitiveArray::from_iter([1.234f32, 1.5, 19.0, std::f32::consts::E, 1_000_000.9]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); assert!(encoded.patches().is_some()); // Not supported! @@ -326,7 +326,7 @@ mod tests { #[test] fn compare_to_null() { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let other = ConstantArray::new( Scalar::null(DType::Primitive(PType::F32, Nullability::Nullable)), @@ -349,7 +349,7 @@ mod tests { #[case(f32::NEG_INFINITY, true)] fn compare_to_non_finite_gt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Gt).unwrap(); let expected = BoolArray::from_iter([result; 10]); @@ -363,7 +363,7 @@ mod tests { #[case(f32::NEG_INFINITY, false)] fn compare_to_non_finite_lt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(&array, None).unwrap(); + let encoded = alp_encode(array.as_view(), None).unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Lt).unwrap(); let expected = BoolArray::from_iter([result; 10]); diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 10acc57ab8f..622a8d59b6b 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -61,7 +61,7 @@ mod test { 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 ].into_array())] fn test_filter_alp_conformance(#[case] array: ArrayRef) { - let alp = alp_encode(&array.to_primitive(), None).unwrap(); + let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); test_filter_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 82eab0ebfa5..137aa263f53 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -79,7 +79,7 @@ mod test { 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0 ].into_array())] fn test_mask_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(&array.to_primitive(), None).unwrap(); + let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); test_mask_conformance(&alp.into_array()); } @@ -91,7 +91,7 @@ mod test { .map(|i| if i % 4 == 3 { PI } else { 1.0 }) .collect(); let array = PrimitiveArray::from_iter(values); - let alp = alp_encode(&array, None).unwrap(); + let alp = alp_encode(array.as_view(), None).unwrap(); assert!(alp.patches().is_some(), "expected patches"); test_mask_conformance(&alp.into_array()); } @@ -99,7 +99,7 @@ mod test { #[test] fn test_mask_alp_with_patches_casts_surviving_patch_values_to_nullable() { let values = PrimitiveArray::from_iter([1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456]); - let alp = alp_encode(&values, None).unwrap(); + let alp = alp_encode(values.as_view(), None).unwrap(); assert!(alp.patches().is_some(), "expected patches"); let keep_mask = BoolArray::from_iter([false, true, true, true, true]).into_array(); diff --git a/encodings/alp/src/alp/compute/mod.rs b/encodings/alp/src/alp/compute/mod.rs index 6bbc1483b2b..14e118a870a 100644 --- a/encodings/alp/src/alp/compute/mod.rs +++ b/encodings/alp/src/alp/compute/mod.rs @@ -23,27 +23,27 @@ mod tests { #[rstest] // Basic float arrays - #[case::f32_array(alp_encode(&PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]), None).unwrap())] - #[case::f64_array(alp_encode(&PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]), None).unwrap())] + #[case::f32_array(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] + #[case::f64_array(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] // Nullable arrays - #[case::nullable_f32(alp_encode(&PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]), None).unwrap())] - #[case::nullable_f64(alp_encode(&PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]), None).unwrap())] + #[case::nullable_f32(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] + #[case::nullable_f64(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] // Edge cases - #[case::single_element(alp_encode(&PrimitiveArray::from_iter([42.42f64]), None).unwrap())] + #[case::single_element(alp_encode(PrimitiveArray::from_iter([42.42f64]).as_view(), None).unwrap())] // Large arrays - #[case::large_f32(alp_encode(&PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)), None).unwrap())] + #[case::large_f32(alp_encode(PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)).as_view(), None).unwrap())] // Arrays with patterns - #[case::repeating_pattern(alp_encode(&PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]), None).unwrap())] - #[case::close_values(alp_encode(&PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]), None).unwrap())] + #[case::repeating_pattern(alp_encode(PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]).as_view(), None).unwrap())] + #[case::close_values(alp_encode(PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]).as_view(), None).unwrap())] fn test_alp_consistency(#[case] array: ALPArray) { test_array_consistency(&array.into_array()); } #[rstest] - #[case::f32_basic(alp_encode(&PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]), None).unwrap())] - #[case::f64_basic(alp_encode(&PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]), None).unwrap())] - #[case::f32_large(alp_encode(&PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)), None).unwrap())] - #[case::f64_large(alp_encode(&PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)), None).unwrap())] + #[case::f32_basic(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] + #[case::f64_basic(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] + #[case::f32_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)).as_view(), None).unwrap())] + #[case::f64_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)).as_view(), None).unwrap())] fn test_alp_binary_numeric(#[case] array: ALPArray) { test_binary_numeric_array(array.into_array()); } diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index cf0b5301684..f3a7d23fbe4 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -55,7 +55,7 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).into_array())] #[case(buffer![42.42f64].into_array())] fn test_take_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(&array.to_primitive(), None).unwrap(); + let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); test_take_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 87f777ee9a2..f57f3d27bf5 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -116,7 +116,7 @@ mod tests { .collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(&parray, None)?; + let alp_encoded = alp_encode(parray.as_view(), None)?; assert!( alp_encoded.patches().is_some(), @@ -165,7 +165,7 @@ mod tests { // Values that encode cleanly without patches. let values: Vec = (0..100).map(|i| i as f64).collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(&parray, None)?; + let alp_encoded = alp_encode(parray.as_view(), None)?; assert!( alp_encoded.patches().is_none(), diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 431b69c125d..c4d6d3b7f02 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -681,7 +681,7 @@ mod test { // Pick a seed that we know will trigger lots of patches. let encoder: alp_rd::RDEncoder = alp_rd::RDEncoder::new(&[seed.powi(-2)]); - let rd_array = encoder.encode(&real_array); + let rd_array = encoder.encode(real_array.as_view()); let decoded = rd_array.as_array().to_primitive(); diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 01eab5d126f..86b6ff9c331 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -65,7 +65,7 @@ mod tests { let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - let alprd = encoder.encode(&arr); + let alprd = encoder.encode(arr.as_view()); let casted = alprd .into_array() @@ -89,7 +89,7 @@ mod tests { PrimitiveArray::from_option_iter([Some(10.0f64), None, Some(10.1), Some(10.2), None]); let values = vec![10.0f64, 10.1, 10.2]; let encoder = RDEncoder::new(&values); - let alprd = encoder.encode(&arr); + let alprd = encoder.encode(arr.as_view()); // Cast to NonNullable should fail since we have nulls let result = alprd @@ -114,31 +114,31 @@ mod tests { let values = vec![1.23f32, 4.56, 7.89, 10.11, 12.13]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::f64({ let values = vec![100.1f64, 200.2, 300.3, 400.4, 500.5]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::single({ let values = vec![42.42f64]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::negative({ let values = vec![0.0f32, -1.5, 2.5, -3.5, 4.5]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::nullable({ let arr = PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]); let values = vec![1.1f32, 2.2, 3.3]; let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] fn test_cast_alprd_conformance(#[case] alprd: crate::alp_rd::ALPRDArray) { test_cast_conformance(&alprd.into_array()); diff --git a/encodings/alp/src/alp_rd/compute/filter.rs b/encodings/alp/src/alp_rd/compute/filter.rs index 8815a049dc6..dfd82695933 100644 --- a/encodings/alp/src/alp_rd/compute/filter.rs +++ b/encodings/alp/src/alp_rd/compute/filter.rs @@ -58,7 +58,7 @@ mod test { #[case(0.1f64, 0.2f64, 3e100f64)] fn test_filter(#[case] a: T, #[case] b: T, #[case] outlier: T) { let array = PrimitiveArray::new(buffer![a, b, outlier], Validity::NonNullable); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); // Make sure that we're testing the exception pathway. assert!(encoded.left_parts_patches().is_some()); @@ -76,7 +76,7 @@ mod test { fn test_filter_simple(#[case] a: T, #[case] b: T, #[case] outlier: T) { test_filter_conformance( &RDEncoder::new(&[a, b]) - .encode(&PrimitiveArray::from_iter([a, b, outlier, b, outlier])) + .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) .into_array(), ); } @@ -87,13 +87,10 @@ mod test { fn test_filter_with_nulls(#[case] a: T, #[case] outlier: T) { test_filter_conformance( &RDEncoder::new(&[a]) - .encode(&PrimitiveArray::from_option_iter([ - Some(a), - None, - Some(outlier), - Some(a), - None, - ])) + .encode( + PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) + .as_view(), + ) .into_array(), ); } diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index 001c44d4825..2773387c738 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -50,7 +50,7 @@ mod tests { fn test_mask_simple(#[case] a: T, #[case] b: T, #[case] outlier: T) { test_mask_conformance( &RDEncoder::new(&[a, b]) - .encode(&PrimitiveArray::from_iter([a, b, outlier, b, outlier])) + .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) .into_array(), ); } @@ -61,13 +61,10 @@ mod tests { fn test_mask_with_nulls(#[case] a: T, #[case] outlier: T) { test_mask_conformance( &RDEncoder::new(&[a]) - .encode(&PrimitiveArray::from_option_iter([ - Some(a), - None, - Some(outlier), - Some(a), - None, - ])) + .encode( + PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) + .as_view(), + ) .into_array(), ); } diff --git a/encodings/alp/src/alp_rd/compute/mod.rs b/encodings/alp/src/alp_rd/compute/mod.rs index 1971dc28b0e..7984a9ce02a 100644 --- a/encodings/alp/src/alp_rd/compute/mod.rs +++ b/encodings/alp/src/alp_rd/compute/mod.rs @@ -23,47 +23,47 @@ mod tests { let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::f64_array({ let values = vec![100.0f64, 100.01, 100.02, 100.03, 100.04]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] // Nullable arrays #[case::nullable_f32({ let values = vec![1.0f32, 1.2, 1.3]; let arr = PrimitiveArray::from_option_iter([Some(1.0f32), None, Some(1.2), Some(1.3), None]); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::nullable_f64({ let values = vec![10.0f64, 10.2, 10.3]; let arr = PrimitiveArray::from_option_iter([Some(10.0f64), None, Some(10.2), Some(10.3), None]); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] // Edge cases #[case::single_element({ let values = vec![42.42f64]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] // Arrays with small deltas (good for RD encoding) #[case::small_deltas({ let values = vec![1000.0f32, 1000.001, 1000.002, 1000.003, 1000.004]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] // Large arrays #[case::large_f32({ let values: Vec = (0..1000).map(|i| 100.0 + i as f32 * 0.01).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] fn test_alp_rd_consistency(#[case] array: ALPRDArray) { test_array_consistency(&array.into_array()); @@ -74,25 +74,25 @@ mod tests { let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::f64_basic({ let values = vec![100.0f64, 100.01, 100.02, 100.03, 100.04]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::f32_large({ let values: Vec = (0..100).map(|i| 50.0 + i as f32 * 0.1).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] #[case::f64_large({ let values: Vec = (0..100).map(|i| 1000.0 + i as f64 * 0.01).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(&arr) + encoder.encode(arr.as_view()) })] fn test_alp_rd_binary_numeric(#[case] array: ALPRDArray) { test_binary_numeric_array(array.into_array()); diff --git a/encodings/alp/src/alp_rd/compute/take.rs b/encodings/alp/src/alp_rd/compute/take.rs index fe12255f063..12cf8e935c1 100644 --- a/encodings/alp/src/alp_rd/compute/take.rs +++ b/encodings/alp/src/alp_rd/compute/take.rs @@ -75,7 +75,7 @@ mod test { use vortex_buffer::buffer; let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); assert!(encoded.left_parts_patches().is_some()); assert!( @@ -99,7 +99,7 @@ mod test { #[case(0.1f64, 0.2f64, 3e100f64)] fn take_with_nulls(#[case] a: T, #[case] b: T, #[case] outlier: T) { let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); assert!(encoded.left_parts_patches().is_some()); assert!( @@ -127,7 +127,7 @@ mod test { fn test_take_conformance_alprd(#[case] a: T, #[case] b: T, #[case] outlier: T) { test_take_conformance( &RDEncoder::new(&[a, b]) - .encode(&PrimitiveArray::from_iter([a, b, outlier, b, outlier])) + .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) .into_array(), ); } @@ -138,13 +138,10 @@ mod test { fn test_take_with_nulls_conformance(#[case] a: T, #[case] outlier: T) { test_take_conformance( &RDEncoder::new(&[a]) - .encode(&PrimitiveArray::from_option_iter([ - Some(a), - None, - Some(outlier), - Some(a), - None, - ])) + .encode( + PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) + .as_view(), + ) .into_array(), ); } diff --git a/encodings/alp/src/alp_rd/mod.rs b/encodings/alp/src/alp_rd/mod.rs index 6202dd799e7..261f95f2ae6 100644 --- a/encodings/alp/src/alp_rd/mod.rs +++ b/encodings/alp/src/alp_rd/mod.rs @@ -25,6 +25,8 @@ use num_traits::Float; use num_traits::One; use num_traits::PrimInt; use rustc_hash::FxBuildHasher; +use vortex_array::ArrayView; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -180,11 +182,11 @@ impl RDEncoder { /// /// Each value will be split into a left and right component, which are compressed individually. // TODO(joe): make fallible - pub fn encode(&self, array: &PrimitiveArray) -> ALPRDArray { + pub fn encode(&self, array: ArrayView<'_, Primitive>) -> ALPRDArray { match_each_alp_float_ptype!(array.ptype(), |P| { self.encode_generic::

(array) }) } - fn encode_generic(&self, array: &PrimitiveArray) -> ALPRDArray + fn encode_generic(&self, array: ArrayView<'_, Primitive>) -> ALPRDArray where T: ALPRDFloat + NativePType, T::UINT: NativePType, diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index ed4dcba2dfd..90f1307f59f 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -79,7 +79,7 @@ mod test { #[case(0.1f64, 0.2f64, 3e100f64)] fn test_slice(#[case] a: T, #[case] b: T, #[case] outlier: T) { let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!(encoded, array); @@ -94,7 +94,7 @@ mod test { #[case] outlier: T, ) { let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!(encoded, array); } @@ -105,7 +105,7 @@ mod test { let b = 0.2f64; let outlier = 3e100f64; let array = PrimitiveArray::from_option_iter([Some(a), Some(b), Some(outlier)]); - let encoded = RDEncoder::new(&[a, b]).encode(&array); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!( encoded, diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index a490a7bd544..5f232f61196 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -49,7 +49,7 @@ fn generate_alp_bit_pack_primitive_array( .map(|_| T::from_usize(rng.random_range(0..10_000)).vortex_expect("")) .collect::(); - let alp = alp_encode(&a, None).vortex_expect(""); + let alp = alp_encode(a.as_view(), None).vortex_expect(""); let encoded = alp.encoded().to_primitive(); diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index e65c373b460..7746cd91168 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -16,7 +16,7 @@ pub fn vortex_fastlanes::bit_transpose::untranspose_validity(validity: &vortex_a pub mod vortex_fastlanes::bitpack_compress -pub fn vortex_fastlanes::bitpack_compress::bit_width_histogram(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::bitpack_compress::bit_width_histogram(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult> pub fn vortex_fastlanes::bitpack_compress::bitpack_encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8, bit_width_freq: core::option::Option<&[usize]>) -> vortex_error::VortexResult @@ -476,7 +476,7 @@ impl vortex_fastlanes::RLE pub const vortex_fastlanes::RLE::ID: vortex_array::array::ArrayId -pub fn vortex_fastlanes::RLE::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray @@ -540,7 +540,7 @@ pub struct vortex_fastlanes::RLEData impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLEData::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult impl vortex_fastlanes::RLEData diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index dfbb0ed7b17..727f13fc35e 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -4,7 +4,9 @@ use fastlanes::BitPacking; use itertools::Itertools; use num_traits::PrimInt; +use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::buffer::BufferHandle; @@ -29,7 +31,7 @@ use crate::BitPackedArray; use crate::bitpack_decompress; pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult { - let bit_width_freq = bit_width_histogram(array)?; + let bit_width_freq = bit_width_histogram(array.as_view())?; let best_bit_width = find_best_bit_width(array.ptype(), &bit_width_freq)?; bitpack_encode(array, best_bit_width, Some(&bit_width_freq)) } @@ -42,7 +44,7 @@ pub fn bitpack_encode( ) -> VortexResult { let bit_width_freq = match bit_width_freq { Some(freq) => freq, - None => &bit_width_histogram(array)?, + None => &bit_width_histogram(array.as_view())?, }; // Check array contains no negative values. @@ -289,18 +291,18 @@ where } } -pub fn bit_width_histogram(array: &PrimitiveArray) -> VortexResult> { +pub fn bit_width_histogram(array: ArrayView<'_, Primitive>) -> VortexResult> { match_each_integer_ptype!(array.ptype(), |P| { bit_width_histogram_typed::

(array) }) } fn bit_width_histogram_typed( - array: &PrimitiveArray, + array: ArrayView<'_, Primitive>, ) -> VortexResult> { let bit_width: fn(T) -> usize = |v: T| (8 * size_of::()) - (PrimInt::leading_zeros(v) as usize); let mut bit_widths = vec![0usize; size_of::() * 8 + 1]; - match array.validity_mask()?.bit_buffer() { + match array.validity_mask().bit_buffer() { AllOr::All => { // All values are valid. for v in array.as_slice::() { diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index cf418ea114e..a44ae69ddbd 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -348,7 +348,7 @@ mod tests { #[test] fn test_rle_serialization() { let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(&primitive).unwrap(); + let rle_array = RLEData::encode(primitive.as_view()).unwrap(); assert_eq!(rle_array.len(), 2048); let original_data = rle_array.as_array().to_primitive(); @@ -383,7 +383,7 @@ mod tests { #[test] fn test_rle_serialization_slice() { let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(&primitive).unwrap(); + let rle_array = RLEData::encode(primitive.as_view()).unwrap(); let sliced = RLE::try_new( rle_array.values().clone(), @@ -442,12 +442,12 @@ mod tests { // Chunk 1 (positions 1024..) is all-null. let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; - // Simulate cascading compression: narrow u16→u8 then re-encode with RLE, + // Simulate cascading compression: narrow u16->u8 then re-encode with RLE, // matching the path taken by the BtrBlocks compressor. let indices_prim = rle.indices().to_primitive().narrow()?; - let re_encoded = RLEData::encode(&indices_prim)?; + let re_encoded = RLEData::encode(indices_prim.as_view())?; // Reconstruct the outer RLE with re-encoded indices. // SAFETY: we only replace the indices child; all other invariants hold. diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index 36ec3a22a03..e19ed3efc8b 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -4,8 +4,10 @@ use std::mem; use fastlanes::RLE as FastLanesRLE; +use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::primitive::NativeValue; @@ -25,8 +27,9 @@ use crate::fill_forward_nulls; impl RLEData { /// Encodes a primitive array of unsigned integers using FastLanes RLE. - pub fn encode(array: &PrimitiveArray) -> VortexResult { - match_each_native_ptype!(array.ptype(), |T| { rle_encode_typed::(array) }) + pub fn encode(array: ArrayView<'_, Primitive>) -> VortexResult { + let array = array.into_owned(); + match_each_native_ptype!(array.ptype(), |T| { rle_encode_typed::(&array) }) } } @@ -166,7 +169,8 @@ mod tests { // u8 let array_u8: Buffer = buffer![1, 1, 2, 2, 3, 3]; let encoded_u8 = - RLEData::encode(&PrimitiveArray::new(array_u8, Validity::NonNullable)).unwrap(); + RLEData::encode(PrimitiveArray::new(array_u8, Validity::NonNullable).as_view()) + .unwrap(); let decoded_u8 = encoded_u8.as_array().to_primitive(); let expected_u8 = PrimitiveArray::from_iter(vec![1u8, 1, 2, 2, 3, 3]); assert_arrays_eq!(decoded_u8, expected_u8); @@ -174,7 +178,8 @@ mod tests { // u16 let array_u16: Buffer = buffer![100, 100, 200, 200]; let encoded_u16 = - RLEData::encode(&PrimitiveArray::new(array_u16, Validity::NonNullable)).unwrap(); + RLEData::encode(PrimitiveArray::new(array_u16, Validity::NonNullable).as_view()) + .unwrap(); let decoded_u16 = encoded_u16.as_array().to_primitive(); let expected_u16 = PrimitiveArray::from_iter(vec![100u16, 100, 200, 200]); assert_arrays_eq!(decoded_u16, expected_u16); @@ -182,7 +187,8 @@ mod tests { // u64 let array_u64: Buffer = buffer![1000, 1000, 2000]; let encoded_u64 = - RLEData::encode(&PrimitiveArray::new(array_u64, Validity::NonNullable)).unwrap(); + RLEData::encode(PrimitiveArray::new(array_u64, Validity::NonNullable).as_view()) + .unwrap(); let decoded_u64 = encoded_u64.as_array().to_primitive(); let expected_u64 = PrimitiveArray::from_iter(vec![1000u64, 1000, 2000]); assert_arrays_eq!(decoded_u64, expected_u64); @@ -191,14 +197,16 @@ mod tests { #[test] fn test_length() { let values: Buffer = buffer![1, 1, 2, 2, 2, 3]; - let encoded = RLEData::encode(&PrimitiveArray::new(values, Validity::NonNullable)).unwrap(); + let encoded = + RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.len(), 6); } #[test] fn test_empty_length() { let values: Buffer = Buffer::empty(); - let encoded = RLEData::encode(&PrimitiveArray::new(values, Validity::NonNullable)).unwrap(); + let encoded = + RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.len(), 0); assert_eq!(encoded.values().len(), 0); @@ -208,7 +216,8 @@ mod tests { fn test_single_value() { let values: Buffer = vec![42; 2000].into_iter().collect(); - let encoded = RLEData::encode(&PrimitiveArray::new(values, Validity::NonNullable)).unwrap(); + let encoded = + RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.values().len(), 2); // 2 chunks, each storing value 42 let decoded = encoded.as_array().to_primitive(); // Verify round-trip @@ -220,7 +229,8 @@ mod tests { fn test_all_different() { let values: Buffer = (0u8..=255).collect(); - let encoded = RLEData::encode(&PrimitiveArray::new(values, Validity::NonNullable)).unwrap(); + let encoded = + RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.values().len(), 256); let decoded = encoded.as_array().to_primitive(); // Verify round-trip @@ -234,7 +244,7 @@ mod tests { let values: Buffer = (0..1500).map(|i| (i / 100) as u32).collect(); let array = PrimitiveArray::new(values, Validity::NonNullable); - let encoded = RLEData::encode(&array).unwrap(); + let encoded = RLEData::encode(array.as_view()).unwrap(); assert_eq!(encoded.len(), 1500); assert_arrays_eq!(encoded, array); @@ -248,7 +258,7 @@ mod tests { let values: Buffer = (0..2048).map(|i| (i / 100) as u32).collect(); let array = PrimitiveArray::new(values, Validity::NonNullable); - let encoded = RLEData::encode(&array).unwrap(); + let encoded = RLEData::encode(array.as_view()).unwrap(); assert_eq!(encoded.len(), 2048); assert_arrays_eq!(encoded, array); @@ -269,7 +279,7 @@ mod tests { #[case::f64((-2000..2000).map(|i| i as f64).collect::>())] fn test_roundtrip_primitive_types(#[case] values: Buffer) { let primitive = values.clone().into_array().to_primitive(); - let result = RLEData::encode(&primitive).unwrap(); + let result = RLEData::encode(primitive.as_view()).unwrap(); let decoded = result.as_array().to_primitive(); let expected = PrimitiveArray::new( values, @@ -306,7 +316,7 @@ mod tests { fn test_encode_all_null_chunk() -> VortexResult<()> { let values: Vec> = vec![None; FL_CHUNK_SIZE]; let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let decoded = with_masked_constant_indices(&rle)?; assert_arrays_eq!(decoded, original); Ok(()) @@ -318,7 +328,7 @@ mod tests { let mut values: Vec> = vec![None; 2 * FL_CHUNK_SIZE]; values[FL_CHUNK_SIZE + 100] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let decoded = with_masked_constant_indices(&rle)?; assert_arrays_eq!(decoded, original); Ok(()) @@ -330,7 +340,7 @@ mod tests { let mut values: Vec> = vec![None; FL_CHUNK_SIZE]; values[1000] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let decoded = with_masked_constant_indices(&rle)?; assert_arrays_eq!(decoded, original); Ok(()) @@ -352,7 +362,7 @@ mod tests { values[pos] = Some(-1); } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let decoded = with_masked_constant_indices(&rle)?; assert_arrays_eq!(decoded, original); Ok(()) @@ -396,7 +406,7 @@ mod tests { fn test_random_invalid_indices_all_null_chunk() -> VortexResult<()> { let values: Vec> = vec![None; FL_CHUNK_SIZE]; let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let clobbered = with_random_invalid_indices(&rle)?; assert_arrays_eq!(clobbered, original); Ok(()) @@ -409,7 +419,7 @@ mod tests { values[500] = Some(20); values[1000] = Some(30); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let clobbered = with_random_invalid_indices(&rle)?; assert_arrays_eq!(clobbered, original); Ok(()) @@ -423,7 +433,7 @@ mod tests { values[500] = Some(20); values[FL_CHUNK_SIZE + 100] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let clobbered = with_random_invalid_indices(&rle)?; assert_arrays_eq!(clobbered, original); Ok(()) @@ -438,7 +448,7 @@ mod tests { values[i] = Some(i as u32); } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let clobbered = with_random_invalid_indices(&rle)?; assert_arrays_eq!(clobbered, original); Ok(()) @@ -454,7 +464,7 @@ mod tests { values[i] = None; } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(&original)?; + let rle = RLEData::encode(original.as_view())?; let clobbered = with_random_invalid_indices(&rle)?; assert_arrays_eq!(clobbered, original); Ok(()) @@ -468,7 +478,7 @@ mod tests { #[case(vec![0f64, -0f64])] fn test_float_zeros(#[case] values: Vec) { let primitive = PrimitiveArray::from_iter(values); - let rle = RLEData::encode(&primitive).unwrap(); + let rle = RLEData::encode(primitive.as_view()).unwrap(); let decoded = rle.as_array().to_primitive(); assert_arrays_eq!(primitive, decoded); } diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 95286d2f3e7..2123d74f136 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -60,7 +60,7 @@ mod tests { use crate::rle::RLEArray; fn rle(primitive: &PrimitiveArray) -> RLEArray { - RLEData::encode(primitive).unwrap() + RLEData::encode(primitive.as_view()).unwrap() } #[test] diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index c0e19efd3af..9587603aacc 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -16,7 +16,7 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; -use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -244,7 +244,7 @@ impl RLE { } /// Encode a primitive array using FastLanes RLE. - pub fn encode(array: &PrimitiveArray) -> VortexResult { + pub fn encode(array: ArrayView<'_, Primitive>) -> VortexResult { RLEData::encode(array) } } diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 25069ecccc1..4d54cd9cf32 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -170,7 +170,7 @@ mod tests { let expected: Vec = (0..3000).map(|i| (i / 50) as u16).collect(); let array = values.into_array(); - let encoded = RLEData::encode(&array.to_primitive()).unwrap(); + let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); // Access scalars from multiple chunks. for &idx in &[1023, 1024, 1025, 2047, 2048, 2049] { @@ -276,7 +276,7 @@ mod tests { let expected: Vec = (0..2100).map(|i| (i / 100) as u32).collect(); let array = values.into_array(); - let encoded = RLEData::encode(&array.to_primitive()).unwrap(); + let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); // Slice across first and second chunk. let slice = encoded.slice(500..1500).unwrap(); diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 2cf1233c7c2..07307c38196 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -6,7 +6,7 @@ impl vortex_pco::Pco pub const vortex_pco::Pco::ID: vortex_array::array::ArrayId -pub fn vortex_pco::Pco::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: usize, values_per_page: usize) -> vortex_error::VortexResult +pub fn vortex_pco::Pco::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_pco::Pco @@ -90,7 +90,7 @@ pub fn vortex_pco::PcoData::decompress(&self, unsliced_validity: &vortex_array:: pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult -pub fn vortex_pco::PcoData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: usize, values_per_page: usize) -> vortex_error::VortexResult +pub fn vortex_pco::PcoData::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize) -> vortex_error::VortexResult pub fn vortex_pco::PcoData::is_empty(&self) -> bool diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 861f80cc6eb..8f83dbd8fa8 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -255,9 +255,9 @@ pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { } } -fn collect_valid(parray: &PrimitiveArray) -> VortexResult { - let mask = parray.validity_mask()?; - Ok(parray.filter(mask)?.to_primitive()) +fn collect_valid(parray: ArrayView<'_, Primitive>) -> VortexResult { + let mask = parray.array().validity_mask()?; + Ok(parray.array().filter(mask)?.to_primitive()) } pub(crate) fn vortex_err_from_pco(err: PcoError) -> VortexError { @@ -290,7 +290,7 @@ impl Pco { /// Compress a primitive array using pcodec. pub fn from_primitive( - parray: &PrimitiveArray, + parray: ArrayView<'_, Primitive>, level: usize, values_per_page: usize, ) -> VortexResult { @@ -399,7 +399,7 @@ impl PcoData { } pub fn from_primitive( - parray: &PrimitiveArray, + parray: ArrayView<'_, Primitive>, level: usize, values_per_page: usize, ) -> VortexResult { @@ -407,7 +407,7 @@ impl PcoData { } pub(crate) fn from_primitive_with_values_per_chunk( - parray: &PrimitiveArray, + parray: ArrayView<'_, Primitive>, level: usize, values_per_chunk: usize, values_per_page: usize, @@ -485,7 +485,7 @@ impl PcoData { a.encoding_id() ) })?; - Self::from_primitive(&parray, level, nums_per_page) + Self::from_primitive(parray.as_view(), level, nums_per_page) } pub fn decompress( @@ -658,7 +658,7 @@ mod tests { buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([false, true, true, true, true, false]), ); - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); assert_arrays_eq!( pco, PrimitiveArray::from_option_iter([ diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 21598191fec..c75e3447a72 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -68,7 +68,7 @@ mod tests { #[test] fn test_cast_pco_f32_to_f64() { let values = PrimitiveArray::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0]); - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); let casted = pco .into_array() @@ -89,7 +89,7 @@ mod tests { fn test_cast_pco_nullability_change() { // Test casting from NonNullable to Nullable let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); let casted = pco .into_array() @@ -107,7 +107,7 @@ mod tests { buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([true, true, true, true, true, true]), ); - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); let sliced = pco.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -130,7 +130,7 @@ mod tests { Some(50), Some(60), ]); - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); let sliced = pco.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -164,7 +164,7 @@ mod tests { Validity::NonNullable, ))] fn test_cast_pco_conformance(#[case] values: PrimitiveArray) { - let pco = Pco::from_primitive(&values, 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); test_cast_conformance(&pco.into_array()); } } diff --git a/encodings/pco/src/compute/mod.rs b/encodings/pco/src/compute/mod.rs index 990a54179ae..62f1ba1fd23 100644 --- a/encodings/pco/src/compute/mod.rs +++ b/encodings/pco/src/compute/mod.rs @@ -15,42 +15,42 @@ mod tests { fn pco_f32() -> PcoArray { let values = PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_f64() -> PcoArray { let values = PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_i32() -> PcoArray { let values = PrimitiveArray::from_iter([100i32, 200, 300, 400, 500]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_u64() -> PcoArray { let values = PrimitiveArray::from_iter([1000u64, 2000, 3000, 4000]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_i16() -> PcoArray { let values = PrimitiveArray::from_iter([10i16, 20, 30, 40, 50]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_i32_alt() -> PcoArray { let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_single() -> PcoArray { let values = PrimitiveArray::from_iter([42.42f64]); - Pco::from_primitive(&values, 0, 128).unwrap() + Pco::from_primitive(values.as_view(), 0, 128).unwrap() } fn pco_large() -> PcoArray { let values = PrimitiveArray::from_iter(0u32..1000); - Pco::from_primitive(&values, 3, 128).unwrap() + Pco::from_primitive(values.as_view(), 3, 128).unwrap() } #[rstest] diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index a431037948a..b9a0f87e52d 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -44,7 +44,7 @@ use crate::Pco; fn test_compress_decompress() { let data: Vec = (0..200).collect(); let array = PrimitiveArray::from_iter(data.clone()); - let compressed = Pco::from_primitive(&array, 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); // this data should be compressible assert!(compressed.pages.len() < array.into_array().nbytes() as usize); @@ -71,7 +71,7 @@ fn test_compress_decompress() { #[test] fn test_compress_decompress_small() { let array = PrimitiveArray::from_option_iter([None, Some(1)]); - let compressed = Pco::from_primitive(&array, 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); let expected = array.into_array(); assert_arrays_eq!(compressed, expected); @@ -89,7 +89,7 @@ fn test_compress_decompress_small() { fn test_empty() { let data: Vec = vec![]; let array = PrimitiveArray::from_iter(data.clone()); - let compressed = Pco::from_primitive(&array, 3, 100).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 100).unwrap(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unsliced_validity = child_to_validity( &compressed.as_ref().slots()[0], @@ -116,7 +116,7 @@ fn test_validity_and_multiple_chunks_and_pages() { let compressed = Pco::try_new( array.dtype().clone(), PcoData::from_primitive_with_values_per_chunk( - &array, + array.as_view(), compression_level, values_per_chunk, values_per_page, @@ -163,7 +163,7 @@ fn test_validity_vtable() { Buffer::from(data), Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); - let compressed = Pco::from_primitive(&array, 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); assert_eq!( compressed.as_array().validity_mask().unwrap(), Mask::from_iter(mask_bools) @@ -177,7 +177,7 @@ fn test_validity_vtable() { #[test] fn test_serde() -> VortexResult<()> { let data: PrimitiveArray = (0i32..1_000_000).collect(); - let pco = Pco::from_primitive(&data, 3, 100)?.into_array(); + let pco = Pco::from_primitive(data.as_view(), 3, 100)?.into_array(); let context = ArrayContext::empty(); diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index aaacb5d3963..6bb8f69d1d2 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -124,6 +124,6 @@ pub vortex_sequence::SequenceDataParts::ptype: vortex_array::dtype::ptype::PType pub fn vortex_sequence::initialize(session: &vortex_session::VortexSession) -pub fn vortex_sequence::sequence_encode(primitive_array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult> +pub fn vortex_sequence::sequence_encode(primitive_array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult> pub type vortex_sequence::SequenceArray = vortex_array::array::typed::Array diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index 6321aa3df78..98bd12bb027 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -6,7 +6,9 @@ use std::ops::Add; use num_traits::CheckedAdd; use num_traits::CheckedSub; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; @@ -84,13 +86,15 @@ pub fn sequence_decompress(array: &SequenceArray) -> VortexResult { /// 3. The array is representable as a sequence `A[i] = base + i * multiplier` for multiplier != 0. /// 4. The sequence has no deviations from the equation, this could be fixed with patches. However, /// we might want a different array for that since sequence provide fast access. -pub fn sequence_encode(primitive_array: &PrimitiveArray) -> VortexResult> { +pub fn sequence_encode( + primitive_array: ArrayView<'_, Primitive>, +) -> VortexResult> { if primitive_array.is_empty() { // we cannot encode an empty array return Ok(None); } - if !primitive_array.all_valid()? { + if !primitive_array.array().all_valid()? { return Ok(None); } @@ -153,7 +157,7 @@ mod tests { #[test] fn test_encode_array_success() { let primitive_array = PrimitiveArray::from_iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let encoded = sequence_encode(&primitive_array).unwrap(); + let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); @@ -162,7 +166,7 @@ mod tests { #[test] fn test_encode_array_1_success() { let primitive_array = PrimitiveArray::from_iter([0]); - let encoded = sequence_encode(&primitive_array).unwrap(); + let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); @@ -172,7 +176,7 @@ mod tests { fn test_encode_array_fail() { let primitive_array = PrimitiveArray::from_iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); - let encoded = sequence_encode(&primitive_array).unwrap(); + let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_none()); } @@ -180,14 +184,14 @@ mod tests { fn test_encode_array_fail_oob() { let primitive_array = PrimitiveArray::from_iter(vec![100i8; 1000]); - let encoded = sequence_encode(&primitive_array).unwrap(); + let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_none()); } #[test] fn test_encode_all_u8_values() { let primitive_array = PrimitiveArray::from_iter(0u8..=255); - let encoded = sequence_encode(&primitive_array).unwrap(); + let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index a366ce1d631..d94894a7ca0 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -120,6 +120,6 @@ pub fn T::ptype(&self) -> vortex_array::dtype::ptype::PType pub fn vortex_zigzag::zigzag_decode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_array::arrays::primitive::vtable::PrimitiveArray -pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult +pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult pub type vortex_zigzag::ZigZagArray = vortex_array::array::typed::Array diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 34b1d2d19f6..851b40b438b 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -282,7 +282,7 @@ mod test { let array = buffer![1i32, -5i32, 2, 3, 4, 5, 6, 7, 8, 9, 10] .into_array() .to_primitive(); - let zigzag = zigzag_encode(array.clone())?; + let zigzag = zigzag_encode(array.as_view())?; assert_eq!( zigzag.statistics().compute_max::(), diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 998a39f2b49..3c96c8f5778 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; @@ -15,7 +17,8 @@ use zigzag::ZigZag as ExternalZigZag; use crate::ZigZag; use crate::ZigZagArray; -pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { +pub fn zigzag_encode(parray: ArrayView<'_, Primitive>) -> VortexResult { + let parray = parray.into_owned(); let validity = parray.validity()?; let encoded = match parray.ptype() { PType::I8 => zigzag_encode_primitive::(parray.into_buffer_mut(), validity), @@ -83,7 +86,7 @@ mod test { #[test] fn test_compress_i8() { - let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i8..100)) + let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i8..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); @@ -94,7 +97,7 @@ mod test { } #[test] fn test_compress_i16() { - let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i16..100)) + let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i16..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); @@ -105,7 +108,7 @@ mod test { } #[test] fn test_compress_i32() { - let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i32..100)) + let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i32..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); @@ -116,7 +119,7 @@ mod test { } #[test] fn test_compress_i64() { - let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i64..100)) + let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i64..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index 7f6741d9aa1..a71eee7a838 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -42,7 +42,7 @@ mod tests { #[test] fn test_cast_zigzag_i32_to_i64() { let values = PrimitiveArray::from_iter([-100i32, -1, 0, 1, 100]); - let zigzag = zigzag_encode(values).unwrap(); + let zigzag = zigzag_encode(values.as_view()).unwrap(); let casted = zigzag .into_array() @@ -68,7 +68,7 @@ mod tests { fn test_cast_zigzag_width_changes() { // Test i32 to i16 (narrowing) let values = PrimitiveArray::from_iter([100i32, -50, 0, 25, -100]); - let zigzag = zigzag_encode(values).unwrap(); + let zigzag = zigzag_encode(values.as_view()).unwrap(); let casted = zigzag .into_array() @@ -87,7 +87,7 @@ mod tests { // Test i16 to i64 (widening) let values16 = PrimitiveArray::from_iter([1000i16, -500, 0, 250, -1000]); - let zigzag16 = zigzag_encode(values16).unwrap(); + let zigzag16 = zigzag_encode(values16.as_view()).unwrap(); let casted64 = zigzag16 .into_array() @@ -109,7 +109,7 @@ mod tests { fn test_cast_zigzag_nullable() { let values = PrimitiveArray::from_option_iter([Some(-10i32), None, Some(0), Some(10), None]); - let zigzag = zigzag_encode(values).unwrap(); + let zigzag = zigzag_encode(values.as_view()).unwrap(); let casted = zigzag .into_array() @@ -122,10 +122,10 @@ mod tests { } #[rstest] - #[case(zigzag_encode(PrimitiveArray::from_iter([-100i32, -50, -1, 0, 1, 50, 100])).unwrap())] - #[case(zigzag_encode(PrimitiveArray::from_iter([-1000i64, -1, 0, 1, 1000])).unwrap())] - #[case(zigzag_encode(PrimitiveArray::from_option_iter([Some(-5i16), None, Some(0), Some(5), None])).unwrap())] - #[case(zigzag_encode(PrimitiveArray::from_iter([i32::MIN, -1, 0, 1, i32::MAX])).unwrap())] + #[case(zigzag_encode(PrimitiveArray::from_iter([-100i32, -50, -1, 0, 1, 50, 100]).as_view()).unwrap())] + #[case(zigzag_encode(PrimitiveArray::from_iter([-1000i64, -1, 0, 1, 1000]).as_view()).unwrap())] + #[case(zigzag_encode(PrimitiveArray::from_option_iter([Some(-5i16), None, Some(0), Some(5), None]).as_view()).unwrap())] + #[case(zigzag_encode(PrimitiveArray::from_iter([i32::MIN, -1, 0, 1, i32::MAX]).as_view()).unwrap())] fn test_cast_zigzag_conformance(#[case] array: ZigZagArray) { test_cast_conformance(&array.into_array()); } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 71938dbcc18..c038e98d629 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -90,10 +90,9 @@ mod tests { #[test] pub fn nullable_scalar_at() -> VortexResult<()> { - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-189, -160, 1], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-189, -160, 1], Validity::AllValid).as_view(), + )?; assert_eq!( zigzag.scalar_at(1)?, Scalar::primitive(-160, Nullability::Nullable) @@ -103,30 +102,30 @@ mod tests { #[test] fn take_zigzag() -> VortexResult<()> { - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-189, -160, 1], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-189, -160, 1], Validity::AllValid).as_view(), + )?; let indices = buffer![0, 2].into_array(); let actual = zigzag.take(indices).unwrap(); let expected = - zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid))?.into_array(); + zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid).as_view())? + .into_array(); assert_arrays_eq!(actual, expected); Ok(()) } #[test] fn filter_zigzag() -> VortexResult<()> { - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-189, -160, 1], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-189, -160, 1], Validity::AllValid).as_view(), + )?; let filter_mask = BitBuffer::from(vec![true, false, true]).into(); let actual = zigzag.filter(filter_mask).unwrap(); let expected = - zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid))?.into_array(); + zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid).as_view())? + .into_array(); assert_arrays_eq!(actual, expected); Ok(()) } @@ -136,23 +135,25 @@ mod tests { use vortex_array::compute::conformance::filter::test_filter_conformance; // Test with i32 values - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-189i32, -160, 1, 42, -73], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-189i32, -160, 1, 42, -73], Validity::AllValid).as_view(), + )?; test_filter_conformance(&zigzag.into_array()); // Test with i64 values - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![1000i64, -2000, 3000, -4000, 5000], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new( + buffer![1000i64, -2000, 3000, -4000, 5000], + Validity::AllValid, + ) + .as_view(), + )?; test_filter_conformance(&zigzag.into_array()); // Test with nullable values let array = PrimitiveArray::from_option_iter([Some(-10i16), None, Some(20), Some(-30), None]); - let zigzag = zigzag_encode(array)?; + let zigzag = zigzag_encode(array.as_view())?; test_filter_conformance(&zigzag.into_array()); Ok(()) } @@ -162,17 +163,16 @@ mod tests { use vortex_array::compute::conformance::mask::test_mask_conformance; // Test with i32 values - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-100i32, 200, -300, 400, -500], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-100i32, 200, -300, 400, -500], Validity::AllValid) + .as_view(), + )?; test_mask_conformance(&zigzag.into_array()); // Test with i8 values - let zigzag = zigzag_encode(PrimitiveArray::new( - buffer![-127i8, 0, 127, -1, 1], - Validity::AllValid, - ))?; + let zigzag = zigzag_encode( + PrimitiveArray::new(buffer![-127i8, 0, 127, -1, 1], Validity::AllValid).as_view(), + )?; test_mask_conformance(&zigzag.into_array()); Ok(()) } @@ -186,36 +186,36 @@ mod tests { fn test_take_zigzag_conformance(#[case] array: ArrayRef) -> VortexResult<()> { use vortex_array::compute::conformance::take::test_take_conformance; - let zigzag = zigzag_encode(array.to_primitive())?; + let zigzag = zigzag_encode(array.to_primitive().as_view())?; test_take_conformance(&zigzag.into_array()); Ok(()) } #[rstest] // Basic ZigZag arrays - #[case::zigzag_i8(zigzag_encode(PrimitiveArray::from_iter([-128i8, -1, 0, 1, 127])).unwrap())] - #[case::zigzag_i16(zigzag_encode(PrimitiveArray::from_iter([-1000i16, -100, 0, 100, 1000])).unwrap())] - #[case::zigzag_i32(zigzag_encode(PrimitiveArray::from_iter([-100000i32, -1000, 0, 1000, 100000])).unwrap())] - #[case::zigzag_i64(zigzag_encode(PrimitiveArray::from_iter([-1000000i64, -10000, 0, 10000, 1000000])).unwrap())] + #[case::zigzag_i8(zigzag_encode(PrimitiveArray::from_iter([-128i8, -1, 0, 1, 127]).as_view()).unwrap())] + #[case::zigzag_i16(zigzag_encode(PrimitiveArray::from_iter([-1000i16, -100, 0, 100, 1000]).as_view()).unwrap())] + #[case::zigzag_i32(zigzag_encode(PrimitiveArray::from_iter([-100000i32, -1000, 0, 1000, 100000]).as_view()).unwrap())] + #[case::zigzag_i64(zigzag_encode(PrimitiveArray::from_iter([-1000000i64, -10000, 0, 10000, 1000000]).as_view()).unwrap())] // Nullable arrays - #[case::zigzag_nullable_i32(zigzag_encode(PrimitiveArray::from_option_iter([Some(-100i32), None, Some(0), Some(100), None])).unwrap())] - #[case::zigzag_nullable_i64(zigzag_encode(PrimitiveArray::from_option_iter([Some(-1000i64), None, Some(0), Some(1000), None])).unwrap())] + #[case::zigzag_nullable_i32(zigzag_encode(PrimitiveArray::from_option_iter([Some(-100i32), None, Some(0), Some(100), None]).as_view()).unwrap())] + #[case::zigzag_nullable_i64(zigzag_encode(PrimitiveArray::from_option_iter([Some(-1000i64), None, Some(0), Some(1000), None]).as_view()).unwrap())] // Edge cases - #[case::zigzag_single(zigzag_encode(PrimitiveArray::from_iter([-42i32])).unwrap())] - #[case::zigzag_alternating(zigzag_encode(PrimitiveArray::from_iter([-1i32, 1, -2, 2, -3, 3])).unwrap())] + #[case::zigzag_single(zigzag_encode(PrimitiveArray::from_iter([-42i32]).as_view()).unwrap())] + #[case::zigzag_alternating(zigzag_encode(PrimitiveArray::from_iter([-1i32, 1, -2, 2, -3, 3]).as_view()).unwrap())] // Large arrays - #[case::zigzag_large_i32(zigzag_encode(PrimitiveArray::from_iter(-500..500)).unwrap())] - #[case::zigzag_large_i64(zigzag_encode(PrimitiveArray::from_iter((-1000..1000).map(|i| i as i64 * 100))).unwrap())] + #[case::zigzag_large_i32(zigzag_encode(PrimitiveArray::from_iter(-500..500).as_view()).unwrap())] + #[case::zigzag_large_i64(zigzag_encode(PrimitiveArray::from_iter((-1000..1000).map(|i| i as i64 * 100)).as_view()).unwrap())] fn test_zigzag_consistency(#[case] array: ZigZagArray) { test_array_consistency(&array.into_array()); } #[rstest] - #[case::zigzag_i8_basic(zigzag_encode(PrimitiveArray::from_iter([-10i8, -5, 0, 5, 10])).unwrap())] - #[case::zigzag_i16_basic(zigzag_encode(PrimitiveArray::from_iter([-100i16, -50, 0, 50, 100])).unwrap())] - #[case::zigzag_i32_basic(zigzag_encode(PrimitiveArray::from_iter([-1000i32, -500, 0, 500, 1000])).unwrap())] - #[case::zigzag_i64_basic(zigzag_encode(PrimitiveArray::from_iter([-10000i64, -5000, 0, 5000, 10000])).unwrap())] - #[case::zigzag_i32_large(zigzag_encode(PrimitiveArray::from_iter((-50..50).map(|i| i * 10))).unwrap())] + #[case::zigzag_i8_basic(zigzag_encode(PrimitiveArray::from_iter([-10i8, -5, 0, 5, 10]).as_view()).unwrap())] + #[case::zigzag_i16_basic(zigzag_encode(PrimitiveArray::from_iter([-100i16, -50, 0, 50, 100]).as_view()).unwrap())] + #[case::zigzag_i32_basic(zigzag_encode(PrimitiveArray::from_iter([-1000i32, -500, 0, 500, 1000]).as_view()).unwrap())] + #[case::zigzag_i64_basic(zigzag_encode(PrimitiveArray::from_iter([-10000i64, -5000, 0, 5000, 10000]).as_view()).unwrap())] + #[case::zigzag_i32_large(zigzag_encode(PrimitiveArray::from_iter((-50..50).map(|i| i * 10)).as_view()).unwrap())] fn test_zigzag_binary_numeric(#[case] array: ZigZagArray) { test_binary_numeric_array(array.into_array()); } diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 6c3dae9ab64..0af2b30974e 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -105,7 +105,7 @@ impl Scheme for ALPScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let alp_encoded = alp_encode(&data.array_as_primitive(), None)?; + let alp_encoded = alp_encode(data.array_as_primitive(), None)?; // Compress the ALP ints. let compressed_alp_ints = @@ -174,7 +174,7 @@ impl Scheme for ALPRDScheme { ptype => vortex_panic!("cannot ALPRD compress ptype {ptype}"), }; - let alp_rd = encoder.encode(&primitive_array); + let alp_rd = encoder.encode(primitive_array); let dtype = alp_rd.dtype().clone(); let right_bit_width = alp_rd.right_bit_width(); let mut parts = ALPRDArrayOwnedExt::into_data_parts(alp_rd); @@ -289,7 +289,7 @@ impl Scheme for PcoScheme { _ctx: CompressorContext, ) -> VortexResult { Ok(vortex_pco::Pco::from_primitive( - &data.array_as_primitive(), + data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, )? diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 1fd75c61910..100ebdfe68c 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -328,15 +328,16 @@ impl Scheme for BitPackingScheme { ) -> VortexResult { let primitive_array = data.array_as_primitive(); - let histogram = bit_width_histogram(&primitive_array)?; + let histogram = bit_width_histogram(primitive_array)?; let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; // If best bw is determined to be the current bit-width, return the original array. if bw as usize == primitive_array.ptype().bit_width() { - return Ok(primitive_array.into_array()); + return Ok(primitive_array.array().clone()); } // Otherwise we can bitpack the array. + let primitive_array = primitive_array.into_owned(); let packed = bitpack_encode(&primitive_array, bw, Some(&histogram))?; let packed_stats = packed.statistics().to_owned(); @@ -615,7 +616,7 @@ impl Scheme for RunEndScheme { ctx: CompressorContext, ) -> VortexResult { // Run-end encode the ends. - let (ends, values) = runend_encode(data.array_as_primitive().as_view()); + let (ends, values) = runend_encode(data.array_as_primitive()); let compressed_values = compressor.compress_child(&values.to_primitive().into_array(), &ctx, self.id(), 0)?; @@ -690,7 +691,7 @@ impl Scheme for SequenceScheme { // why do we divide the ratio by 2??? CompressionEstimate::Estimate(Box::new(|_compressor, data, _ctx| { - let Some(encoded) = sequence_encode(&data.array_as_primitive())? else { + let Some(encoded) = sequence_encode(data.array_as_primitive())? else { // If we are unable to sequence encode this array, make sure we skip. return Ok(CompressionEstimate::Skip); }; @@ -713,7 +714,7 @@ impl Scheme for SequenceScheme { if stats.null_count() > 0 { vortex_bail!("sequence encoding does not support nulls"); } - sequence_encode(&data.array_as_primitive())? + sequence_encode(data.array_as_primitive())? .ok_or_else(|| vortex_err!("cannot sequence encode array")) } } @@ -750,7 +751,7 @@ impl Scheme for PcoScheme { _ctx: CompressorContext, ) -> VortexResult { Ok(vortex_pco::Pco::from_primitive( - &data.array_as_primitive(), + data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, )? @@ -765,8 +766,7 @@ pub(crate) fn rle_compress( data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let array = data.array_as_primitive(); - let rle_array = RLE::encode(&array)?; + let rle_array = RLE::encode(data.array_as_primitive())?; let compressed_values = compressor.compress_child( &rle_array.values().to_primitive().into_array(), diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 7aec016534e..331fad9f8cb 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -82,7 +82,7 @@ impl Scheme for FSSTScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let utf8 = data.array_as_utf8(); + let utf8 = data.array_as_utf8().into_owned(); let compressor_fsst = fsst_train_compressor(&utf8); let fsst = fsst_compress(&utf8, utf8.len(), utf8.dtype(), &compressor_fsst); @@ -225,7 +225,7 @@ impl Scheme for ZstdScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { - let compacted = data.array_as_utf8().compact_buffers()?; + let compacted = data.array_as_utf8().into_owned().compact_buffers()?; Ok(vortex_zstd::Zstd::from_var_bin_view_without_dict(&compacted, 3, 8192)?.into_array()) } } diff --git a/vortex-compressor/benches/dict_encode.rs b/vortex-compressor/benches/dict_encode.rs index 1c47e951fb8..a1a22636e18 100644 --- a/vortex-compressor/benches/dict_encode.rs +++ b/vortex-compressor/benches/dict_encode.rs @@ -43,7 +43,7 @@ fn encode_specialized(bencher: Bencher) { let stats = IntegerStats::generate(&array); bencher .with_inputs(|| &stats) - .bench_refs(|stats| integer_dictionary_encode(array.clone(), stats)); + .bench_refs(|stats| integer_dictionary_encode(array.as_view(), stats)); } fn main() { diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 0f16ccc61ab..694f3d676f0 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -268,9 +268,9 @@ pub fn vortex_compressor::builtins::StringDictScheme::scheme_name(&self) -> &'st pub fn vortex_compressor::builtins::StringDictScheme::stats_options(&self) -> vortex_compressor::stats::GenerateStatsOptions -pub fn vortex_compressor::builtins::float_dictionary_encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray, stats: &vortex_compressor::stats::FloatStats) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::float_dictionary_encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, stats: &vortex_compressor::stats::FloatStats) -> vortex_error::VortexResult -pub fn vortex_compressor::builtins::integer_dictionary_encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray, stats: &vortex_compressor::stats::IntegerStats) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::integer_dictionary_encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, stats: &vortex_compressor::stats::IntegerStats) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::is_float_primitive(canonical: &vortex_array::canonical::Canonical) -> bool @@ -668,9 +668,9 @@ impl vortex_compressor::stats::ArrayAndStats pub fn vortex_compressor::stats::ArrayAndStats::array(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_compressor::stats::ArrayAndStats::array_as_primitive(&self) -> vortex_array::arrays::primitive::vtable::PrimitiveArray +pub fn vortex_compressor::stats::ArrayAndStats::array_as_primitive(&self) -> vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive> -pub fn vortex_compressor::stats::ArrayAndStats::array_as_utf8(&self) -> vortex_array::arrays::varbinview::vtable::VarBinViewArray +pub fn vortex_compressor::stats::ArrayAndStats::array_as_utf8(&self) -> vortex_array::array::view::ArrayView<'_, vortex_array::arrays::varbinview::vtable::VarBinView> pub fn vortex_compressor::stats::ArrayAndStats::array_len(&self) -> usize diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index c2cdbeafaf2..eca60891568 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -7,9 +7,11 @@ //! external compatibility. use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; use vortex_array::arrays::dict::DictArraySlotsExt; @@ -182,7 +184,10 @@ macro_rules! typed_encode { /// # Errors /// /// Returns an error if unable to compute validity. -pub fn dictionary_encode(array: PrimitiveArray, stats: &FloatStats) -> VortexResult { +pub fn dictionary_encode( + array: ArrayView<'_, Primitive>, + stats: &FloatStats, +) -> VortexResult { match stats.erased() { FloatErasedStats::F16(typed) => typed_encode!(array, stats, typed, f16), FloatErasedStats::F32(typed) => typed_encode!(array, stats, typed, f32), @@ -260,7 +265,7 @@ mod tests { count_distinct_values: true, }, ); - let dict_array = dictionary_encode(array, &stats).unwrap(); + let dict_array = dictionary_encode(array.as_view(), &stats).unwrap(); assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 8671f9d79e6..1fcf9c62903 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -7,9 +7,11 @@ //! for external compatibility. use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; +use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; use vortex_array::arrays::dict::DictArraySlotsExt; @@ -177,7 +179,10 @@ macro_rules! typed_encode { clippy::cognitive_complexity, reason = "complexity from match on all integer types" )] -pub fn dictionary_encode(array: PrimitiveArray, stats: &IntegerStats) -> VortexResult { +pub fn dictionary_encode( + array: ArrayView<'_, Primitive>, + stats: &IntegerStats, +) -> VortexResult { match stats.erased() { IntegerErasedStats::U8(typed) => typed_encode!(array, stats, typed, u8), IntegerErasedStats::U16(typed) => typed_encode!(array, stats, typed, u16), @@ -265,7 +270,7 @@ mod tests { count_distinct_values: true, }, ); - let dict_array = dictionary_encode(array, &stats).unwrap(); + let dict_array = dictionary_encode(array.as_view(), &stats).unwrap(); assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); diff --git a/vortex-compressor/src/stats/cache.rs b/vortex-compressor/src/stats/cache.rs index 4582f9e3b09..30860b93bbe 100644 --- a/vortex-compressor/src/stats/cache.rs +++ b/vortex-compressor/src/stats/cache.rs @@ -7,11 +7,10 @@ use std::any::Any; use std::any::TypeId; use vortex_array::ArrayRef; +use vortex_array::ArrayView; use vortex_array::ToCanonical; use vortex_array::arrays::Primitive; -use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinView; -use vortex_array::arrays::VarBinViewArray; use vortex_error::VortexExpect; use super::BoolStats; @@ -105,30 +104,26 @@ impl ArrayAndStats { &self.array } - // TODO(connor): This should return an `ArrayView` once more vtable changes land. - /// Returns the array as a [`PrimitiveArray`]. + /// Returns the array as an [`ArrayView`]. /// /// # Panics /// /// Panics if the array is not a primitive array. - pub fn array_as_primitive(&self) -> PrimitiveArray { + pub fn array_as_primitive(&self) -> ArrayView<'_, Primitive> { self.array .as_opt::() .vortex_expect("the array is guaranteed to already be canonical by construction") - .into_owned() } - // TODO(connor): This should return an `ArrayView` once more vtable changes land. - /// Returns the array as a [`VarBinViewArray`]. + /// Returns the array as an [`ArrayView`]. /// /// # Panics /// /// Panics if the array is not a UTF-8 string array. - pub fn array_as_utf8(&self) -> VarBinViewArray { + pub fn array_as_utf8(&self) -> ArrayView<'_, VarBinView> { self.array .as_opt::() .vortex_expect("the array is guaranteed to already be canonical by construction") - .into_owned() } /// Consumes the bundle and returns the array. diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index e346119d35e..3a6083e8bba 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -382,7 +382,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable); // Encode: ALP → FoR → BitPacked - let alp = alp_encode(&float_prim, Some(exponents)).vortex_expect("alp_encode"); + let alp = alp_encode(float_prim.as_view(), Some(exponents)).vortex_expect("alp_encode"); assert!(alp.patches().is_none()); let for_arr = FoRData::encode(alp.encoded().to_primitive()).vortex_expect("for encode"); let bp = diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index e82cce668cb..2bc12a536d0 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -874,7 +874,7 @@ mod tests { .collect(); let float_prim = PrimitiveArray::new(Buffer::from(floats.clone()), NonNullable); - let alp = alp_encode(&float_prim, Some(exponents))?; + let alp = alp_encode(float_prim.as_view(), Some(exponents))?; assert!(alp.patches().is_none()); let for_arr = FoR::encode(alp.encoded().to_primitive())?; let bp = BitPacked::encode(for_arr.encoded(), 6)?; diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index c163b92203f..2f6e2a74901 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -205,7 +205,7 @@ mod tests { Some(5.0), ]; let prim = PrimitiveArray::from_option_iter(values); - let alp_array = alp_encode(&prim, None)?; + let alp_array = alp_encode(prim.as_view(), None)?; let cpu_result = alp_array.to_canonical()?.into_array(); @@ -232,7 +232,7 @@ mod tests { Buffer::from(vec![1.0f32, 2.0, 3.0, 4.0, 5.0]), Validity::AllValid, ); - let alp_array = alp_encode(&values, None)?; + let alp_array = alp_encode(values.as_view(), None)?; let cpu_result = alp_array.to_canonical()?.into_array(); diff --git a/vortex-python/src/arrays/compressed.rs b/vortex-python/src/arrays/compressed.rs index 96abfa4f1a1..fd21c54fe69 100644 --- a/vortex-python/src/arrays/compressed.rs +++ b/vortex-python/src/arrays/compressed.rs @@ -90,7 +90,7 @@ impl PyZigZagArray { #[staticmethod] pub fn encode(array: PyArrayRef) -> PyVortexResult { Ok(PyVortex( - zigzag_encode(array.inner().clone().to_primitive())?.into_array(), + zigzag_encode(array.inner().clone().to_primitive().as_view())?.into_array(), )) } } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs index cf20093cf21..9de554a8ee7 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs @@ -129,17 +129,17 @@ impl FlatLayoutFixture for AlpFixture { "f64_boundary_specials", ]), vec![ - alp_encode(&f64_prices, None)?.into_array(), - alp_encode(&f32_near_int, None)?.into_array(), - alp_encode(&f64_negative_near_int, None)?.into_array(), - alp_encode(&f64_currency, None)?.into_array(), - alp_encode(&f64_nullable, None)?.into_array(), - alp_encode(&f64_patched, None)?.into_array(), - alp_encode(&f64_patch_heavy, None)?.into_array(), - alp_encode(&f64_special_values, None)?.into_array(), - alp_encode(&f32_special_values, None)?.into_array(), - alp_encode(&f64_extremes, None)?.into_array(), - alp_encode(&f64_boundary_specials, None)?.into_array(), + alp_encode(f64_prices.as_view(), None)?.into_array(), + alp_encode(f32_near_int.as_view(), None)?.into_array(), + alp_encode(f64_negative_near_int.as_view(), None)?.into_array(), + alp_encode(f64_currency.as_view(), None)?.into_array(), + alp_encode(f64_nullable.as_view(), None)?.into_array(), + alp_encode(f64_patched.as_view(), None)?.into_array(), + alp_encode(f64_patch_heavy.as_view(), None)?.into_array(), + alp_encode(f64_special_values.as_view(), None)?.into_array(), + alp_encode(f32_special_values.as_view(), None)?.into_array(), + alp_encode(f64_extremes.as_view(), None)?.into_array(), + alp_encode(f64_boundary_specials.as_view(), None)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs index a6ca54363b6..977fc7cab36 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs @@ -149,16 +149,22 @@ impl FlatLayoutFixture for AlprdFixture { "nullable_specials", ]), vec![ - sensor_enc.encode(&sensor).into_array(), - drift_enc.encode(&drift).into_array(), - constant_enc.encode(&constant_series).into_array(), - decreasing_enc.encode(&decreasing).into_array(), - oscillating_enc.encode(&oscillating).into_array(), - periodic_resets_enc.encode(&periodic_resets).into_array(), - nullable_enc.encode(&sensor_nullable).into_array(), - special_enc.encode(&special_values).into_array(), - boundary_enc.encode(&boundary_specials).into_array(), - nullable_special_enc.encode(&nullable_specials).into_array(), + sensor_enc.encode(sensor.as_view()).into_array(), + drift_enc.encode(drift.as_view()).into_array(), + constant_enc.encode(constant_series.as_view()).into_array(), + decreasing_enc.encode(decreasing.as_view()).into_array(), + oscillating_enc.encode(oscillating.as_view()).into_array(), + periodic_resets_enc + .encode(periodic_resets.as_view()) + .into_array(), + nullable_enc.encode(sensor_nullable.as_view()).into_array(), + special_enc.encode(special_values.as_view()).into_array(), + boundary_enc + .encode(boundary_specials.as_view()) + .into_array(), + nullable_special_enc + .encode(nullable_specials.as_view()) + .into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs index 33dd4b78e37..a80be0c5c13 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs @@ -69,14 +69,14 @@ impl FlatLayoutFixture for PcoFixture { "narrow_i16", ]), vec![ - Pco::from_primitive(&irregular_i64, 8, 0)?.into_array(), - Pco::from_primitive(&smooth_f64, 8, 0)?.into_array(), - Pco::from_primitive(&pattern_u32, 8, 0)?.into_array(), - Pco::from_primitive(&nullable_f32, 8, 0)?.into_array(), - Pco::from_primitive(&negative_i32, 8, 0)?.into_array(), - Pco::from_primitive(&constant_u16, 8, 0)?.into_array(), - Pco::from_primitive(&spike_outliers, 8, 0)?.into_array(), - Pco::from_primitive(&narrow_i16, 8, 0)?.into_array(), + Pco::from_primitive(irregular_i64.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(smooth_f64.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(pattern_u32.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(nullable_f32.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(negative_i32.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(constant_u16.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(spike_outliers.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(narrow_i16.as_view(), 8, 0)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs index 46235ed1fa0..ac0d4ba85e7 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs @@ -68,14 +68,14 @@ impl FlatLayoutFixture for RleFixture { "short_runs_u8", ]), vec![ - RLE::encode(&runs_i32)?.into_array(), - RLE::encode(&single_run)?.into_array(), - RLE::encode(&nullable_runs)?.into_array(), - RLE::encode(&alternating_singletons)?.into_array(), - RLE::encode(&exact_boundary_runs)?.into_array(), - RLE::encode(&giant_final_run)?.into_array(), - RLE::encode(&all_null_i32)?.into_array(), - RLE::encode(&short_runs_u8)?.into_array(), + RLE::encode(runs_i32.as_view())?.into_array(), + RLE::encode(single_run.as_view())?.into_array(), + RLE::encode(nullable_runs.as_view())?.into_array(), + RLE::encode(alternating_singletons.as_view())?.into_array(), + RLE::encode(exact_boundary_runs.as_view())?.into_array(), + RLE::encode(giant_final_run.as_view())?.into_array(), + RLE::encode(all_null_i32.as_view())?.into_array(), + RLE::encode(short_runs_u8.as_view())?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs index 2e1bb972f5d..ec852bf8deb 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs @@ -80,17 +80,17 @@ impl FlatLayoutFixture for ZigZagFixture { "head_tail_nulls", ]), vec![ - zigzag_encode(alternating_i32)?.into_array(), - zigzag_encode(small_i64)?.into_array(), - zigzag_encode(deltas_i32)?.into_array(), - zigzag_encode(small_i16)?.into_array(), - zigzag_encode(small_i8)?.into_array(), - zigzag_encode(nullable_zigzag)?.into_array(), - zigzag_encode(extremes_i32)?.into_array(), - zigzag_encode(zero_heavy_outliers)?.into_array(), - zigzag_encode(repeated_negative)?.into_array(), - zigzag_encode(zero_crossing)?.into_array(), - zigzag_encode(head_tail_nulls)?.into_array(), + zigzag_encode(alternating_i32.as_view())?.into_array(), + zigzag_encode(small_i64.as_view())?.into_array(), + zigzag_encode(deltas_i32.as_view())?.into_array(), + zigzag_encode(small_i16.as_view())?.into_array(), + zigzag_encode(small_i8.as_view())?.into_array(), + zigzag_encode(nullable_zigzag.as_view())?.into_array(), + zigzag_encode(extremes_i32.as_view())?.into_array(), + zigzag_encode(zero_heavy_outliers.as_view())?.into_array(), + zigzag_encode(repeated_negative.as_view())?.into_array(), + zigzag_encode(zero_crossing.as_view())?.into_array(), + zigzag_encode(head_tail_nulls.as_view())?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 077fda0a536..381e54a1f55 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -103,7 +103,7 @@ mod setup { /// Create ALP <- FoR <- BitPacked encoding tree for f64 pub fn alp_for_bp_f64() -> ArrayRef { let (_, _, float_array) = setup_primitive_arrays(); - let alp_compressed = alp_encode(&float_array, None).unwrap(); + let alp_compressed = alp_encode(float_array.as_view(), None).unwrap(); // Manually construct ALP <- FoR <- BitPacked tree let for_array = FoR::encode(alp_compressed.encoded().to_primitive()).unwrap(); diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 091b9a6747f..656d1a6137d 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -217,13 +217,13 @@ fn bench_zigzag_compress_i32(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| int_array.clone()) - .bench_values(|a| zigzag_encode(a).unwrap()); + .bench_values(|a| zigzag_encode(a.as_view()).unwrap()); } #[divan::bench(name = "zigzag_decompress_i32")] fn bench_zigzag_decompress_i32(bencher: Bencher) { let (_, int_array, _) = setup_primitive_arrays(); - let compressed = zigzag_encode(int_array).unwrap().into_array(); + let compressed = zigzag_encode(int_array.as_view()).unwrap().into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -237,7 +237,7 @@ fn bench_sequence_compress_u32(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| seq_array.clone()) - .bench_values(|a| sequence_encode(&a).unwrap().unwrap()); + .bench_values(|a| sequence_encode(a.as_view()).unwrap().unwrap()); } #[expect(clippy::cast_possible_truncation)] @@ -258,13 +258,13 @@ fn bench_alp_compress_f64(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &float_array) - .bench_refs(|a| alp_encode(a, None).unwrap()); + .bench_refs(|a| alp_encode(a.as_view(), None).unwrap()); } #[divan::bench(name = "alp_decompress_f64")] fn bench_alp_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); - let compressed = alp_encode(&float_array, None).unwrap(); + let compressed = alp_encode(float_array.as_view(), None).unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &compressed) @@ -279,7 +279,7 @@ fn bench_alp_rd_compress_f64(bencher: Bencher) { .with_inputs(|| &float_array) .bench_refs(|a| { let encoder = RDEncoder::new(a.as_slice::()); - encoder.encode(a) + encoder.encode(a.as_view()) }); } @@ -287,7 +287,7 @@ fn bench_alp_rd_compress_f64(bencher: Bencher) { fn bench_alp_rd_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); let encoder = RDEncoder::new(float_array.as_slice::()); - let compressed = encoder.encode(&float_array); + let compressed = encoder.encode(float_array.as_view()); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &compressed) @@ -300,13 +300,13 @@ fn bench_pcodec_compress_f64(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &float_array) - .bench_refs(|a| Pco::from_primitive(a, 3, 0).unwrap()); + .bench_refs(|a| Pco::from_primitive(a.as_view(), 3, 0).unwrap()); } #[divan::bench(name = "pcodec_decompress_f64")] fn bench_pcodec_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); - let compressed = Pco::from_primitive(&float_array, 3, 0).unwrap(); + let compressed = Pco::from_primitive(float_array.as_view(), 3, 0).unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &compressed) From d296528b787cff7c187ecfb94fd2040feb9b9a55 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 10 Apr 2026 10:34:50 +0100 Subject: [PATCH 005/250] Remove MaskMut and unused functions from BitBufferMut (#7379) A lot of this logic dates back to earlier expriments. We don't need this logic and in some cases it's duplicate with a logic we have optimised elsewhere Signed-off-by: Robert Kruszewski --- vortex-array/public-api.lock | 2 - vortex-array/src/patches.rs | 134 ----- vortex-buffer/public-api.lock | 18 +- vortex-buffer/src/bit/buf_mut.rs | 129 +---- vortex-mask/public-api.lock | 82 --- vortex-mask/src/arrow.rs | 24 - vortex-mask/src/iter_bools.rs | 87 --- vortex-mask/src/lib.rs | 14 +- vortex-mask/src/mask_mut.rs | 874 ------------------------------- 9 files changed, 5 insertions(+), 1359 deletions(-) delete mode 100644 vortex-mask/src/arrow.rs delete mode 100644 vortex-mask/src/iter_bools.rs delete mode 100644 vortex-mask/src/mask_mut.rs diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index de53b12a037..d5ab9cc61f8 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -13576,8 +13576,6 @@ pub struct vortex_array::patches::Patches impl vortex_array::patches::Patches -pub unsafe fn vortex_array::patches::Patches::apply_to_buffer(&self, buffer: &mut [P], validity: &mut vortex_mask::mask_mut::MaskMut, ctx: &mut vortex_array::ExecutionCtx) - pub fn vortex_array::patches::Patches::array_len(&self) -> usize pub fn vortex_array::patches::Patches::cast_values(self, values_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 1a614ceacf8..daf121d3a56 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -6,28 +6,23 @@ use std::fmt::Debug; use std::hash::Hash; use std::ops::Range; -use itertools::Itertools; use num_traits::NumCast; use vortex_buffer::BitBuffer; use vortex_buffer::BufferMut; use vortex_error::VortexError; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_mask::AllOr; use vortex_mask::Mask; -use vortex_mask::MaskMut; use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; -use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; -use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::IntegerPType; @@ -868,59 +863,6 @@ impl Patches { })) } - /// Apply patches to a mutable buffer and validity mask. - /// - /// This method applies the patch values to the given buffer at the positions specified by the - /// patch indices. For non-null patch values, it updates the buffer and marks the position as - /// valid. For null patch values, it marks the position as invalid. - /// - /// # Safety - /// - /// - All patch indices after offset adjustment must be valid indices into the buffer. - /// - The buffer and validity mask must have the same length. - pub unsafe fn apply_to_buffer( - &self, - buffer: &mut [P], - validity: &mut MaskMut, - ctx: &mut ExecutionCtx, - ) { - let patch_indices = self - .indices - .clone() - .execute::(ctx) - .vortex_expect("patch indices must be convertible to PrimitiveArray"); - let patch_values = self - .values - .clone() - .execute::(ctx) - .vortex_expect("patch values must be convertible to PrimitiveArray"); - let patches_validity = patch_values - .validity() - .vortex_expect("patch values validity should be derivable"); - - let patch_values_slice = patch_values.as_slice::

(); - match_each_unsigned_integer_ptype!(patch_indices.ptype(), |I| { - let patch_indices_slice = patch_indices.as_slice::(); - - // SAFETY: - // - `Patches` invariant guarantees indices are sorted and within array bounds. - // - `patch_indices` and `patch_values` have equal length (from `Patches` invariant). - // - `buffer` and `validity` have equal length (precondition). - // - All patch indices are valid after offset adjustment (precondition). - unsafe { - apply_patches_to_buffer_inner( - buffer, - validity, - patch_indices_slice, - self.offset, - patch_values_slice, - &patches_validity, - ctx, - ); - } - }); - } - pub fn map_values(self, f: F) -> VortexResult where F: FnOnce(ArrayRef) -> VortexResult, @@ -945,82 +887,6 @@ impl Patches { } } -/// Helper function to apply patches to a buffer. -/// -/// # Safety -/// -/// - All indices in `patch_indices` after subtracting `patch_offset` must be valid indices -/// into both `buffer` and `validity`. -/// - `patch_indices` must be sorted in ascending order. -/// - `patch_indices` and `patch_values` must have the same length. -/// - `buffer` and `validity` must have the same length. -unsafe fn apply_patches_to_buffer_inner( - buffer: &mut [P], - validity: &mut MaskMut, - patch_indices: &[I], - patch_offset: usize, - patch_values: &[P], - patches_validity: &Validity, - ctx: &mut ExecutionCtx, -) where - P: NativePType, - I: UnsignedPType, -{ - debug_assert!(!patch_indices.is_empty()); - debug_assert_eq!(patch_indices.len(), patch_values.len()); - debug_assert_eq!(buffer.len(), validity.len()); - - match patches_validity { - Validity::NonNullable | Validity::AllValid => { - // All patch values are valid, apply them all. - for (&i, &value) in patch_indices.iter().zip_eq(patch_values) { - let index = i.as_() - patch_offset; - - // SAFETY: `index` is valid because caller guarantees all patch indices are within - // bounds after offset adjustment. - unsafe { - validity.set_unchecked(index); - } - buffer[index] = value; - } - } - Validity::AllInvalid => { - // All patch values are null, just mark positions as invalid. - for &i in patch_indices { - let index = i.as_() - patch_offset; - - // SAFETY: `index` is valid because caller guarantees all patch indices are within - // bounds after offset adjustment. - unsafe { - validity.unset_unchecked(index); - } - } - } - Validity::Array(array) => { - // Some patch values may be null, check each one. - let bool_array = array - .clone() - .execute::(ctx) - .vortex_expect("validity array must be convertible to BoolArray"); - let mask = bool_array.to_bit_buffer(); - for (patch_idx, (&i, &value)) in patch_indices.iter().zip_eq(patch_values).enumerate() { - let index = i.as_() - patch_offset; - - // SAFETY: `index` and `patch_idx` are valid because caller guarantees all patch - // indices are within bounds after offset adjustment. - unsafe { - if mask.value_unchecked(patch_idx) { - buffer[index] = value; - validity.set_unchecked(index); - } else { - validity.unset_unchecked(index); - } - } - } - } - } -} - #[allow(clippy::too_many_arguments)] // private function, can clean up one day fn take_map, T: NativePType>( indices: &[I], diff --git a/vortex-buffer/public-api.lock b/vortex-buffer/public-api.lock index 29b941c77d4..c89264da253 100644 --- a/vortex-buffer/public-api.lock +++ b/vortex-buffer/public-api.lock @@ -430,16 +430,12 @@ pub fn vortex_buffer::BitBufferMut::append_n(&mut self, value: bool, n: usize) pub fn vortex_buffer::BitBufferMut::append_true(&mut self) -pub fn vortex_buffer::BitBufferMut::as_mut_ptr(&mut self) -> *mut u8 - pub fn vortex_buffer::BitBufferMut::as_mut_slice(&mut self) -> &mut [u8] pub fn vortex_buffer::BitBufferMut::as_slice(&self) -> &[u8] pub fn vortex_buffer::BitBufferMut::capacity(&self) -> usize -pub fn vortex_buffer::BitBufferMut::chunks(&self) -> arrow_buffer::util::bit_chunk_iterator::BitChunks<'_> - pub fn vortex_buffer::BitBufferMut::clear(&mut self) pub fn vortex_buffer::BitBufferMut::collect_bool bool>(len: usize, f: F) -> Self @@ -448,8 +444,6 @@ pub fn vortex_buffer::BitBufferMut::copy_from(bit_buffer: &vortex_buffer::BitBuf pub fn vortex_buffer::BitBufferMut::empty() -> Self -pub fn vortex_buffer::BitBufferMut::false_count(&self) -> usize - pub fn vortex_buffer::BitBufferMut::fill_range(&mut self, start: usize, end: usize, value: bool) pub unsafe fn vortex_buffer::BitBufferMut::fill_range_unchecked(&mut self, start: usize, end: usize, value: bool) @@ -486,14 +480,10 @@ pub fn vortex_buffer::BitBufferMut::set_to(&mut self, index: usize, value: bool) pub unsafe fn vortex_buffer::BitBufferMut::set_to_unchecked(&mut self, index: usize, value: bool) -pub fn vortex_buffer::BitBufferMut::split_off(&mut self, at: usize) -> Self - -pub fn vortex_buffer::BitBufferMut::true_count(&self) -> usize +pub unsafe fn vortex_buffer::BitBufferMut::set_unchecked(&mut self, index: usize) pub fn vortex_buffer::BitBufferMut::truncate(&mut self, len: usize) -pub fn vortex_buffer::BitBufferMut::unaligned_chunks(&self) -> arrow_buffer::util::bit_chunk_iterator::UnalignedBitChunk<'_> - pub fn vortex_buffer::BitBufferMut::unset(&mut self, index: usize) pub unsafe fn vortex_buffer::BitBufferMut::unset_unchecked(&mut self, index: usize) @@ -510,12 +500,6 @@ impl core::clone::Clone for vortex_buffer::BitBufferMut pub fn vortex_buffer::BitBufferMut::clone(&self) -> vortex_buffer::BitBufferMut -impl core::cmp::Eq for vortex_buffer::BitBufferMut - -impl core::cmp::PartialEq for vortex_buffer::BitBufferMut - -pub fn vortex_buffer::BitBufferMut::eq(&self, other: &Self) -> bool - impl core::convert::From<&[bool]> for vortex_buffer::BitBufferMut pub fn vortex_buffer::BitBufferMut::from(value: &[bool]) -> Self diff --git a/vortex-buffer/src/bit/buf_mut.rs b/vortex-buffer/src/bit/buf_mut.rs index e0e2f85877d..f46a00a8fe0 100644 --- a/vortex-buffer/src/bit/buf_mut.rs +++ b/vortex-buffer/src/bit/buf_mut.rs @@ -3,8 +3,6 @@ use std::ops::Not; -use arrow_buffer::bit_chunk_iterator::BitChunks; -use arrow_buffer::bit_chunk_iterator::UnalignedBitChunk; use bitvec::view::BitView; use crate::BitBuffer; @@ -90,7 +88,7 @@ fn fill_bits(slice: &mut [u8], start_bit: usize, end_bit: usize, value: bool) { /// ``` /// /// See also: [`BitBuffer`]. -#[derive(Debug, Clone, Eq)] +#[derive(Debug, Clone)] pub struct BitBufferMut { buffer: ByteBufferMut, /// Represents the offset of the bit buffer into the first byte. @@ -100,19 +98,6 @@ pub struct BitBufferMut { len: usize, } -impl PartialEq for BitBufferMut { - fn eq(&self, other: &Self) -> bool { - if self.len != other.len { - return false; - } - - self.chunks() - .iter_padded() - .zip(other.chunks().iter_padded()) - .all(|(a, b)| a == b) - } -} - impl BitBufferMut { /// Create new bit buffer from given byte buffer and logical bit length pub fn from_buffer(buffer: ByteBufferMut, offset: usize, len: usize) -> Self { @@ -271,13 +256,6 @@ impl BitBufferMut { unsafe { get_bit_unchecked(self.buffer.as_ptr(), self.offset + index) } } - /// Access chunks of the underlying buffer as 8 byte chunks with a final trailer - /// - /// If you're performing operations on a single buffer, prefer [BitBuffer::unaligned_chunks] - pub fn chunks(&self) -> BitChunks<'_> { - BitChunks::new(self.buffer.as_slice(), self.offset, self.len) - } - /// Get the bit capacity of the buffer. #[inline(always)] pub fn capacity(&self) -> usize { @@ -354,7 +332,8 @@ impl BitBufferMut { /// # Safety /// /// The caller must ensure that `index` does not exceed the largest bit index in the backing buffer. - unsafe fn set_unchecked(&mut self, index: usize) { + #[inline] + pub unsafe fn set_unchecked(&mut self, index: usize) { // SAFETY: checked by caller unsafe { set_bit_unchecked(self.buffer.as_mut_ptr(), self.offset + index) } } @@ -536,52 +515,6 @@ impl BitBufferMut { self.len += bit_len; } - /// Splits the bit buffer into two at the given index. - /// - /// Afterward, self contains elements `[0, at)`, and the returned buffer contains elements - /// `[at, capacity)`. - /// - /// Unlike bytes, if the split position is not on a byte-boundary this operation will copy - /// data into the result type, and mutate self. - #[must_use = "consider BitBufferMut::truncate if you don't need the other half"] - pub fn split_off(&mut self, at: usize) -> Self { - assert!( - at <= self.capacity(), - "index {at} exceeds capacity {}", - self.capacity() - ); - - // The length of the tail is any bits after `at` - let tail_len = self.len.saturating_sub(at); - let byte_pos = (self.offset + at).div_ceil(8); - - // If we are splitting on a byte boundary, we can just slice the buffer - // Or if `at > self.len`, then the tail is empty anyway and we can just return as much - // of the existing capacity as possible. - if at > self.len() || (self.offset + at).is_multiple_of(8) { - let tail_buffer = self.buffer.split_off(byte_pos); - self.len = self.len.min(at); - - // Return the tail buffer - return Self { - buffer: tail_buffer, - offset: 0, - len: tail_len, - }; - } - - // Otherwise, we truncate ourselves, and copy any bits into a new tail buffer. - // Note that in this case we do not preserve the capacity. - let u64_cap = tail_len.div_ceil(8); - let mut tail_buffer_u64 = BufferMut::::with_capacity(u64_cap); - tail_buffer_u64.extend( - BitChunks::new(self.buffer.as_slice(), self.offset + at, tail_len).iter_padded(), - ); - - self.truncate(at); - BitBufferMut::from_buffer(tail_buffer_u64.into_byte_buffer(), 0, tail_len) - } - /// Absorbs a mutable buffer that was previously split off. /// /// If the two buffers were previously contiguous and not mutated in a way that causes @@ -615,26 +548,6 @@ impl BitBufferMut { pub fn as_mut_slice(&mut self) -> &mut [u8] { self.buffer.as_mut_slice() } - - /// Returns a raw mutable pointer to the internal buffer. - pub fn as_mut_ptr(&mut self) -> *mut u8 { - self.buffer.as_mut_ptr() - } - - /// Access chunks of the buffer aligned to 8 byte boundary as [prefix, \, suffix] - pub fn unaligned_chunks(&self) -> UnalignedBitChunk<'_> { - UnalignedBitChunk::new(self.buffer.as_slice(), self.offset, self.len) - } - - /// Get the number of set bits in the buffer. - pub fn true_count(&self) -> usize { - self.unaligned_chunks().count_ones() - } - - /// Get the number of unset bits in the buffer. - pub fn false_count(&self) -> usize { - self.len - self.true_count() - } } impl Default for BitBufferMut { @@ -1176,40 +1089,4 @@ mod tests { assert_eq!(bit_buf.value(i), i % 2 == 0); } } - - #[test] - fn test_split_off() { - // Test splitting at various positions and across a byte boundary - for i in 0..10 { - let buf = bitbuffer![0 1 0 1 0 1 0 1 0 1]; - - let mut buf_mut = BitBufferMut::copy_from(&buf); - assert_eq!(buf_mut.len(), 10); - - let tail = buf_mut.split_off(i); - assert_eq!(buf_mut.len(), i); - assert_eq!(buf_mut.freeze(), buf.slice(0..i)); - - assert_eq!(tail.len(), 10 - i); - assert_eq!(tail.freeze(), buf.slice(i..10)); - } - } - - #[test] - fn test_split_off_with_offset() { - // Test splitting at various positions and across a byte boundary - for i in 0..10 { - let buf = bitbuffer![0 1 0 1 0 1 0 1 0 1 0 1].slice(2..); - - let mut buf_mut = BitBufferMut::copy_from(&buf); - assert_eq!(buf_mut.len(), 10); - - let tail = buf_mut.split_off(i); - assert_eq!(buf_mut.len(), i); - assert_eq!(buf_mut.freeze(), buf.slice(0..i)); - - assert_eq!(tail.len(), 10 - i); - assert_eq!(tail.freeze(), buf.slice(i..10)); - } - } } diff --git a/vortex-mask/public-api.lock b/vortex-mask/public-api.lock index a986381bcff..941c422c4f8 100644 --- a/vortex-mask/public-api.lock +++ b/vortex-mask/public-api.lock @@ -106,16 +106,6 @@ impl vortex_mask::Mask pub fn vortex_mask::Mask::intersect_by_rank(&self, mask: &vortex_mask::Mask) -> vortex_mask::Mask -impl vortex_mask::Mask - -pub fn vortex_mask::Mask::into_mut(self) -> vortex_mask::MaskMut - -pub fn vortex_mask::Mask::try_into_mut(self) -> core::result::Result - -impl vortex_mask::Mask - -pub fn vortex_mask::Mask::iter_bools(&self, f: F) -> T where F: core::ops::function::FnMut(&mut dyn core::iter::traits::iterator::Iterator) -> T - impl core::clone::Clone for vortex_mask::Mask pub fn vortex_mask::Mask::clone(&self) -> vortex_mask::Mask @@ -176,78 +166,6 @@ pub vortex_mask::MaskIter::Indices(&'a [usize]) pub vortex_mask::MaskIter::Slices(&'a [(usize, usize)]) -pub struct vortex_mask::MaskMut(_) - -impl vortex_mask::MaskMut - -pub fn vortex_mask::MaskMut::all_false(&self) -> bool - -pub fn vortex_mask::MaskMut::all_true(&self) -> bool - -pub fn vortex_mask::MaskMut::append_mask(&mut self, other: &vortex_mask::Mask) - -pub fn vortex_mask::MaskMut::append_n(&mut self, new_value: bool, n: usize) - -pub fn vortex_mask::MaskMut::as_bit_buffer_mut(&mut self) -> core::option::Option<&mut vortex_buffer::bit::buf_mut::BitBufferMut> - -pub fn vortex_mask::MaskMut::capacity(&self) -> usize - -pub fn vortex_mask::MaskMut::clear(&mut self) - -pub fn vortex_mask::MaskMut::empty() -> Self - -pub fn vortex_mask::MaskMut::freeze(self) -> vortex_mask::Mask - -pub fn vortex_mask::MaskMut::from_buffer(bit_buffer: vortex_buffer::bit::buf_mut::BitBufferMut) -> Self - -pub fn vortex_mask::MaskMut::is_empty(&self) -> bool - -pub fn vortex_mask::MaskMut::len(&self) -> usize - -pub fn vortex_mask::MaskMut::new(len: usize, value: bool) -> Self - -pub fn vortex_mask::MaskMut::new_false(len: usize) -> Self - -pub fn vortex_mask::MaskMut::new_true(len: usize) -> Self - -pub fn vortex_mask::MaskMut::reserve(&mut self, additional: usize) - -pub fn vortex_mask::MaskMut::set(&mut self, index: usize) - -pub unsafe fn vortex_mask::MaskMut::set_len(&mut self, new_len: usize) - -pub fn vortex_mask::MaskMut::set_to(&mut self, index: usize, value: bool) - -pub unsafe fn vortex_mask::MaskMut::set_to_unchecked(&mut self, index: usize, value: bool) - -pub unsafe fn vortex_mask::MaskMut::set_unchecked(&mut self, index: usize) - -pub fn vortex_mask::MaskMut::split_off(&mut self, at: usize) -> Self - -pub fn vortex_mask::MaskMut::truncate(&mut self, len: usize) - -pub fn vortex_mask::MaskMut::unset(&mut self, index: usize) - -pub unsafe fn vortex_mask::MaskMut::unset_unchecked(&mut self, index: usize) - -pub fn vortex_mask::MaskMut::unsplit(&mut self, other: Self) - -pub fn vortex_mask::MaskMut::value(&self, index: usize) -> bool - -pub fn vortex_mask::MaskMut::with_capacity(capacity: usize) -> Self - -impl core::clone::Clone for vortex_mask::MaskMut - -pub fn vortex_mask::MaskMut::clone(&self) -> vortex_mask::MaskMut - -impl core::default::Default for vortex_mask::MaskMut - -pub fn vortex_mask::MaskMut::default() -> Self - -impl core::fmt::Debug for vortex_mask::MaskMut - -pub fn vortex_mask::MaskMut::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub struct vortex_mask::MaskValues impl vortex_mask::MaskValues diff --git a/vortex-mask/src/arrow.rs b/vortex-mask/src/arrow.rs deleted file mode 100644 index 1fb8d98eb89..00000000000 --- a/vortex-mask/src/arrow.rs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use arrow_buffer::NullBuffer; - -use crate::Mask; - -impl From for Option { - fn from(value: Mask) -> Self { - match value { - Mask::AllTrue(_) => None, - Mask::AllFalse(len) => Some(NullBuffer::new_null(len)), - Mask::Values(values) => { - // SAFETY: we maintain our own validated true count. - Some(unsafe { - NullBuffer::new_unchecked( - values.bit_buffer().clone().into(), - values.len() - values.true_count(), - ) - }) - } - } - } -} diff --git a/vortex-mask/src/iter_bools.rs b/vortex-mask/src/iter_bools.rs deleted file mode 100644 index c3d982db45a..00000000000 --- a/vortex-mask/src/iter_bools.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::iter; - -use crate::AllOr; -use crate::Mask; - -impl Mask { - /// Provides a closure with an iterator over the boolean values of the mask. - /// - /// This allows us to provide different implementations of the iterator based on the underlying - /// representation of the mask, while avoiding a heap allocation to return a boxed iterator. - /// - /// Note that bool iteration might not be the fastest way to achieve whatever is it you're - /// trying to do! - #[inline] - pub fn iter_bools(&self, mut f: F) -> T - where - F: FnMut(&mut dyn Iterator) -> T, - { - match self.bit_buffer() { - AllOr::All => f(&mut iter::repeat_n(true, self.len())), - AllOr::None => f(&mut iter::repeat_n(false, self.len())), - AllOr::Some(buffer) => f(&mut buffer.iter()), - } - } -} - -#[cfg(test)] -mod test { - use itertools::Itertools; - - use crate::Mask; - - #[test] - fn iter_bools_all_true() { - let mask = Mask::new_true(10); - assert_eq!(mask.iter_bools(|iter| iter.collect_vec()), vec![true; 10]); - } - - #[test] - fn iter_bools_all_false() { - let mask = Mask::new_false(10); - assert_eq!(mask.iter_bools(|iter| iter.collect_vec()), vec![false; 10]); - } - - #[test] - fn iter_bools_indices() { - assert_eq!( - Mask::from_indices(5, vec![]).iter_bools(|iter| iter.collect_vec()), - vec![false; 5], - ); - assert_eq!( - Mask::from_indices(5, vec![0, 1, 2, 3, 4]).iter_bools(|iter| iter.collect_vec()), - vec![true; 5], - ); - assert_eq!( - Mask::from_indices(5, vec![0, 4]).iter_bools(|iter| iter.collect_vec()), - vec![true, false, false, false, true], - ); - assert_eq!( - Mask::from_indices(5, vec![1, 2, 3]).iter_bools(|iter| iter.collect_vec()), - vec![false, true, true, true, false], - ); - } - - #[test] - fn iter_bools_slices() { - assert_eq!( - Mask::from_slices(5, vec![]).iter_bools(|iter| iter.collect_vec()), - vec![false; 5], - ); - assert_eq!( - Mask::from_slices(5, vec![(0, 5)]).iter_bools(|iter| iter.collect_vec()), - vec![true; 5], - ); - assert_eq!( - Mask::from_slices(5, vec![(0, 1), (4, 5)]).iter_bools(|iter| iter.collect_vec()), - vec![true, false, false, false, true], - ); - assert_eq!( - Mask::from_slices(5, vec![(1, 4)]).iter_bools(|iter| iter.collect_vec()), - vec![false, true, true, true, false], - ); - } -} diff --git a/vortex-mask/src/lib.rs b/vortex-mask/src/lib.rs index 9c551837de1..da2a9f6a6d9 100644 --- a/vortex-mask/src/lib.rs +++ b/vortex-mask/src/lib.rs @@ -7,11 +7,7 @@ mod bitops; mod eq; mod intersect_by_rank; -mod iter_bools; -mod mask_mut; -#[cfg(feature = "arrow")] -mod arrow; #[cfg(test)] mod tests; @@ -24,10 +20,8 @@ use std::sync::Arc; use std::sync::OnceLock; use itertools::Itertools; -pub use mask_mut::*; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; -use vortex_buffer::set_bit_unchecked; use vortex_error::VortexResult; use vortex_error::vortex_panic; @@ -635,11 +629,10 @@ impl Mask { let mut new_buffer_builder = BitBufferMut::new_unset(mask_values.len()); debug_assert!(limit < mask_values.len()); - let ptr = new_buffer_builder.as_mut_ptr(); for index in existing_buffer.set_indices().take(limit) { // SAFETY: We checked that `limit` was less than the mask values length, // therefore `index` must be within the bounds of the bit buffer. - unsafe { set_bit_unchecked(ptr, index) } + unsafe { new_buffer_builder.set_unchecked(index) } } Self::from(new_buffer_builder.freeze()) @@ -765,11 +758,6 @@ impl MaskValues { MaskIter::Indices(self.indices()) } } - - /// Extracts the internal [`BitBuffer`]. - pub(crate) fn into_buffer(self) -> BitBuffer { - self.buffer - } } /// Iterator over the indices or slices of a mask. diff --git a/vortex-mask/src/mask_mut.rs b/vortex-mask/src/mask_mut.rs deleted file mode 100644 index 91232ec9327..00000000000 --- a/vortex-mask/src/mask_mut.rs +++ /dev/null @@ -1,874 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::sync::Arc; - -use vortex_buffer::BitBufferMut; -use vortex_error::vortex_panic; - -use crate::Mask; - -/// A mutable mask, used for lazily allocating the bit buffer as required. -#[derive(Debug, Clone)] -pub struct MaskMut(Inner); - -impl Default for MaskMut { - fn default() -> Self { - Self::empty() - } -} - -#[derive(Debug, Clone)] -enum Inner { - /// Initially, the mask is empty but may have some capacity. - Empty { capacity: usize }, - /// When the first value is pushed, the mask becomes constant. - Constant { - value: bool, - len: usize, - capacity: usize, - }, - /// When the first non-constant value is written, we allocate the bit buffer and switch - /// into the builder state. - Builder(BitBufferMut), -} - -impl MaskMut { - /// Creates a new empty mask. - pub fn empty() -> Self { - Self::with_capacity(0) - } - - /// Creates a new empty mask with the default capacity. - pub fn with_capacity(capacity: usize) -> Self { - Self(Inner::Empty { capacity }) - } - - /// Creates a new mask with the specified capacity. - pub fn new(len: usize, value: bool) -> Self { - Self(Inner::Constant { - value, - len, - capacity: len, - }) - } - - /// Creates a new mask with all values set to `true`. - pub fn new_true(len: usize) -> Self { - Self(Inner::Constant { - value: true, - len, - capacity: len, - }) - } - - /// Creates a new mask with all values set to `false`. - pub fn new_false(len: usize) -> Self { - Self(Inner::Constant { - value: false, - len, - capacity: len, - }) - } - - /// Creates a new mask from an existing bit buffer. - pub fn from_buffer(bit_buffer: BitBufferMut) -> Self { - Self(Inner::Builder(bit_buffer)) - } - - /// Returns the boolean value at a given index. - /// - /// # Panics - /// - /// Panics if the index is out of bounds. - pub fn value(&self, index: usize) -> bool { - match &self.0 { - Inner::Empty { .. } => { - vortex_panic!("index out of bounds: the length is 0 but the index is {index}") - } - Inner::Constant { value, len, .. } => { - assert!( - index < *len, - "index out of bounds: the length is {} but the index is {index}", - *len - ); - - *value - } - Inner::Builder(bit_buffer) => bit_buffer.value(index), - } - } - - /// Reserve capacity for at least `additional` more values to be appended. - pub fn reserve(&mut self, additional: usize) { - match &mut self.0 { - Inner::Empty { capacity } => { - *capacity += additional; - } - Inner::Constant { capacity, .. } => { - *capacity += additional; - } - Inner::Builder(bits) => { - bits.reserve(additional); - } - } - } - - /// Set the length of the mask. - /// - /// # Safety - /// - /// - `new_len` must be less than or equal to [`capacity()`]. - /// - The elements at `old_len..new_len` must be initialized. - /// - /// [`capacity()`]: Self::capacity - pub unsafe fn set_len(&mut self, new_len: usize) { - debug_assert!(new_len < self.capacity()); - match &mut self.0 { - Inner::Empty { capacity, .. } => { - self.0 = Inner::Constant { - value: false, // Pick any value - len: new_len, - capacity: *capacity, - } - } - Inner::Constant { len, .. } => { - *len = new_len; - } - Inner::Builder(bits) => { - unsafe { bits.set_len(new_len) }; - } - } - } - - /// Returns the capacity of the mask. - pub fn capacity(&self) -> usize { - match &self.0 { - Inner::Empty { capacity } => *capacity, - Inner::Constant { capacity, .. } => *capacity, - Inner::Builder(bits) => bits.capacity(), - } - } - - /// Clears the mask. - /// - /// Note that this method has no effect on the allocated capacity of the mask. - pub fn clear(&mut self) { - match &mut self.0 { - Inner::Empty { .. } => {} - Inner::Constant { capacity, .. } => { - self.0 = Inner::Empty { - capacity: *capacity, - } - } - Inner::Builder(bit_buffer) => bit_buffer.clear(), - }; - } - - /// Shortens the mask, keeping the first `len` bits. - /// - /// If `len` is greater or equal to the vector’s current length, this has no effect. - /// - /// Note that this method has no effect on the allocated capacity of the mask. - pub fn truncate(&mut self, len: usize) { - let truncated_len = len; - if truncated_len > self.len() { - return; - } - - match &mut self.0 { - Inner::Empty { .. } => {} - Inner::Constant { len, .. } => *len = truncated_len.min(*len), - Inner::Builder(bit_buffer) => bit_buffer.truncate(truncated_len), - }; - } - - /// Append n values to the mask. - pub fn append_n(&mut self, new_value: bool, n: usize) { - match &mut self.0 { - Inner::Empty { capacity } => { - self.0 = Inner::Constant { - value: new_value, - len: n, - capacity: (*capacity).max(n), - } - } - Inner::Constant { - value, - len, - capacity, - } => { - if *value == new_value { - // Same value, just increase length. - self.0 = Inner::Constant { - value: *value, - len: *len + n, - capacity: (*capacity).max(*len + n), - } - } else { - // Different value, need to allocate the bit buffer. - // Note: materialize() already appends the existing constant values - let bits = self.materialize(); - bits.append_n(new_value, n); - } - } - Inner::Builder(bits) => { - bits.append_n(new_value, n); - } - } - } - - /// Append a [`Mask`] to this mutable mask. - pub fn append_mask(&mut self, other: &Mask) { - match other { - Mask::AllTrue(len) => self.append_n(true, *len), - Mask::AllFalse(len) => self.append_n(false, *len), - Mask::Values(values) => { - let bitbuffer = values.buffer.clone(); - self.materialize().append_buffer(&bitbuffer); - } - } - } - - /// Ensures that the internal bit buffer is allocated and returns a mutable reference to it. - fn materialize(&mut self) -> &mut BitBufferMut { - let needs_materialization = !matches!(self.0, Inner::Builder(_)); - - if needs_materialization { - let new_builder = match &self.0 { - Inner::Empty { capacity } => BitBufferMut::with_capacity(*capacity), - Inner::Constant { - value, - len, - capacity, - } => { - let required_capacity = (*capacity).max(*len); - let mut bits = BitBufferMut::with_capacity(required_capacity); - bits.append_n(*value, *len); - bits - } - Inner::Builder(_) => unreachable!(), - }; - self.0 = Inner::Builder(new_builder); - } - - match &mut self.0 { - Inner::Builder(bits) => bits, - _ => unreachable!(), - } - } - - /// Split-off the mask at the given index, returning a new mask with the - /// values from `at` to the end, and leaving `self` with the values from - /// the start to `at`. - pub fn split_off(&mut self, at: usize) -> Self { - assert!(at <= self.capacity(), "split_off index out of bounds"); - match &mut self.0 { - Inner::Empty { capacity } => { - let new_capacity = *capacity - at; - *capacity = at; - Self(Inner::Empty { - capacity: new_capacity, - }) - } - Inner::Constant { - value, - len, - capacity, - } => { - // Adjust the lengths, given that length may be < at - let new_len = len.saturating_sub(at); - let new_capacity = *capacity - at; - *len = (*len).min(at); - *capacity = at; - - Self(Inner::Constant { - value: *value, - len: new_len, - capacity: new_capacity, - }) - } - Inner::Builder(bits) => { - let new_bits = bits.split_off(at); - Self(Inner::Builder(new_bits)) - } - } - } - - /// Absorb another mask into this one, appending its values. - pub fn unsplit(&mut self, other: Self) { - match other.0 { - Inner::Empty { .. } => { - // No work to do - } - Inner::Constant { value, len, .. } => { - self.append_n(value, len); - } - Inner::Builder(bits) => { - self.materialize().unsplit(bits); - } - } - } - - /// Freezes the mutable mask into an immutable one. - pub fn freeze(self) -> Mask { - match self.0 { - Inner::Empty { .. } => Mask::new_true(0), - Inner::Constant { value, len, .. } => { - if value { - Mask::new_true(len) - } else { - Mask::new_false(len) - } - } - Inner::Builder(bits) => Mask::from_buffer(bits.freeze()), - } - } - - /// Returns the logical length of the mask. - pub fn len(&self) -> usize { - match &self.0 { - Inner::Empty { .. } => 0, - Inner::Constant { len, .. } => *len, - Inner::Builder(bits) => bits.len(), - } - } - - /// Returns true if the mask is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns true if all values in the mask are true. - pub fn all_true(&self) -> bool { - match &self.0 { - Inner::Empty { .. } => true, - Inner::Constant { value, .. } => *value, - Inner::Builder(bits) => bits.true_count() == bits.len(), - } - } - - /// Returns true if all values in the mask are false. - pub fn all_false(&self) -> bool { - match &self.0 { - Inner::Empty { .. } => true, - Inner::Constant { value, .. } => !*value, - Inner::Builder(bits) => !bits.is_empty() && bits.true_count() == 0, - } - } - - /// Returns the internal bit buffer if it exists. - pub fn as_bit_buffer_mut(&mut self) -> Option<&mut BitBufferMut> { - match &mut self.0 { - Inner::Builder(bits) => Some(bits), - _ => None, - } - } - - /// Set the value at the given index to true. - /// - /// # Panics - /// - /// Panics if the index is out of bounds. - pub fn set(&mut self, index: usize) { - self.set_to(index, true); - } - - /// Set the value at the given index to false. - /// - /// # Panics - /// - /// Panics if the index is out of bounds. - pub fn unset(&mut self, index: usize) { - self.set_to(index, false); - } - - /// Set the value at the given index to the specified boolean value. - /// - /// # Panics - /// - /// Panics if the index is out of bounds. - pub fn set_to(&mut self, index: usize, value: bool) { - match &mut self.0 { - Inner::Empty { .. } => { - vortex_panic!("index out of bounds: the length is 0 but the index is {index}") - } - Inner::Constant { - value: current_value, - len, - .. - } => { - assert!( - index < *len, - "index out of bounds: the length is {} but the index is {index}", - *len - ); - - if *current_value != value { - // Need to materialize the buffer since we're changing from constant. - self.materialize().set_to(index, value); - } - // If the value is the same as the constant, no action needed. - } - Inner::Builder(bit_buffer) => { - bit_buffer.set_to(index, value); - } - } - } - - /// Set the value at the given index to true without bounds checking. - /// - /// # Safety - /// - /// The caller must ensure that `index < self.len()`. - pub unsafe fn set_unchecked(&mut self, index: usize) { - unsafe { self.set_to_unchecked(index, true) } - } - - /// Set the value at the given index to false without bounds checking. - /// - /// # Safety - /// - /// The caller must ensure that `index < self.len()`. - pub unsafe fn unset_unchecked(&mut self, index: usize) { - unsafe { self.set_to_unchecked(index, false) } - } - - /// Set the value at the given index to the specified boolean value without bounds checking. - /// - /// # Safety - /// - /// The caller must ensure that `index < self.len()`. - pub unsafe fn set_to_unchecked(&mut self, index: usize, value: bool) { - unsafe { - match &mut self.0 { - Inner::Empty { .. } => { - // In debug mode, we still want to catch this error. - debug_assert!(false, "cannot set value in empty mask"); - } - Inner::Constant { - value: current_value, - len, - .. - } => { - debug_assert!( - index < *len, - "index out of bounds: the length is {} but the index is {index}", - *len - ); - - if *current_value != value { - // Need to materialize the buffer since we're changing from constant. - self.materialize().set_to_unchecked(index, value); - } - // If the value is the same as the constant, no action needed. - } - Inner::Builder(bit_buffer) => { - bit_buffer.set_to_unchecked(index, value); - } - } - } - } -} - -impl Mask { - /// Attempts to convert an immutable mask into a mutable one, returning an error of `Self` if - /// the underlying [`BitBuffer`](crate::BitBuffer) data if there are any other references. - pub fn try_into_mut(self) -> Result { - match self { - Mask::AllTrue(len) => Ok(MaskMut::new_true(len)), - Mask::AllFalse(len) => Ok(MaskMut::new_false(len)), - Mask::Values(values) => { - // We need to check for uniqueness twice, first for the `Arc` with `try_unwrap`, - // then for the internal `BitBuffer` with `try_into_mut`. - let owned_values = Arc::try_unwrap(values).map_err(Mask::Values)?; - let bit_buffer = owned_values.into_buffer(); - let mut_buffer = bit_buffer.try_into_mut().map_err(Mask::from_buffer)?; - - Ok(MaskMut(Inner::Builder(mut_buffer))) - } - } - } - - /// Convert an immutable mask into a mutable one, cloning the underlying - /// [`BitBuffer`](crate::BitBuffer) data if there are any other references. - pub fn into_mut(self) -> MaskMut { - match self { - Mask::AllTrue(len) => MaskMut::new_true(len), - Mask::AllFalse(len) => MaskMut::new_false(len), - Mask::Values(values) => { - let bit_buffer_mut = match Arc::try_unwrap(values) { - Ok(mask_values) => mask_values - .into_buffer() - .try_into_mut() - .unwrap_or_else(|bb| BitBufferMut::copy_from(&bb)), - Err(arc_mask_values) => { - let bit_buffer = arc_mask_values.bit_buffer(); - BitBufferMut::copy_from(bit_buffer) - } - }; - - MaskMut(Inner::Builder(bit_buffer_mut)) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_split_off_empty() { - let mut mask = MaskMut::with_capacity(10); - assert_eq!(mask.len(), 0); - - let other = mask.split_off(0); - assert_eq!(mask.len(), 0); - assert_eq!(other.len(), 0); - } - - #[test] - fn test_split_off_constant_true_at_zero() { - let mut mask = MaskMut::new_true(10); - let other = mask.split_off(0); - - assert_eq!(mask.len(), 0); - assert_eq!(other.len(), 10); - - let frozen = other.freeze(); - assert_eq!(frozen.true_count(), 10); - } - - #[test] - fn test_split_off_constant_true_at_end() { - let mut mask = MaskMut::new_true(10); - let other = mask.split_off(10); - - assert_eq!(mask.len(), 10); - assert_eq!(other.len(), 0); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 10); - } - - #[test] - fn test_split_off_constant_true_in_middle() { - let mut mask = MaskMut::new_true(10); - let other = mask.split_off(6); - - assert_eq!(mask.len(), 6); - assert_eq!(other.len(), 4); - - let frozen_first = mask.freeze(); - assert_eq!(frozen_first.true_count(), 6); - - let frozen_second = other.freeze(); - assert_eq!(frozen_second.true_count(), 4); - } - - #[test] - fn test_split_off_constant_false() { - let mut mask = MaskMut::new_false(20); - let other = mask.split_off(12); - - assert_eq!(mask.len(), 12); - assert_eq!(other.len(), 8); - - let frozen_first = mask.freeze(); - assert_eq!(frozen_first.true_count(), 0); - - let frozen_second = other.freeze(); - assert_eq!(frozen_second.true_count(), 0); - } - - // Note: Tests using BitBuffer operations are marked as ignored under miri - // because bitvec uses raw pointer operations that miri cannot verify. - #[test] - fn test_split_off_builder_at_byte_boundary() { - let mut mask = MaskMut::with_capacity(16); - // Create a pattern: 8 true, 8 false - mask.append_n(true, 8); - mask.append_n(false, 8); - - let mask_ptr = match &mask.0 { - Inner::Builder(bits) => bits.as_slice().as_ptr(), - _ => unreachable!(), - }; - - let other = mask.split_off(8); - - assert_eq!(mask.len(), 8); - assert_eq!(other.len(), 8); - - // Ensure the unsplit was zero-copy. - mask.unsplit(other); - let new_mask_ptr = match &mask.0 { - Inner::Builder(bits) => bits.as_slice().as_ptr(), - _ => unreachable!(), - }; - assert_eq!(mask_ptr, new_mask_ptr); - } - - #[test] - fn test_split_off_builder_not_byte_aligned() { - let mut mask = MaskMut::with_capacity(20); - // Create a pattern: 10 true, 10 false - mask.append_n(true, 10); - mask.append_n(false, 10); - - let other = mask.split_off(10); - - assert_eq!(mask.len(), 10); - assert_eq!(other.len(), 10); - - let frozen_first = mask.freeze(); - assert_eq!(frozen_first.true_count(), 10); - - let frozen_second = other.freeze(); - assert_eq!(frozen_second.true_count(), 0); - } - - #[test] - fn test_split_off_builder_mixed_pattern() { - let mut mask = MaskMut::with_capacity(15); - // Create pattern: TFTFTFTFTFTFTFT (alternating) - for i in 0..15 { - mask.append_n(i % 2 == 0, 1); - } - - let other = mask.split_off(7); - - assert_eq!(mask.len(), 7); - assert_eq!(other.len(), 8); - - let frozen_first = mask.freeze(); - assert_eq!(frozen_first.true_count(), 4); // positions 0,2,4,6 - - let frozen_second = other.freeze(); - assert_eq!(frozen_second.true_count(), 4); // positions 7,9,11,13 => 0,2,4,6 in split - } - - #[test] - fn test_unsplit_empty_with_empty() { - let mut mask = MaskMut::with_capacity(10); - let other = MaskMut::with_capacity(10); - - mask.unsplit(other); - assert_eq!(mask.len(), 0); - } - - #[test] - fn test_unsplit_empty_with_constant() { - let mut mask = MaskMut::with_capacity(10); - let other = MaskMut::new_true(5); - - mask.unsplit(other); - assert_eq!(mask.len(), 5); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 5); - } - - #[test] - fn test_unsplit_constant_with_constant_same() { - let mut mask = MaskMut::new_true(5); - let other = MaskMut::new_true(5); - - mask.unsplit(other); - assert_eq!(mask.len(), 10); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 10); - } - - #[test] - fn test_unsplit_constant_with_constant_different() { - let mut mask = MaskMut::new_true(5); - let other = MaskMut::new_false(5); - - mask.unsplit(other); - assert_eq!(mask.len(), 10); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 5); - } - - #[test] - fn test_unsplit_constant_with_builder() { - let mut mask = MaskMut::new_true(5); - - let mut other = MaskMut::with_capacity(10); - other.append_n(true, 3); - other.append_n(false, 2); - - mask.unsplit(other); - assert_eq!(mask.len(), 10); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 8); // 5 from first + 3 from second - } - - #[test] - fn test_unsplit_builder_with_constant() { - let mut mask = MaskMut::with_capacity(10); - mask.append_n(true, 3); - mask.append_n(false, 2); - - let other = MaskMut::new_true(5); - - mask.unsplit(other); - assert_eq!(mask.len(), 10); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 8); // 3 from first + 5 from second - } - - #[test] - fn test_unsplit_builder_with_builder() { - let mut mask = MaskMut::with_capacity(10); - mask.append_n(true, 3); - mask.append_n(false, 2); - - let mut other = MaskMut::with_capacity(10); - other.append_n(false, 3); - other.append_n(true, 2); - - mask.unsplit(other); - assert_eq!(mask.len(), 10); - - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 5); // 3 from first + 2 from second - } - - #[test] - fn test_round_trip_split_unsplit() { - let mut original = MaskMut::with_capacity(20); - // Pattern: 10 true, 10 false - original.append_n(true, 10); - original.append_n(false, 10); - - let original_frozen = original.freeze(); - let original_true_count = original_frozen.true_count(); - - // Convert back to mutable for split - let mut mask = original_frozen.try_into_mut().unwrap(); - - // Split at 10 - let other = mask.split_off(10); - - // Unsplit back together - mask.unsplit(other); - - assert_eq!(mask.len(), 20); - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), original_true_count); - } - - #[test] - #[should_panic(expected = "split_off index out of bounds")] - fn test_split_off_out_of_bounds() { - let mut mask = MaskMut::new_true(10); - mask.split_off(11); - } - - #[test] - fn test_split_off_builder_at_bit_1() { - let mut mask = MaskMut::with_capacity(16); - mask.append_n(true, 16); - - let other = mask.split_off(1); - - assert_eq!(mask.len(), 1); - assert_eq!(other.len(), 15); - - let frozen_first = mask.freeze(); - assert_eq!(frozen_first.true_count(), 1); - - let frozen_second = other.freeze(); - assert_eq!(frozen_second.true_count(), 15); - } - - #[test] - fn test_multiple_split_unsplit() { - let mut mask = MaskMut::new_true(30); - - // Split into 3 parts - let third = mask.split_off(20); // 20-30 - let second = mask.split_off(10); // 10-20 - // first is 0-10 - - assert_eq!(mask.len(), 10); - assert_eq!(second.len(), 10); - assert_eq!(third.len(), 10); - - // Recombine in order - mask.unsplit(second); - mask.unsplit(third); - - assert_eq!(mask.len(), 30); - let frozen = mask.freeze(); - assert_eq!(frozen.true_count(), 30); - } - - #[test] - fn test_try_into_mut_all_variants() { - // Test AllTrue and AllFalse variants. - let mask_true = Mask::new_true(100); - let mut_mask_true = mask_true.try_into_mut().unwrap(); - assert_eq!(mut_mask_true.len(), 100); - assert_eq!(mut_mask_true.freeze().true_count(), 100); - - let mask_false = Mask::new_false(50); - let mut_mask_false = mask_false.try_into_mut().unwrap(); - assert_eq!(mut_mask_false.len(), 50); - assert_eq!(mut_mask_false.freeze().true_count(), 0); - } - - #[test] - fn test_try_into_mut_with_references() { - // Create a MaskValues variant. - let mut mask_mut = MaskMut::with_capacity(10); - mask_mut.append_n(true, 5); - mask_mut.append_n(false, 5); - let mask = mask_mut.freeze(); - - // Should succeed with unique reference (no clones). - let mask2 = { - let mut mask_mut2 = MaskMut::with_capacity(10); - mask_mut2.append_n(true, 5); - mask_mut2.append_n(false, 5); - mask_mut2.freeze() - }; - let result = mask2.try_into_mut(); - assert!(result.is_ok()); - assert_eq!(result.unwrap().len(), 10); - - // Should fail with shared references. - let _cloned = mask.clone(); - let result = mask.try_into_mut(); - assert!(result.is_err()); - if let Err(returned_mask) = result { - assert_eq!(returned_mask.len(), 10); - assert_eq!(returned_mask.true_count(), 5); - } - } - - #[test] - fn test_try_into_mut_round_trip() { - // Test freeze -> try_into_mut -> modify -> freeze cycle. - let mut original = MaskMut::with_capacity(20); - original.append_n(true, 10); - original.append_n(false, 10); - - let frozen = original.freeze(); - assert_eq!(frozen.true_count(), 10); - - let mut mut_mask = frozen.try_into_mut().unwrap(); - mut_mask.append_n(true, 5); - assert_eq!(mut_mask.len(), 25); - - let frozen_again = mut_mask.freeze(); - assert_eq!(frozen_again.true_count(), 15); - } -} From ac87fe0bc4b9dcee565419238122fdc9607e9ff4 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 10 Apr 2026 11:36:44 +0100 Subject: [PATCH 006/250] AllInvalid arrays have non empty distinct info (#7369) This is required to handle cases where we are attempting to dict compress and a sample ends up being all null Signed-off-by: Robert Kruszewski Signed-off-by: Robert Kruszewski --- vortex-compressor/src/stats/float.rs | 8 +++++++- vortex-compressor/src/stats/integer.rs | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index c89de9c9893..37b06c52eca 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -179,7 +179,13 @@ where null_count: u32::try_from(array.len())?, value_count: 0, average_run_length: 0, - erased: TypedStats { distinct: None }.into(), + erased: TypedStats { + distinct: Some(DistinctInfo { + distinct_values: HashSet::with_capacity_and_hasher(0, FxBuildHasher), + distinct_count: 0, + }), + } + .into(), }); } diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index b1ff843b667..2203cad30ef 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -338,7 +338,12 @@ where erased: TypedStats { min: T::max_value(), max: T::min_value(), - distinct: None, + distinct: Some(DistinctInfo { + distinct_values: HashMap::with_capacity_and_hasher(0, FxBuildHasher), + distinct_count: 0, + most_frequent_value: T::zero(), + top_frequency: 0, + }), } .into(), }); From a78897fb6924c6395656c780eaf4bd1d0cc5d248 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 10 Apr 2026 14:23:40 +0100 Subject: [PATCH 007/250] duckdb: Reset validity instead of filling (#7365) 1. Reset validity if it's "all true" instead of filling manually. 2. Set validity to true if parent array is constant (slicing won't affect that). 3. Count ones while copying to destination buffer Signed-off-by: Mikhail Kot --- vortex-duckdb/cpp/include/duckdb_vx/vector.h | 4 + vortex-duckdb/cpp/vector.cpp | 18 +++++ vortex-duckdb/src/duckdb/vector.rs | 6 +- vortex-duckdb/src/exporter/mod.rs | 83 +++++++++++++------- vortex-duckdb/src/exporter/vector.rs | 56 +++++++------ 5 files changed, 111 insertions(+), 56 deletions(-) diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector.h b/vortex-duckdb/cpp/include/duckdb_vx/vector.h index ea3e49184f7..906937f1629 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/vector.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/vector.h @@ -47,6 +47,10 @@ void duckdb_vx_string_vector_add_vector_data_buffer(duckdb_vector ffi_vector, du // valid. void duckdb_vx_vector_set_vector_data_buffer(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer); +// Reset vector's validity mask to nullptr, making all vector's elements valid. +// vector must not be a DictionaryVector or a SequenceVector +void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector); + // Set the data pointer for the vector. This is the start of the values array in the vector. void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr); diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp index 3b6439f4c95..0328b7aff3e 100644 --- a/vortex-duckdb/cpp/vector.cpp +++ b/vortex-duckdb/cpp/vector.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "include/duckdb_vx/vector.h" #include "duckdb_vx/duckdb_diagnostics.h" DUCKDB_INCLUDES_BEGIN @@ -106,3 +107,20 @@ const char *duckdb_vector_to_string(duckdb_vector vector, unsigned long len, duc return nullptr; } } + +void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector) { + using enum VectorType; + Vector &vector = *reinterpret_cast(ffi_vector); + const VectorType type = vector.GetVectorType(); + D_ASSERT(type != DICTIONARY_VECTOR && type != SEQUENCE_VECTOR); + switch (type) { + case CONSTANT_VECTOR: + return ConstantVector::Validity(vector).Reset(); + case FLAT_VECTOR: + return FlatVector::Validity(vector).Reset(); + case FSST_VECTOR: + return FSSTVector::Validity(vector).Reset(); + default: + __builtin_unreachable(); + } +} diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs index 4a822629980..ef2d35b2803 100644 --- a/vortex-duckdb/src/duckdb/vector.rs +++ b/vortex-duckdb/src/duckdb/vector.rs @@ -207,7 +207,11 @@ impl VectorRef { /// /// The provided capacity *must* be the actual capacity of this vector. pub unsafe fn validity_bitslice_mut(&mut self, capacity: usize) -> Option<&mut BitSlice> { - unsafe { self.validity_slice_mut(capacity) }.map(|slice| slice.view_bits_mut()) + // capacity is always less than BitSlice::MAX_ELTS + unsafe { + self.validity_slice_mut(capacity) + .map(|slice| BitSlice::from_slice_unchecked_mut(slice)) + } } pub fn validity_ref(&self, len: usize) -> ValidityRef<'_> { diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index 502038b858c..b5b26340efe 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -19,8 +19,6 @@ mod validity; mod varbinview; mod vector; -use bitvec::prelude::Lsb0; -use bitvec::view::BitView; pub use cache::ConversionCache; pub use decimal::precision_to_duckdb_storage_size; use vortex::array::ArrayRef; @@ -32,6 +30,7 @@ use vortex::array::arrays::List; use vortex::array::arrays::StructArray; use vortex::array::arrays::TemporalArray; use vortex::array::arrays::struct_::StructArrayExt; +use vortex::buffer::BitChunks; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::Sequence; use vortex::error::VortexResult; @@ -187,17 +186,33 @@ fn new_array_exporter_with_flatten( } } -/// Copy the sliced bits from source into target. +/// Copy the sliced bits from source into target, returning whether all copied bits are zero, +/// all are one, or mixed. /// -/// Offset and length are a _bit_ offset and a _bit_ length into source. +/// offset and len are a _bit_ offset and a _bit_ length into `source`. /// -/// `target.len()` must equal `len`. -fn copy_from_slice(target: &mut [u64], source: &[u8], offset: usize, len: usize) { - let (start, middle, end) = unsafe { target.align_to_mut::() }; - assert!(start.is_empty()); - assert!(end.is_empty()); - let target = &mut middle.view_bits_mut::()[..len]; - target.copy_from_bitslice(&source.view_bits()[offset..][..len]); +/// target must have at least len.div_ceil(64) elements. +/// Returns the number of ones in copied slice. +fn copy_from_slice(target: &mut [u64], source: &[u8], offset: usize, len: usize) -> usize { + if len == 0 { + return 0; + } + + let mut ones = 0usize; + let chunks = BitChunks::new(source, offset, len); + let chunk_len = chunks.chunk_len(); + let remainder_len = chunks.remainder_len(); + let remainder = chunks.remainder_bits(); + for (slot, chunk) in target.iter_mut().zip(chunks) { + *slot = chunk; + ones += chunk.count_ones() as usize; + } + + if remainder_len > 0 { + target[chunk_len] = remainder; + ones += remainder.count_ones() as usize; + } + ones } #[cfg(test)] @@ -359,23 +374,23 @@ mod tests { fn test_copy_from_slice_empty_to_empty() { let target = &mut []; let source = Vec::::new(); - copy_from_slice(target, &source, 0, 0); + assert_eq!(copy_from_slice(target, &source, 0, 0), 0); } #[test] fn test_copy_from_slice_64_to_empty() { let target = &mut []; let source = [1u8, 2, 3, 50, 51, 52, 100, 101]; - copy_from_slice(target, &source, 0, 0); - copy_from_slice(target, &source, 5, 0); - copy_from_slice(target, &source, 8, 0); + assert_eq!(copy_from_slice(target, &source, 0, 0), 0); + assert_eq!(copy_from_slice(target, &source, 5, 0), 0); + assert_eq!(copy_from_slice(target, &source, 8, 0), 0); } #[test] fn test_copy_from_slice_64_to_64() { let mut target = vec![0u64]; let source = [1u8, 2, 3, 50, 51, 52, 100, 101]; - copy_from_slice(&mut target, &source, 0, 64); + assert_eq!(copy_from_slice(&mut target, &source, 0, 64), 21); assert_eq!( target[0], 0x65_64_34_33_32_03_02_01_u64, "{:#08x} == {:#08x}", @@ -383,20 +398,27 @@ mod tests { ); } + #[test] + fn test_copy_from_slice_64_ones() { + let mut target = [0u64]; + let source = [u8::MAX; 8]; + assert_eq!(copy_from_slice(&mut target, &source, 0, 64), 64); + } + #[test] fn test_copy_from_slice_80_to_0() { let target = &mut []; let source = [1u8, 2, 3, 50, 51, 52, 100, 101, 254, 255]; - copy_from_slice(target, &source, 0, 0); - copy_from_slice(target, &source, 8, 0); - copy_from_slice(target, &source, 10, 0); + assert_eq!(copy_from_slice(target, &source, 0, 0), 0); + assert_eq!(copy_from_slice(target, &source, 8, 0), 0); + assert_eq!(copy_from_slice(target, &source, 10, 0), 0,); } #[test] fn test_copy_from_slice_80_to_64_case_1() { let mut target = [0u64]; let source = [1u8, 2, 3, 50, 51, 52, 100, 101, 254, 255]; - copy_from_slice(&mut target, &source, 16, 64); + assert_eq!(copy_from_slice(&mut target, &source, 16, 64), 34); assert_eq!( target[0], 0xff_fe_65_64_34_33_32_03_u64, "{:#08x} == {:#08x}", @@ -408,7 +430,7 @@ mod tests { fn test_copy_from_slice_80_to_64_case_2() { let mut target = [0u64]; let source = [1u8, 2, 3, 50, 51, 52, 100, 101, 254, 255]; - copy_from_slice(&mut target, &source, 8, 64); + assert_eq!(copy_from_slice(&mut target, &source, 8, 64), 27); assert_eq!( target[0], 0xfe_65_64_34_33_32_03_02_u64, "{:#08x} == {:#08x}", @@ -420,7 +442,7 @@ mod tests { fn test_copy_from_slice_80_to_64_case_3() { let mut target = [0u64]; let source = [1u8, 2, 3, 50, 51, 52, 100, 101, 254, 255]; - copy_from_slice(&mut target, &source, 0, 64); + assert_eq!(copy_from_slice(&mut target, &source, 0, 64), 21,); assert_eq!( target[0], 0x65_64_34_33_32_03_02_01_u64, "{:#08x} == {:#08x}", @@ -432,7 +454,7 @@ mod tests { fn test_copy_from_slice_80_to_64_case_4() { let mut target = [0u64]; let source = [1u8, 2, 3, 50, 51, 52, 100, 101, 254, 255]; - copy_from_slice(&mut target, &source, 10, 64); + assert_eq!(copy_from_slice(&mut target, &source, 10, 64), 28,); assert_eq!( target[0], 0xff_99_59_0d_0c_cc_80_c0_u64, // Python: hex(0xff_fe_65_64_34_33_32_03_02 >> 2), then remove the high two hexits @@ -454,7 +476,7 @@ mod tests { let (_, middle, _) = unsafe { source.align_to::() }; assert!(!middle.is_empty()); - copy_from_slice(&mut target, &source, 0, 128); + assert_eq!(copy_from_slice(&mut target, &source, 0, 128), 66,); assert_eq!( target[0], 0xfc_fd_fe_ff_04_03_02_01_u64, "{:#08x} == {:#08x}", @@ -466,7 +488,7 @@ mod tests { target[1], 0xf9_fa_fb_fc_08_07_06_05_u64, ); - copy_from_slice(&mut target, &source, 8, 128); + assert_eq!(copy_from_slice(&mut target, &source, 8, 128), 66); assert_eq!( target[0], 0x05_fc_fd_fe_ff_04_03_02_u64, "{:#08x} == {:#08x}", @@ -478,7 +500,7 @@ mod tests { target[1], 0x01_f9_fa_fb_fc_08_07_06_u64, ); - copy_from_slice(&mut target, &source, 8 * 8, 128); + assert_eq!(copy_from_slice(&mut target, &source, 8 * 8, 128), 66,); assert_eq!( target[0], 0xf9_fa_fb_fc_08_07_06_05_u64, "{:#08x} == {:#08x}", @@ -490,7 +512,7 @@ mod tests { target[1], 0xfc_fd_fe_ff_04_03_02_01_u64, ); - copy_from_slice(&mut target, &source, 8 * 12, 128); + assert_eq!(copy_from_slice(&mut target, &source, 8 * 12, 128), 66,); assert_eq!( target[0], 0x04_03_02_01_f9_fa_fb_fc_u64, "{:#08x} == {:#08x}", @@ -502,7 +524,7 @@ mod tests { target[1], 0x08_07_06_05_fc_fd_fe_ff_u64, ); - copy_from_slice(&mut target, &source, 8 * 12 + 4, 128); + assert_eq!(copy_from_slice(&mut target, &source, 8 * 12 + 4, 128), 66,); // Find the 12th byte, skip the first hexit, take the next 32 hexits (i.e. 16 bytesor 128 // bits). assert_eq!( @@ -517,7 +539,10 @@ mod tests { ); // Take the above and shift one bit towards the right-hand-side. - copy_from_slice(&mut target, &source, 8 * 12 + 4 + 1, 128); + assert_eq!( + copy_from_slice(&mut target, &source, 8 * 12 + 4 + 1, 128), + 66, + ); assert_eq!( target[0], 0xf8_20_18_10_0f_cf_d7_df_u64, "{:#08x} == {:#08x}", diff --git a/vortex-duckdb/src/exporter/vector.rs b/vortex-duckdb/src/exporter/vector.rs index 5e4cef97d11..c8b9e36f2eb 100644 --- a/vortex-duckdb/src/exporter/vector.rs +++ b/vortex-duckdb/src/exporter/vector.rs @@ -2,7 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex::mask::Mask; +use vortex::mask::MaskValues; +use crate::cpp::duckdb_vx_vector_set_all_valid; use crate::duckdb::Value; use crate::duckdb::VectorRef; use crate::exporter::copy_from_slice; @@ -11,44 +13,46 @@ impl VectorRef { pub(super) unsafe fn set_validity(&mut self, mask: &Mask, offset: usize, len: usize) -> bool { match mask { Mask::AllTrue(_) => { - // We only need to blank out validity if there is already a slice allocated. - // SAFETY: Caller guarantees this. - unsafe { self.set_all_true_validity(len) } + self.set_all_true_validity(); false } Mask::AllFalse(_) => { - // SAFETY: Caller guarantees this. self.set_all_false_validity(); true } - Mask::Values(arr) => { - let true_count = arr.bit_buffer().slice(offset..(offset + len)).true_count(); - if true_count == len { - unsafe { self.set_all_true_validity(len) } - } else if true_count == 0 { - self.set_all_false_validity() - } else { - let source = arr.bit_buffer().inner().as_slice(); - copy_from_slice( - unsafe { self.ensure_validity_slice(len) }, - source, - offset, - len, - ); - } - - true_count == 0 - } + Mask::Values(arr) => self.set_validity_with_array(arr, len, offset), } } - pub(super) unsafe fn set_all_true_validity(&mut self, len: usize) { - if let Some(validity) = unsafe { self.validity_bitslice_mut(len) } { - validity.fill(true); + fn set_validity_with_array(&mut self, arr: &MaskValues, len: usize, offset: usize) -> bool { + let true_count = arr.true_count(); + if true_count == arr.len() { + self.set_all_true_validity(); + return false; + } else if true_count == 0 { + self.set_all_false_validity(); + return true; + } + + let dest = unsafe { self.ensure_validity_slice(len) }; + let source = arr.bit_buffer().inner().as_slice(); + let ones = copy_from_slice(dest, source, offset, len); + if ones == 0 { + self.set_all_false_validity(); + true + } else if ones == len { + self.set_all_true_validity(); + false + } else { + false } } - pub(super) fn set_all_false_validity(&mut self) { + fn set_all_true_validity(&mut self) { + unsafe { duckdb_vx_vector_set_all_valid(self.as_ptr()) }; + } + + fn set_all_false_validity(&mut self) { self.reference_value(&Value::null(&self.logical_type())); } } From d8c156f6eb29840444268709244c23bb4a16bfd3 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 10 Apr 2026 15:14:27 +0100 Subject: [PATCH 008/250] use expect over allow in clippy (#7373) remove #[allow(....) --- benchmarks/datafusion-bench/src/lib.rs | 2 +- benchmarks/datafusion-bench/src/main.rs | 2 +- benchmarks/datafusion-bench/src/metrics.rs | 2 +- benchmarks/duckdb-bench/build.rs | 3 +-- benchmarks/random-access-bench/src/main.rs | 4 ++-- encodings/alp/benches/alp_compress.rs | 2 +- encodings/alp/src/alp/compress.rs | 6 +++--- encodings/alp/src/alp/compute/compare.rs | 2 -- encodings/alp/src/alp_rd/mod.rs | 2 +- encodings/fastlanes/benches/bit_transpose.rs | 4 +--- encodings/fastlanes/benches/bitpacking_take.rs | 4 ++-- .../fastlanes/src/bit_transpose/aarch64.rs | 6 +++--- encodings/fastlanes/src/bit_transpose/mod.rs | 2 +- .../fastlanes/src/bit_transpose/scalar.rs | 2 -- encodings/fastlanes/src/bit_transpose/x86.rs | 18 ++++++++---------- .../src/bitpacking/array/bitpack_compress.rs | 2 +- .../fastlanes/src/bitpacking/compute/take.rs | 2 +- encodings/fastlanes/src/for/compute/compare.rs | 1 - encodings/fastlanes/src/lib.rs | 2 +- encodings/fsst/benches/fsst_compress.rs | 2 +- encodings/fsst/benches/fsst_like.rs | 2 +- encodings/fsst/benches/fsst_url_compare.rs | 2 +- encodings/fsst/src/array.rs | 4 +--- encodings/fsst/src/canonical.rs | 4 ++-- encodings/fsst/src/compute/like.rs | 2 -- encodings/fsst/src/test_utils.rs | 2 +- encodings/pco/src/test.rs | 2 +- encodings/runend/benches/run_end_compress.rs | 4 ++-- encodings/runend/benches/run_end_decode.rs | 2 +- encodings/runend/benches/run_end_null_count.rs | 2 +- encodings/runend/src/ops.rs | 1 - encodings/sequence/src/compress.rs | 2 +- encodings/zstd/benches/listview_rebuild.rs | 2 +- encodings/zstd/src/array.rs | 2 +- encodings/zstd/src/test.rs | 2 +- fuzz/fuzz_targets/array_ops.rs | 2 +- fuzz/fuzz_targets/array_ops_wasm.rs | 2 -- fuzz/fuzz_targets/compress_gpu.rs | 2 +- fuzz/fuzz_targets/compress_roundtrip.rs | 1 - fuzz/fuzz_targets/file_io.rs | 1 - fuzz/fuzz_targets/fsst_like.rs | 2 +- fuzz/src/array/cast.rs | 3 +-- fuzz/src/array/compare.rs | 2 +- fuzz/src/array/mod.rs | 10 +++++----- fuzz/src/array/slice.rs | 1 - fuzz/src/compress.rs | 2 +- fuzz/src/fsst_like.rs | 2 +- fuzz/src/gpu/mod.rs | 1 - fuzz/src/lib.rs | 2 +- java/testfiles/src/main.rs | 2 +- vortex-array/benches/compare.rs | 2 +- vortex-array/benches/dict_compare.rs | 2 +- vortex-array/benches/dict_compress.rs | 2 +- vortex-array/benches/dict_mask.rs | 2 +- vortex-array/benches/dict_unreferenced_mask.rs | 10 +++++----- vortex-array/benches/expr/case_when_bench.rs | 4 ++-- vortex-array/benches/expr/large_struct_pack.rs | 2 +- vortex-array/benches/filter_bool.rs | 8 ++++---- vortex-array/benches/listview_rebuild.rs | 4 ++-- vortex-array/benches/scalar_at_struct.rs | 2 +- vortex-array/benches/scalar_subtract.rs | 2 +- vortex-array/benches/search_sorted.rs | 2 +- vortex-array/benches/take_fsl.rs | 4 ++-- vortex-array/benches/take_patches.rs | 4 ++-- vortex-array/benches/take_primitive.rs | 4 ++-- vortex-array/benches/take_struct.rs | 2 +- vortex-array/benches/varbinview_zip.rs | 2 +- vortex-array/src/array/erased.rs | 2 -- vortex-array/src/array/plugin.rs | 1 - vortex-array/src/array/typed.rs | 17 ----------------- vortex-array/src/arrays/assertions.rs | 4 +--- .../src/arrays/chunked/paired_chunks.rs | 2 +- vortex-array/src/arrays/decimal/array.rs | 2 +- vortex-array/src/arrays/dict/array.rs | 6 +----- vortex-array/src/arrays/dict/compute/mod.rs | 2 +- vortex-array/src/arrays/dict_test.rs | 2 +- .../src/arrays/filter/execute/bitbuffer.rs | 2 +- .../src/arrays/filter/execute/listview.rs | 2 +- .../src/arrays/filter/execute/primitive.rs | 2 +- vortex-array/src/arrays/list/tests.rs | 2 +- .../src/arrays/listview/compute/take.rs | 2 +- vortex-array/src/arrays/listview/rebuild.rs | 1 - vortex-array/src/arrays/masked/tests.rs | 2 +- vortex-array/src/arrays/null/compute/take.rs | 2 +- vortex-array/src/arrays/patched/array.rs | 3 +-- .../src/arrays/patched/compute/take.rs | 2 +- vortex-array/src/arrays/patched/vtable/mod.rs | 1 - .../src/arrays/primitive/compute/cast.rs | 1 - .../src/arrays/primitive/compute/take/mod.rs | 2 -- vortex-array/src/arrays/scalar_fn/array.rs | 3 --- vortex-array/src/arrays/shared/array.rs | 2 +- vortex-array/src/arrays/varbin/array.rs | 2 +- vortex-array/src/buffer.rs | 4 ++-- vortex-array/src/builders/dict/primitive.rs | 2 +- vortex-array/src/builders/tests.rs | 2 +- vortex-array/src/builders/varbinview.rs | 4 ++-- .../src/compute/conformance/search_sorted.rs | 2 +- vortex-array/src/compute/conformance/take.rs | 10 +++++----- vortex-array/src/dtype/bigint/mod.rs | 2 +- vortex-array/src/dtype/extension/typed.rs | 2 +- vortex-array/src/dtype/ptype.rs | 1 - vortex-array/src/dtype/serde/mod.rs | 2 +- vortex-array/src/executor.rs | 2 +- vortex-array/src/patches.rs | 2 +- vortex-array/src/scalar/truncation.rs | 2 +- .../src/scalar/typed_view/decimal/tests.rs | 2 -- vortex-array/src/scalar/typed_view/utf8.rs | 2 +- .../src/scalar_fn/fns/binary/numeric.rs | 1 - vortex-array/src/test_harness.rs | 2 +- vortex-bench/src/lib.rs | 4 ++-- vortex-bench/src/polarsignals/data.rs | 6 +++--- vortex-bench/src/statpopgen/builder.rs | 4 +--- .../src/statpopgen/statpopgen_benchmark.rs | 2 +- vortex-bench/src/tpch/tpchgen.rs | 6 +++--- vortex-btrblocks/benches/compress.rs | 3 +-- vortex-btrblocks/benches/compress_listview.rs | 5 ++--- vortex-buffer/benches/vortex_bitbuffer.rs | 2 -- vortex-buffer/benches/vortex_buffer.rs | 2 -- vortex-buffer/src/lib.rs | 2 -- vortex-compressor/benches/dict_encode.rs | 2 +- vortex-compressor/benches/stats_calc.rs | 3 --- vortex-compressor/src/builtins/dict/float.rs | 2 +- vortex-compressor/src/builtins/dict/integer.rs | 2 +- vortex-cuda/benches/bitpacked_cuda.rs | 4 ++-- vortex-cuda/benches/date_time_parts_cuda.rs | 4 ++-- vortex-cuda/benches/dict_cuda.rs | 4 ++-- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 6 +++--- vortex-cuda/benches/filter_cuda.rs | 4 ++-- vortex-cuda/benches/for_cuda.rs | 4 ++-- vortex-cuda/benches/runend_cuda.rs | 4 ++-- vortex-cuda/benches/throughput_cuda.rs | 3 +-- vortex-cuda/benches/transpose_patches.rs | 3 +-- vortex-cuda/benches/zstd_cuda.rs | 3 --- vortex-cuda/build.rs | 6 +++--- vortex-cuda/cub/src/filter.rs | 8 ++++---- vortex-cuda/cub/src/lib.rs | 8 +------- vortex-cuda/nvcomp/src/lib.rs | 3 +-- vortex-cuda/nvcomp/src/zstd.rs | 4 ++-- vortex-cuda/src/arrow/mod.rs | 1 - vortex-cuda/src/executor.rs | 2 +- .../src/kernel/encodings/date_time_parts.rs | 2 +- vortex-cuda/src/kernel/patches/mod.rs | 2 +- vortex-cuda/src/kernel/patches/types.rs | 6 +++--- vortex-cuda/src/pinned.rs | 4 +--- vortex-cxx/src/lib.rs | 1 + vortex-datafusion/src/persistent/cache.rs | 2 +- vortex-datafusion/src/persistent/opener.rs | 2 +- vortex-datafusion/src/v2/source.rs | 2 +- vortex-duckdb/build.rs | 4 ++-- vortex-duckdb/src/convert/expr.rs | 1 - vortex-duckdb/src/copy.rs | 2 +- vortex-duckdb/src/datasource.rs | 4 ++-- vortex-duckdb/src/duckdb/logical_type.rs | 7 ------- .../src/duckdb/table_function/init.rs | 1 - vortex-duckdb/src/lib.rs | 2 +- vortex-duckdb/src/multi_file.rs | 1 - vortex-error/tests/fmt.rs | 2 +- vortex-ffi/build.rs | 3 +-- vortex-ffi/examples/hello_vortex.rs | 2 +- vortex-ffi/src/array.rs | 2 +- vortex-ffi/src/dtype.rs | 4 ++-- vortex-ffi/src/expression.rs | 2 +- vortex-ffi/src/file.rs | 2 -- vortex-ffi/src/lib.rs | 1 - vortex-ffi/src/log.rs | 2 +- vortex-ffi/src/macros.rs | 8 ++++---- vortex-ffi/src/ptype.rs | 2 +- vortex-ffi/src/sink.rs | 2 +- vortex-file/src/lib.rs | 2 +- vortex-file/src/tests.rs | 2 +- vortex-file/tests/test_write_table.rs | 2 +- vortex-io/src/limit.rs | 2 +- vortex-io/src/runtime/current.rs | 2 +- vortex-io/src/runtime/handle.rs | 2 +- vortex-io/src/runtime/tests.rs | 2 +- vortex-mask/benches/intersect_by_rank.rs | 3 +-- vortex-mask/src/bitops.rs | 2 +- vortex-mask/src/tests.rs | 3 --- vortex-metrics/src/histogram.rs | 4 ++-- vortex-metrics/src/timer.rs | 4 ++-- vortex-python/src/arrays/mod.rs | 2 +- vortex-python/src/arrow.rs | 2 +- vortex-python/src/object_store/registry.rs | 2 +- vortex-python/src/scalar/factory.rs | 1 - vortex-sqllogictest/build.rs | 2 +- vortex-sqllogictest/src/utils.rs | 2 +- .../src/fixtures/arrays/datasets/mod.rs | 1 - .../src/fixtures/arrays/synthetic/mod.rs | 2 +- vortex-test/e2e-cuda/src/lib.rs | 2 +- vortex-tui/src/browse/mod.rs | 1 - vortex-tui/src/browse/ui/query.rs | 4 ++-- vortex-tui/src/datafusion_helper.rs | 2 +- .../benches/common_encoding_tree_throughput.rs | 11 +++++------ vortex/benches/pipeline.rs | 6 +----- vortex/benches/single_encoding_throughput.rs | 7 +++---- vortex/examples/compression_showcase.rs | 2 +- vortex/examples/tracing_vortex.rs | 4 ++-- 197 files changed, 235 insertions(+), 350 deletions(-) diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index 94ea053ba8f..469c9be6177 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -30,7 +30,7 @@ use vortex_datafusion::VortexFormat; use vortex_datafusion::VortexFormatFactory; use vortex_datafusion::VortexTableOptions; -#[allow(clippy::expect_used)] +#[expect(clippy::expect_used)] pub fn get_session_context() -> SessionContext { let mut rt_builder = RuntimeEnvBuilder::new(); diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 8d85dd8f4eb..920d4346361 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -156,7 +156,7 @@ async fn main() -> anyhow::Result<()> { // Collect execution plans for metrics if show_metrics is enabled // Structure: (query_idx, format, execution_plan) - #[allow(clippy::type_complexity)] + #[expect(clippy::type_complexity)] let collected_plans: Arc)>>> = Arc::new(Mutex::new(Vec::new())); let show_metrics = args.show_metrics; diff --git a/benchmarks/datafusion-bench/src/metrics.rs b/benchmarks/datafusion-bench/src/metrics.rs index bb29d447469..9f41d75398b 100644 --- a/benchmarks/datafusion-bench/src/metrics.rs +++ b/benchmarks/datafusion-bench/src/metrics.rs @@ -47,7 +47,7 @@ impl MetricsSetExt for MetricsSet { } } - #[allow(clippy::disallowed_types)] + #[expect(clippy::disallowed_types)] fn aggregate(&self) -> Self { use std::collections::HashMap; let mut map = HashMap::new(); diff --git a/benchmarks/duckdb-bench/build.rs b/benchmarks/duckdb-bench/build.rs index 4232c385bbb..7e2b12e5d18 100644 --- a/benchmarks/duckdb-bench/build.rs +++ b/benchmarks/duckdb-bench/build.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::expect_used)] +#![expect(clippy::unwrap_used)] /// Adds a dynamic linker runtime path pointing to the DuckDB dylib dir. /// diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 2fb617e8787..852332528a8 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -91,14 +91,14 @@ fn generate_indices(dataset: &dyn BenchDataset, pattern: AccessPattern) -> Vec 0, row_count > 0). - #[allow(clippy::unwrap_used)] + #[expect(clippy::unwrap_used)] let exp = Exp::new(rate).unwrap(); let mut indices = Vec::with_capacity(POISSON_EXPECTED_COUNT); let mut pos = 0.0_f64; loop { let gap: f64 = exp.sample(&mut rng); pos += gap; - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let idx = pos as u64; if idx >= row_count { break; diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index f4706c30bb2..c8745926845 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 56050b05c6f..f67779996da 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -173,7 +173,7 @@ mod tests { } #[test] - #[allow(clippy::approx_constant)] // Clippy objects to 2.718, an approximation of e, the base of the natural logarithm. + #[expect(clippy::approx_constant)] // Clippy objects to 2.718, an approximation of e, the base of the natural logarithm. fn test_patched_compress() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values.clone(), Validity::NonNullable); @@ -190,7 +190,7 @@ mod tests { } #[test] - #[allow(clippy::approx_constant)] // Clippy objects to 2.718, an approximation of e, the base of the natural logarithm. + #[expect(clippy::approx_constant)] // Clippy objects to 2.718, an approximation of e, the base of the natural logarithm. fn test_compress_ignores_invalid_exceptional_values() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values, Validity::from_iter([true, true, false, true])); @@ -207,7 +207,7 @@ mod tests { } #[test] - #[allow(clippy::approx_constant)] // ALP doesn't like E + #[expect(clippy::approx_constant)] // ALP doesn't like E fn test_nullable_patched_scalar_at() { let array = PrimitiveArray::from_option_iter([ Some(1.234f64), diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index 8c4e272bcaf..a6f475a3df2 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -213,14 +213,12 @@ mod tests { vec![1234; 1025] ); - #[allow(clippy::excessive_precision)] let r_eq = alp_scalar_compare(encoded.as_view(), 1.234444_f32, CompareOperator::Eq) .unwrap() .unwrap(); let expected = BoolArray::from_iter([false; 1025]); assert_arrays_eq!(r_eq, expected); - #[allow(clippy::excessive_precision)] let r_neq = alp_scalar_compare(encoded.as_view(), 1.234444f32, CompareOperator::NotEq) .unwrap() .unwrap(); diff --git a/encodings/alp/src/alp_rd/mod.rs b/encodings/alp/src/alp_rd/mod.rs index 261f95f2ae6..29356ad812f 100644 --- a/encodings/alp/src/alp_rd/mod.rs +++ b/encodings/alp/src/alp_rd/mod.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] pub use array::*; use vortex_array::ExecutionCtx; diff --git a/encodings/fastlanes/benches/bit_transpose.rs b/encodings/fastlanes/benches/bit_transpose.rs index 4bc9027cc28..08c3ffb12e5 100644 --- a/encodings/fastlanes/benches/bit_transpose.rs +++ b/encodings/fastlanes/benches/bit_transpose.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] - use divan::Bencher; use vortex_fastlanes::bit_transpose::scalar::transpose_bits_scalar; use vortex_fastlanes::bit_transpose::scalar::untranspose_bits_scalar; @@ -12,7 +10,7 @@ fn main() { } /// Generate deterministic test data. -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn generate_test_data(seed: usize) -> [u8; 128] { let mut data = [0u8; 128]; for (i, byte) in data.iter_mut().enumerate() { diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index 61bd234c0f1..9298a426d99 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use divan::Bencher; use rand::RngExt; diff --git a/encodings/fastlanes/src/bit_transpose/aarch64.rs b/encodings/fastlanes/src/bit_transpose/aarch64.rs index 7a854b43a3f..85dbb1e0690 100644 --- a/encodings/fastlanes/src/bit_transpose/aarch64.rs +++ b/encodings/fastlanes/src/bit_transpose/aarch64.rs @@ -113,7 +113,7 @@ static SCATTER_8X8_NEON: [[u8; 16]; 4] = [ ]; /// Perform 8x8 bit transpose on two u64s packed in a `uint64x2_t`. -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(unsafe_op_in_unsafe_fn)] unsafe fn bit_transpose_8x8_neon(mut v: uint64x2_t) -> uint64x2_t { let mask1 = vdupq_n_u64(TRANSPOSE_2X2); let t = vandq_u64(veorq_u64(v, vshrq_n_u64::<7>(v)), mask1); @@ -137,7 +137,7 @@ unsafe fn bit_transpose_8x8_neon(mut v: uint64x2_t) -> uint64x2_t { /// /// # Safety /// Requires `AArch64` with NEON (always available on `AArch64`). -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(unsafe_op_in_unsafe_fn)] #[inline(never)] pub unsafe fn transpose_bits_neon(input: &[u8; 128], output: &mut [u8; 128]) { // Load all 128 input bytes into two uint8x16x4_t tables (64 bytes each) @@ -197,7 +197,7 @@ pub unsafe fn transpose_bits_neon(input: &[u8; 128], output: &mut [u8; 128]) { /// /// # Safety /// Requires `AArch64` with NEON (always available on `AArch64`). -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(unsafe_op_in_unsafe_fn)] #[inline(never)] pub unsafe fn untranspose_bits_neon(input: &[u8; 128], output: &mut [u8; 128]) { // Load scatter indices (SCATTER_8X8 is self-inverse, so same table un-scatters) diff --git a/encodings/fastlanes/src/bit_transpose/mod.rs b/encodings/fastlanes/src/bit_transpose/mod.rs index 01e5d4a0b1c..99b01bc242c 100644 --- a/encodings/fastlanes/src/bit_transpose/mod.rs +++ b/encodings/fastlanes/src/bit_transpose/mod.rs @@ -98,7 +98,7 @@ pub fn untranspose_bits(input: &[u8; 128], output: &mut [u8; 128]) { } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn generate_test_data(seed: u8) -> [u8; 128] { let mut data = [0u8; 128]; for (i, byte) in data.iter_mut().enumerate() { diff --git a/encodings/fastlanes/src/bit_transpose/scalar.rs b/encodings/fastlanes/src/bit_transpose/scalar.rs index 425e74087ab..152b2a830c1 100644 --- a/encodings/fastlanes/src/bit_transpose/scalar.rs +++ b/encodings/fastlanes/src/bit_transpose/scalar.rs @@ -12,7 +12,6 @@ use crate::bit_transpose::TRANSPOSE_8X8; /// This version uses 64-bit gather + parallel bit operations instead of /// extracting bits one by one. Typically 5-10x faster than the basic scalar version. #[inline(never)] -#[allow(dead_code)] pub fn transpose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { // Helper to perform 8x8 bit transpose on a u64 (each byte becomes a row) fn transpose_8x8(mut x: u64) -> u64 { @@ -85,7 +84,6 @@ pub fn transpose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { /// Fast scalar untranspose using the 8x8 bit matrix transpose algorithm. #[inline(never)] -#[allow(dead_code)] pub fn untranspose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { fn transpose_8x8(mut x: u64) -> u64 { let t = (x ^ (x >> 7)) & TRANSPOSE_2X2; diff --git a/encodings/fastlanes/src/bit_transpose/x86.rs b/encodings/fastlanes/src/bit_transpose/x86.rs index 6277d76137f..9eaef248d28 100644 --- a/encodings/fastlanes/src/bit_transpose/x86.rs +++ b/encodings/fastlanes/src/bit_transpose/x86.rs @@ -45,8 +45,7 @@ pub fn has_vbmi() -> bool { /// Requires BMI2 support. Check with `has_bmi2()` before calling. #[target_feature(enable = "bmi2")] #[inline(never)] -#[allow(clippy::too_many_lines)] -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(clippy::too_many_lines)] pub unsafe fn transpose_bits_bmi2(input: &[u8; 128], output: &mut [u8; 128]) { // Helper to gather 8 bytes at stride 16 into a u64 fn gather(input: &[u8; 128], base: usize) -> u64 { @@ -233,8 +232,7 @@ pub unsafe fn transpose_bits_bmi2(input: &[u8; 128], output: &mut [u8; 128]) { /// Requires BMI2 support. Check with `has_bmi2()` before calling. #[target_feature(enable = "bmi2")] #[inline(never)] -#[allow(clippy::too_many_lines)] -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(clippy::too_many_lines)] pub unsafe fn untranspose_bits_bmi2(input: &[u8; 128], output: &mut [u8; 128]) { // Helper: scatter a u64 to 8 output bytes at stride 16 fn scatter(output: &mut [u8; 128], base: usize, val: u64) { @@ -492,9 +490,9 @@ static SCATTER_8X8: [u8; 64] = [ /// Requires AVX-512F, AVX-512BW, and AVX-512VBMI support. #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "avx512vbmi")] #[inline(never)] -#[allow(clippy::cast_possible_wrap)] -#[allow(clippy::cast_ptr_alignment)] -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(clippy::cast_possible_wrap)] +#[expect(clippy::cast_ptr_alignment)] +#[expect(unsafe_op_in_unsafe_fn)] pub unsafe fn transpose_bits_vbmi(input: &[u8; 128], output: &mut [u8; 128]) { // Load all 128 input bytes into two ZMM registers let in_lo = _mm512_loadu_si512(input.as_ptr().cast::<__m512i>()); @@ -547,9 +545,9 @@ pub unsafe fn transpose_bits_vbmi(input: &[u8; 128], output: &mut [u8; 128]) { /// Requires AVX-512F, AVX-512BW, and AVX-512VBMI support. #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "avx512vbmi")] #[inline(never)] -#[allow(clippy::cast_possible_wrap)] -#[allow(clippy::cast_ptr_alignment)] -#[allow(unsafe_op_in_unsafe_fn)] +#[expect(clippy::cast_possible_wrap)] +#[expect(clippy::cast_ptr_alignment)] +#[expect(unsafe_op_in_unsafe_fn)] pub unsafe fn untranspose_bits_vbmi(input: &[u8; 128], output: &mut [u8; 128]) { // For untranspose, we gather consecutive bytes from transposed layout, // then scatter back to stride-16 positions diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 727f13fc35e..b2b6ce8648c 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -36,7 +36,7 @@ pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult( } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] mod test { use rand::RngExt; use rand::distr::Uniform; diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index e5eae1b90b0..8cd2fb9c011 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -220,7 +220,6 @@ mod tests { #[test] fn compare_large_constant() { let reference = Scalar::from(-9219218377546224477i64); - #[allow(clippy::cast_possible_truncation)] let lhs = for_arr( PrimitiveArray::new( buffer![0i64, 9654309310445864926u64 as i64], diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index a00e13b53fa..763c55484ea 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] pub use bitpacking::*; pub use delta::*; diff --git a/encodings/fsst/benches/fsst_compress.rs b/encodings/fsst/benches/fsst_compress.rs index e84eef9cbce..7102e3fa98a 100644 --- a/encodings/fsst/benches/fsst_compress.rs +++ b/encodings/fsst/benches/fsst_compress.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use std::sync::LazyLock; diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index b4b0e959351..57489ba158e 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use std::fmt; use std::sync::LazyLock; diff --git a/encodings/fsst/benches/fsst_url_compare.rs b/encodings/fsst/benches/fsst_url_compare.rs index a8572650efb..705a8c0cef4 100644 --- a/encodings/fsst/benches/fsst_url_compare.rs +++ b/encodings/fsst/benches/fsst_url_compare.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use std::sync::LazyLock; diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 1a5cc6ef02b..5526276aeb4 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -311,10 +311,8 @@ impl VTable for FSST { /// Lengths of the original values before compression, can be compressed. pub(crate) const UNCOMPRESSED_LENGTHS_SLOT: usize = 0; /// The offsets array for the FSST-compressed codes. -#[allow(dead_code, reason = "reserved for back-compat slot numbering")] pub(crate) const CODES_OFFSETS_SLOT: usize = 1; /// The validity bitmap for the compressed codes. -#[allow(dead_code, reason = "reserved for back-compat slot numbering")] pub(crate) const CODES_VALIDITY_SLOT: usize = 2; pub(crate) const NUM_SLOTS: usize = 3; pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = @@ -526,7 +524,7 @@ impl FSSTData { } /// Validate using the decomposed components (codes bytes + offsets + nullability). - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] fn validate_parts( symbols: &Buffer, symbol_lengths: &Buffer, diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 14d06de20f4..35730c9a1f0 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -59,7 +59,7 @@ pub(crate) fn fsst_decode_views( .clone() .execute::(ctx)?; - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let total_size: usize = match_each_integer_ptype!(uncompressed_lens_array.ptype(), |P| { uncompressed_lens_array .as_slice::

() @@ -148,7 +148,7 @@ mod tests { } fn make_data_chunked() -> (ChunkedArray, Vec>>) { - #[allow(clippy::type_complexity)] + #[expect(clippy::type_complexity)] let (arr_vec, data_vec): (Vec, Vec>>>) = (0..10) .map(|_| { let (array, data) = make_data(); diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index 2e618ef6588..e5e8bba9cde 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] - use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; diff --git a/encodings/fsst/src/test_utils.rs b/encodings/fsst/src/test_utils.rs index 421a5d4c3f5..d0de924ad68 100644 --- a/encodings/fsst/src/test_utils.rs +++ b/encodings/fsst/src/test_utils.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use rand::RngExt; use rand::SeedableRng; diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index b9a0f87e52d..f674fb7d1bb 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] use std::sync::LazyLock; diff --git a/encodings/runend/benches/run_end_compress.rs b/encodings/runend/benches/run_end_compress.rs index 024f3281c36..a177aa25021 100644 --- a/encodings/runend/benches/run_end_compress.rs +++ b/encodings/runend/benches/run_end_compress.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use itertools::repeat_n; @@ -77,7 +77,7 @@ fn decompress(bencher: Bencher, (length, run_step): (usize, usi } #[divan::bench(args = BENCH_ARGS)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn take_indices(bencher: Bencher, (length, run_step): (usize, usize)) { let values = PrimitiveArray::new( (0..length) diff --git a/encodings/runend/benches/run_end_decode.rs b/encodings/runend/benches/run_end_decode.rs index b2ee2aaf3ab..d44611e1e88 100644 --- a/encodings/runend/benches/run_end_decode.rs +++ b/encodings/runend/benches/run_end_decode.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] use std::fmt; diff --git a/encodings/runend/benches/run_end_null_count.rs b/encodings/runend/benches/run_end_null_count.rs index e101abf6799..f1473cc47aa 100644 --- a/encodings/runend/benches/run_end_null_count.rs +++ b/encodings/runend/benches/run_end_null_count.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/encodings/runend/src/ops.rs b/encodings/runend/src/ops.rs index c27885cc39a..b8a199bf51a 100644 --- a/encodings/runend/src/ops.rs +++ b/encodings/runend/src/ops.rs @@ -156,7 +156,6 @@ mod tests { } #[test] - #[allow(clippy::cognitive_complexity)] fn slice_along_run_boundaries() { // Create a runend array with runs: [1, 1, 1] [4, 4, 4] [2, 2] [5, 5, 5, 5] // Run ends at indices: 3, 6, 8, 12 diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index 98bd12bb027..0ab08cf057f 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -146,7 +146,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu #[cfg(test)] mod tests { - #[allow(unused_imports)] + #[expect(unused_imports)] use itertools::Itertools; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index 82b16c392fa..e8696d2d0d4 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use vortex_array::IntoArray; diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index e8a3612b06d..a11d48defb6 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -1035,7 +1035,7 @@ impl OperationsVTable for Zstd { } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] mod tests { use vortex_buffer::ByteBuffer; diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 01800577d7b..76a8909645e 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; diff --git a/fuzz/fuzz_targets/array_ops.rs b/fuzz/fuzz_targets/array_ops.rs index dded332b64c..17324abe3c3 100644 --- a/fuzz/fuzz_targets/array_ops.rs +++ b/fuzz/fuzz_targets/array_ops.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![no_main] -#![allow(clippy::unwrap_used, clippy::result_large_err)] +#![expect(clippy::unwrap_used)] use std::str::FromStr; diff --git a/fuzz/fuzz_targets/array_ops_wasm.rs b/fuzz/fuzz_targets/array_ops_wasm.rs index ddbec35e505..35b31a54fd6 100644 --- a/fuzz/fuzz_targets/array_ops_wasm.rs +++ b/fuzz/fuzz_targets/array_ops_wasm.rs @@ -7,8 +7,6 @@ //! - `LLVMFuzzerTestOneInput` - the fuzzing entry point //! - `wasmfuzz_malloc` / `wasmfuzz_free` - memory allocation for wasmfuzz to pass input data -#![allow(clippy::unwrap_used, clippy::result_large_err)] - use std::alloc::Layout; use std::alloc::alloc; use std::alloc::dealloc; diff --git a/fuzz/fuzz_targets/compress_gpu.rs b/fuzz/fuzz_targets/compress_gpu.rs index eea7ff6c5e3..f88df52d2d5 100644 --- a/fuzz/fuzz_targets/compress_gpu.rs +++ b/fuzz/fuzz_targets/compress_gpu.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![no_main] -#![allow(clippy::unwrap_used, clippy::result_large_err)] +#![expect(clippy::unwrap_used)] use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; diff --git a/fuzz/fuzz_targets/compress_roundtrip.rs b/fuzz/fuzz_targets/compress_roundtrip.rs index 22e2ee80a8e..bb5b9e464ce 100644 --- a/fuzz/fuzz_targets/compress_roundtrip.rs +++ b/fuzz/fuzz_targets/compress_roundtrip.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![no_main] -#![allow(clippy::unwrap_used, clippy::result_large_err)] use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 7d728905148..92e45c5b0c0 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![no_main] -#![allow(clippy::result_large_err)] use itertools::Itertools; use libfuzzer_sys::Corpus; diff --git a/fuzz/fuzz_targets/fsst_like.rs b/fuzz/fuzz_targets/fsst_like.rs index 8e03badff00..bf61b2430cf 100644 --- a/fuzz/fuzz_targets/fsst_like.rs +++ b/fuzz/fuzz_targets/fsst_like.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![no_main] -#![allow(clippy::unwrap_used, clippy::result_large_err)] +#![expect(clippy::unwrap_used)] use std::str::FromStr; diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index a6112bec0ff..f9ddba1fd31 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -31,7 +31,6 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult VortexResult { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] Ok(Some( PrimitiveArray::new( array diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index e6d069d47b1..849a7fdd694 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -123,7 +123,7 @@ pub fn compare_canonical_array( let binary_value = value.as_binary(); compare_to( // Don't understand the lifetime problem here but identity map makes it go away - #[allow(clippy::map_identity)] + #[expect(clippy::map_identity)] iter.map(|v| v), binary_value.value().vortex_expect("nulls handled before"), operator, diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index b0dd2c2b242..4576eb7b167 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -564,7 +564,7 @@ pub fn compress_array(array: &ArrayRef, _strategy: CompressorStrategy) -> ArrayR /// - `Ok(true)` - keep in corpus /// - `Ok(false)` - reject from corpus /// - `Err(_)` - a bug was found -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { let FuzzArrayAction { array, actions } = fuzz_action; let mut current_array = array; @@ -678,7 +678,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { Ok(true) // Keep in corpus } -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] fn assert_search_sorted( array: ArrayRef, s: Scalar, @@ -705,7 +705,7 @@ fn assert_search_sorted( } /// Assert two arrays are equal. -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuzzResult<()> { if lhs.dtype() != rhs.dtype() { return Err(VortexFuzzError::DTypeMismatch( @@ -746,7 +746,7 @@ pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuz } /// Assert two scalars are equal. -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn assert_scalar_eq(lhs: &Scalar, rhs: &Scalar, step: usize) -> VortexFuzzResult<()> { if lhs != rhs { return Err(VortexFuzzError::ScalarMismatch( @@ -760,7 +760,7 @@ pub fn assert_scalar_eq(lhs: &Scalar, rhs: &Scalar, step: usize) -> VortexFuzzRe } /// Assert two min/max results are equal. -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn assert_min_max_eq( lhs: &Option, rhs: &Option, diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index 65952ee49da..eeb1b8adf52 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -22,7 +22,6 @@ use vortex_array::match_each_native_ptype; use vortex_array::validity::Validity; use vortex_error::VortexResult; -#[allow(clippy::unnecessary_fallible_conversions)] pub fn slice_canonical_array( array: &ArrayRef, start: usize, diff --git a/fuzz/src/compress.rs b/fuzz/src/compress.rs index 9ca6fadb348..ab1d2cc38f9 100644 --- a/fuzz/src/compress.rs +++ b/fuzz/src/compress.rs @@ -59,7 +59,7 @@ impl<'a> Arbitrary<'a> for FuzzCompressRoundtrip { /// - `Ok(true)` - keep in corpus /// - `Ok(false)` - reject from corpus /// - `Err(_)` - a bug was found -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn run_compress_roundtrip(fuzz: FuzzCompressRoundtrip) -> crate::error::VortexFuzzResult { use crate::error::Backtrace; use crate::error::VortexFuzzError; diff --git a/fuzz/src/fsst_like.rs b/fuzz/src/fsst_like.rs index b962c4244c8..a36e6b91974 100644 --- a/fuzz/src/fsst_like.rs +++ b/fuzz/src/fsst_like.rs @@ -91,7 +91,7 @@ impl<'a> Arbitrary<'a> for FuzzFsstLike { /// - `Ok(true)` — keep in corpus /// - `Ok(false)` — reject (e.g. too few strings) /// - `Err(_)` — mismatch found (bug) -#[allow(clippy::result_large_err)] +#[expect(clippy::result_large_err)] pub fn run_fsst_like_fuzz(fuzz: FuzzFsstLike) -> VortexFuzzResult { let FuzzFsstLike { strings, diff --git a/fuzz/src/gpu/mod.rs b/fuzz/src/gpu/mod.rs index 6d762813000..57b233bb686 100644 --- a/fuzz/src/gpu/mod.rs +++ b/fuzz/src/gpu/mod.rs @@ -93,7 +93,6 @@ fn arbitrary_gpu_primitive_dtype(u: &mut Unstructured) -> Result VortexFuzzResult { use vortex_cuda::CanonicalCudaExt; use vortex_cuda::CudaSession; diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index f6712e3540c..5bdc1b33485 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::use_debug)] +#![expect(clippy::use_debug)] mod array; pub mod compress; diff --git a/java/testfiles/src/main.rs b/java/testfiles/src/main.rs index 1db61c7f08b..549fbce2624 100644 --- a/java/testfiles/src/main.rs +++ b/java/testfiles/src/main.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::expect_used)] +#![expect(clippy::expect_used)] use std::path::Path; diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 4aceffd6de1..6fe5be688e9 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/dict_compare.rs b/vortex-array/benches/dict_compare.rs index b361ba1006e..602ad8638f7 100644 --- a/vortex-array/benches/dict_compare.rs +++ b/vortex-array/benches/dict_compare.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use std::str::from_utf8; diff --git a/vortex-array/benches/dict_compress.rs b/vortex-array/benches/dict_compress.rs index 5ecf06378cb..401d250f813 100644 --- a/vortex-array/benches/dict_compress.rs +++ b/vortex-array/benches/dict_compress.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::distr::Distribution; diff --git a/vortex-array/benches/dict_mask.rs b/vortex-array/benches/dict_mask.rs index 9f0be5e585d..6f60691cc97 100644 --- a/vortex-array/benches/dict_mask.rs +++ b/vortex-array/benches/dict_mask.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/dict_unreferenced_mask.rs b/vortex-array/benches/dict_unreferenced_mask.rs index 24cf366f692..1ec143682af 100644 --- a/vortex-array/benches/dict_unreferenced_mask.rs +++ b/vortex-array/benches/dict_unreferenced_mask.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; @@ -32,7 +32,7 @@ fn bench_many_codes_few_values(bencher: Bencher, num_values: i32) { let values = PrimitiveArray::from_iter(0..num_values).into_array(); // Create codes that randomly reference the values - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let codes = PrimitiveArray::from_iter( (0..num_codes).map(|_| rng.random_range(0..num_values as usize) as u32), ) @@ -63,7 +63,7 @@ fn bench_many_nulls(bencher: Bencher, fraction_valid: f64) { let values = PrimitiveArray::from_iter(0..num_values).into_array(); // Create codes with many nulls based on fraction_valid - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let codes = PrimitiveArray::from_option_iter((0..num_codes).map(|_| { rng.random_bool(fraction_valid) .then(|| rng.random_range(0..num_values as usize) as u32) @@ -94,11 +94,11 @@ fn bench_sparse_coverage(bencher: Bencher, fraction_coverage: f64) { let values = PrimitiveArray::from_iter(0..num_values).into_array(); // Calculate how many unique values we'll actually reference - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let num_referenced = (num_values as f64 * fraction_coverage).max(1.0) as usize; // Create codes that only reference a subset of values - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let codes = PrimitiveArray::from_iter( (0..num_codes).map(|_| rng.random_range(0..num_referenced) as u32), ) diff --git a/vortex-array/benches/expr/case_when_bench.rs b/vortex-array/benches/expr/case_when_bench.rs index b49ba4944a0..7ab1d3d4bfc 100644 --- a/vortex-array/benches/expr/case_when_bench.rs +++ b/vortex-array/benches/expr/case_when_bench.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use std::sync::LazyLock; diff --git a/vortex-array/benches/expr/large_struct_pack.rs b/vortex-array/benches/expr/large_struct_pack.rs index c114581bce0..31471629cd9 100644 --- a/vortex-array/benches/expr/large_struct_pack.rs +++ b/vortex-array/benches/expr/large_struct_pack.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use vortex_array::dtype::DType; diff --git a/vortex-array/benches/filter_bool.rs b/vortex-array/benches/filter_bool.rs index c346f653814..f947518d6cd 100644 --- a/vortex-array/benches/filter_bool.rs +++ b/vortex-array/benches/filter_bool.rs @@ -6,10 +6,10 @@ //! Tests multiple mask patterns (mostly-true, mostly-false, random, correlated runs) //! with both uniform-random and power-law distributions, across array sizes from 1K to 250K. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::cast_sign_loss)] -#![allow(clippy::cast_precision_loss)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::cast_sign_loss)] +#![expect(clippy::cast_precision_loss)] use divan::Bencher; use rand::prelude::*; diff --git a/vortex-array/benches/listview_rebuild.rs b/vortex-array/benches/listview_rebuild.rs index 6affedca45f..5ce7fea1128 100644 --- a/vortex-array/benches/listview_rebuild.rs +++ b/vortex-array/benches/listview_rebuild.rs @@ -3,8 +3,8 @@ //! Benchmarks for ListView rebuild across different element types and scenarios. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use divan::Bencher; use vortex_array::IntoArray; diff --git a/vortex-array/benches/scalar_at_struct.rs b/vortex-array/benches/scalar_at_struct.rs index a4ab5c32df8..7563a149f8c 100644 --- a/vortex-array/benches/scalar_at_struct.rs +++ b/vortex-array/benches/scalar_at_struct.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index 0d026db5d3e..bdbcdd36f3c 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/search_sorted.rs b/vortex-array/benches/search_sorted.rs index 31ed741765a..2b5f7fccbac 100644 --- a/vortex-array/benches/search_sorted.rs +++ b/vortex-array/benches/search_sorted.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/take_fsl.rs b/vortex-array/benches/take_fsl.rs index 5c66847669e..baf2d4ede37 100644 --- a/vortex-array/benches/take_fsl.rs +++ b/vortex-array/benches/take_fsl.rs @@ -7,8 +7,8 @@ //! - Number of indices to take //! - Fixed size list length (elements per list) -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/take_patches.rs b/vortex-array/benches/take_patches.rs index 5360aa57c7e..85d7f6ffcd0 100644 --- a/vortex-array/benches/take_patches.rs +++ b/vortex-array/benches/take_patches.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/take_primitive.rs b/vortex-array/benches/take_primitive.rs index f5f3ce16c75..a290aaa6fa1 100644 --- a/vortex-array/benches/take_primitive.rs +++ b/vortex-array/benches/take_primitive.rs @@ -5,8 +5,8 @@ //! //! Both are tracked by number of indices/codes for fair comparison. -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::distr::Uniform; diff --git a/vortex-array/benches/take_struct.rs b/vortex-array/benches/take_struct.rs index 2eabaab62dc..6346bb705ac 100644 --- a/vortex-array/benches/take_struct.rs +++ b/vortex-array/benches/take_struct.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use rand::RngExt; diff --git a/vortex-array/benches/varbinview_zip.rs b/vortex-array/benches/varbinview_zip.rs index b3ce220f558..b7283bd7c52 100644 --- a/vortex-array/benches/varbinview_zip.rs +++ b/vortex-array/benches/varbinview_zip.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use vortex_array::IntoArray; diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index c2fea73dd73..cfefe316ca4 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -125,8 +125,6 @@ impl ArrayEq for ArrayRef { self.0.dyn_array_eq(other, precision) } } - -#[allow(clippy::same_name_method)] impl ArrayRef { /// Returns the length of the array. #[inline] diff --git a/vortex-array/src/array/plugin.rs b/vortex-array/src/array/plugin.rs index bff76a30c59..e1ea3657d70 100644 --- a/vortex-array/src/array/plugin.rs +++ b/vortex-array/src/array/plugin.rs @@ -40,7 +40,6 @@ pub trait ArrayPlugin: 'static + Send + Sync { /// /// The returned array doesn't necessary have to match this plugin's encoding ID. This is /// useful for implementing back-compat logic and deserializing arrays into the new version. - #[allow(clippy::too_many_arguments)] fn deserialize( &self, dtype: &DType, diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 0d8239af589..55656409128 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -192,7 +192,6 @@ pub struct Array { _phantom: PhantomData, } -#[allow(clippy::same_name_method)] impl Array { /// Create a typed array from explicit construction parameters. pub fn try_from_parts(new: ArrayParts) -> VortexResult { @@ -229,7 +228,6 @@ impl Array { /// /// # Safety /// Caller must ensure the `ArrayRef` contains an `ArrayInner`. - #[allow(dead_code)] pub(crate) unsafe fn from_array_ref_unchecked(array: ArrayRef) -> Self { Self { inner: array, @@ -342,52 +340,42 @@ impl Array { /// Public API methods that shadow `DynArray` / `ArrayRef` methods. impl Array { - #[allow(clippy::same_name_method)] pub fn slice(&self, range: std::ops::Range) -> VortexResult { self.inner.slice(range) } - #[allow(clippy::same_name_method)] pub fn scalar_at(&self, index: usize) -> VortexResult { self.inner.scalar_at(index) } - #[allow(clippy::same_name_method)] pub fn filter(&self, mask: vortex_mask::Mask) -> VortexResult { self.inner.filter(mask) } - #[allow(clippy::same_name_method)] pub fn take(&self, indices: ArrayRef) -> VortexResult { self.inner.take(indices) } - #[allow(clippy::same_name_method)] pub fn validity(&self) -> VortexResult { self.inner.validity() } - #[allow(clippy::same_name_method)] pub fn is_valid(&self, index: usize) -> VortexResult { self.inner.is_valid(index) } - #[allow(clippy::same_name_method)] pub fn is_invalid(&self, index: usize) -> VortexResult { self.inner.is_invalid(index) } - #[allow(clippy::same_name_method)] pub fn all_valid(&self) -> VortexResult { self.inner.all_valid() } - #[allow(clippy::same_name_method)] pub fn all_invalid(&self) -> VortexResult { self.inner.all_invalid() } - #[allow(clippy::same_name_method)] pub fn to_canonical(&self) -> VortexResult { self.inner.to_canonical() } @@ -396,7 +384,6 @@ impl Array { self.inner.nbytes() } - #[allow(clippy::same_name_method)] pub fn nbuffers(&self) -> usize { self.inner.nbuffers() } @@ -405,17 +392,14 @@ impl Array { self.inner.as_constant() } - #[allow(clippy::same_name_method)] pub fn valid_count(&self) -> VortexResult { self.inner.valid_count() } - #[allow(clippy::same_name_method)] pub fn invalid_count(&self) -> VortexResult { self.inner.invalid_count() } - #[allow(clippy::same_name_method)] pub fn append_to_builder( &self, builder: &mut dyn crate::builders::ArrayBuilder, @@ -424,7 +408,6 @@ impl Array { self.inner.append_to_builder(builder, ctx) } - #[allow(clippy::same_name_method)] pub fn validity_mask(&self) -> VortexResult { self.inner.validity_mask() } diff --git a/vortex-array/src/arrays/assertions.rs b/vortex-array/src/arrays/assertions.rs index 990dde4b37d..2dab3a23c44 100644 --- a/vortex-array/src/arrays/assertions.rs +++ b/vortex-array/src/arrays/assertions.rs @@ -105,9 +105,7 @@ macro_rules! assert_arrays_eq { right.display_values() ); - #[allow(deprecated)] let left = left.clone(); - #[allow(deprecated)] let right = right.clone(); $crate::arrays::assert_arrays_eq_impl(&left, &right); }}; @@ -116,7 +114,7 @@ macro_rules! assert_arrays_eq { /// Implementation of `assert_arrays_eq!` — called by the macro after converting inputs to /// `ArrayRef`. #[track_caller] -#[allow(clippy::panic)] +#[expect(clippy::panic)] pub fn assert_arrays_eq_impl(left: &ArrayRef, right: &ArrayRef) { let executed = execute_to_canonical(left.clone(), &mut LEGACY_SESSION.create_execution_ctx()); diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-array/src/arrays/chunked/paired_chunks.rs index f786d62b9fb..d81ee13c3b8 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-array/src/arrays/chunked/paired_chunks.rs @@ -131,7 +131,7 @@ mod tests { DType::Primitive(PType::I32, Nullability::NonNullable) } - #[allow(clippy::type_complexity)] + #[expect(clippy::type_complexity)] fn collect_pairs( left: &ChunkedArray, right: &ChunkedArray, diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index 7bddbb8afb2..ae6ccad60b3 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -529,7 +529,7 @@ impl Array { } } - #[allow( + #[expect( clippy::cognitive_complexity, reason = "patching depends on both patch and value physical types" )] diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index b7d0cc6c0f4..b5f007a05d3 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -138,10 +138,6 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { Ok(()) } - #[allow( - clippy::cognitive_complexity, - reason = "branching depends on validity representation and code type" - )] fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { let codes_validity = self.codes().validity_mask()?; let codes_primitive = self.codes().to_primitive(); @@ -252,7 +248,7 @@ impl Array { #[cfg(test)] mod test { - #[allow(unused_imports)] + #[expect(unused_imports)] use itertools::Itertools; use rand::RngExt; use rand::SeedableRng; diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index 575dab50f50..84facbd34bf 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -54,7 +54,7 @@ impl FilterReduce for Dict { #[cfg(test)] mod test { - #[allow(unused_imports)] + #[expect(unused_imports)] use itertools::Itertools; use vortex_buffer::buffer; diff --git a/vortex-array/src/arrays/dict_test.rs b/vortex-array/src/arrays/dict_test.rs index 8402b66c670..49be0744cf0 100644 --- a/vortex-array/src/arrays/dict_test.rs +++ b/vortex-array/src/arrays/dict_test.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use rand::RngExt; use rand::SeedableRng; diff --git a/vortex-array/src/arrays/filter/execute/bitbuffer.rs b/vortex-array/src/arrays/filter/execute/bitbuffer.rs index e1c61b2ebc1..1ad07801967 100644 --- a/vortex-array/src/arrays/filter/execute/bitbuffer.rs +++ b/vortex-array/src/arrays/filter/execute/bitbuffer.rs @@ -32,7 +32,7 @@ fn filter_bitbuffer_by_indices(bb: &BitBuffer, indices: &[usize]) -> BitBuffer { .freeze() } -#[allow(unused)] +#[expect(unused)] fn filter_bitbuffer_by_slices(bb: &BitBuffer, slices: &[(usize, usize)]) -> BitBuffer { let bools = bb.inner().as_ref(); let bit_offset = bb.offset(); diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index e6c30416e17..1c0cd000c10 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -21,7 +21,7 @@ use crate::arrays::listview::ListViewRebuildMode; /// However, we also do not want to carry around a large amount of garbage data. Below this /// threshold of the density of the selection mask, we will rebuild the [`ListViewArray`], removing /// any garbage data. -#[allow(unused)] +#[expect(unused)] const REBUILD_DENSITY_THRESHOLD: f64 = 0.1; /// [`ListViewArray`] filter implementation. diff --git a/vortex-array/src/arrays/filter/execute/primitive.rs b/vortex-array/src/arrays/filter/execute/primitive.rs index 86ef8ecbbb2..4a1b6367428 100644 --- a/vortex-array/src/arrays/filter/execute/primitive.rs +++ b/vortex-array/src/arrays/filter/execute/primitive.rs @@ -27,7 +27,7 @@ pub fn filter_primitive(array: &PrimitiveArray, mask: &Arc) -> Primi } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] mod test { use itertools::Itertools; use rstest::rstest; diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index e2e7407d111..3d70e17fbdb 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -443,7 +443,7 @@ fn test_offset_to_0() { type OptVec = Vec>; // Helper function to create a list of lists from a 3D vector with Option types. -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn create_list_of_lists_nullable(data: OptVec>>) -> ListArray { // Flatten all elements and track offsets and validity. let mut all_elements = Vec::new(); diff --git a/vortex-array/src/arrays/listview/compute/take.rs b/vortex-array/src/arrays/listview/compute/take.rs index 5cfadd40214..69f40a400a4 100644 --- a/vortex-array/src/arrays/listview/compute/take.rs +++ b/vortex-array/src/arrays/listview/compute/take.rs @@ -26,7 +26,7 @@ use crate::scalar::Scalar; /// However, we also do not want to carry around a large amount of garbage data. Below this /// threshold of the density of the selection mask, we will rebuild the [`ListViewArray`], removing /// any garbage data. -#[allow(unused)] +#[expect(unused)] const REBUILD_DENSITY_THRESHOLD: f64 = 0.1; /// [`ListViewArray`] take implementation. diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 4bcfb8311f8..12c2c653162 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -368,7 +368,6 @@ impl ListViewArray { } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] mod tests { use vortex_buffer::BitBuffer; use vortex_error::VortexResult; diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 35d533d4f23..760383bcce0 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -94,7 +94,7 @@ fn test_masked_child_preserves_length(#[case] validity: Validity) { _ => 3, }; - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let child = PrimitiveArray::from_iter(0..len as i32).into_array(); let array = MaskedArray::try_new(child, validity.clone()).unwrap(); diff --git a/vortex-array/src/arrays/null/compute/take.rs b/vortex-array/src/arrays/null/compute/take.rs index f24b89aea45..76ae248eded 100644 --- a/vortex-array/src/arrays/null/compute/take.rs +++ b/vortex-array/src/arrays/null/compute/take.rs @@ -16,7 +16,7 @@ use crate::match_each_integer_ptype; use crate::optimizer::rules::ParentRuleSet; impl TakeReduce for Null { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn take(array: ArrayView<'_, Null>, indices: &ArrayRef) -> VortexResult> { let indices = indices.to_primitive(); diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 869671d396a..b2be4205b7f 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -247,7 +247,6 @@ impl Patched { } /// Transpose a set of patches from the default sorted layout into the data parallel layout. -#[allow(clippy::cognitive_complexity)] fn transpose_patches(patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult { let array_len = patches.array_len(); let offset = patches.offset(); @@ -285,7 +284,7 @@ fn transpose_patches(patches: &Patches, ctx: &mut ExecutionCtx) -> VortexResult< }) } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn transpose( indices_in: &[I], values_in: &[V], diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 49aa726c51c..ef3ba7fbbfe 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -87,7 +87,7 @@ impl TakeExecute for Patched { /// /// First, builds a hashmap from index to patch value, then uses the hashmap in a loop to collect /// the values. -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] fn take_map( output: &mut [V], indices: &[I], diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 4d430277bc8..c49e681a411 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -318,7 +318,6 @@ impl VTable for Patched { } /// Apply patches on top of the existing value types. -#[allow(clippy::too_many_arguments)] fn apply_patches_primitive( output: &mut [V], offset: usize, diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index 9053afaf66f..dc98495ef11 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -149,7 +149,6 @@ mod test { use crate::dtype::PType; use crate::validity::Validity; - #[allow(clippy::cognitive_complexity)] #[test] fn cast_u32_u8() { let arr = buffer![0u32, 10, 200].into_array(); diff --git a/vortex-array/src/arrays/primitive/compute/take/mod.rs b/vortex-array/src/arrays/primitive/compute/take/mod.rs index 230595039fa..4a0c87ac800 100644 --- a/vortex-array/src/arrays/primitive/compute/take/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/take/mod.rs @@ -60,7 +60,6 @@ trait TakeImpl: Send + Sync { ) -> VortexResult; } -#[allow(unused)] struct TakeKernelScalar; impl TakeImpl for TakeKernelScalar { @@ -113,7 +112,6 @@ impl TakeExecute for Primitive { } // Compiler may see this as unused based on enabled features -#[allow(unused)] #[inline(always)] fn take_primitive_scalar( buffer: &[T], diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 301713761b5..6e568f7aa08 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -43,7 +43,6 @@ impl ScalarFnData { } /// Get the scalar function bound to this array. - #[allow(clippy::same_name_method)] #[inline(always)] pub fn scalar_fn(&self) -> &ScalarFnRef { &self.scalar_fn @@ -65,12 +64,10 @@ pub trait ScalarFnArrayExt: TypedArrayRef { self.as_ref().slots().len() } - #[allow(clippy::same_name_method)] fn nchildren(&self) -> usize { self.child_count() } - #[allow(clippy::same_name_method)] fn get_child(&self, idx: usize) -> &ArrayRef { self.child_at(idx) } diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index 37d704db4d1..5e90aece98e 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -41,7 +41,7 @@ impl Display for SharedData { } } -#[allow(async_fn_in_trait)] +#[expect(async_fn_in_trait)] pub trait SharedArrayExt: TypedArrayRef { fn source(&self) -> &ArrayRef { self.as_ref().slots()[SOURCE_SLOT] diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 7fbbcb27b2c..fb35ee503bc 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -255,7 +255,7 @@ impl VarBinData { let string_bytes = &bytes.as_ref()[start..end]; simdutf8::basic::from_utf8(string_bytes).map_err(|_| { - #[allow(clippy::unwrap_used)] + #[expect(clippy::unwrap_used)] // run validation using `compat` package to get more detailed error message let err = simdutf8::compat::from_utf8(string_bytes).unwrap_err(); vortex_err!("invalid utf-8: {err} at index {i}") diff --git a/vortex-array/src/buffer.rs b/vortex-array/src/buffer.rs index 07baf843ead..29f9d3582be 100644 --- a/vortex-array/src/buffer.rs +++ b/vortex-array/src/buffer.rs @@ -222,7 +222,7 @@ impl BufferHandle { self.slice(start..end) } - #[allow(clippy::panic)] + #[expect(clippy::panic)] /// Unwraps the handle as host memory. /// /// # Panics @@ -235,7 +235,7 @@ impl BufferHandle { } } - #[allow(clippy::panic)] + #[expect(clippy::panic)] /// Unwraps the handle as device memory. /// /// # Panics diff --git a/vortex-array/src/builders/dict/primitive.rs b/vortex-array/src/builders/dict/primitive.rs index 9f41e462734..d4ae633c202 100644 --- a/vortex-array/src/builders/dict/primitive.rs +++ b/vortex-array/src/builders/dict/primitive.rs @@ -152,7 +152,7 @@ where #[cfg(test)] mod test { - #[allow(unused_imports)] + #[expect(unused_imports)] use itertools::Itertools; use vortex_buffer::buffer; diff --git a/vortex-array/src/builders/tests.rs b/vortex-array/src/builders/tests.rs index 1a738954058..b93a4ad2c30 100644 --- a/vortex-array/src/builders/tests.rs +++ b/vortex-array/src/builders/tests.rs @@ -548,7 +548,7 @@ fn test_append_scalar_comprehensive(#[case] dtype: DType) { } /// Helper function to create test scalars for a given dtype. -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn create_test_scalars_for_dtype(dtype: &DType, count: usize) -> Vec { let mut scalars = Vec::with_capacity(count); diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 294ac5cd551..e3adcb8f630 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -391,7 +391,7 @@ impl Default for CompletedBuffers { } // Self::push enforces len < u32::max -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] impl CompletedBuffers { fn len(&self) -> u32 { match self { @@ -477,7 +477,7 @@ pub struct DeduplicatedBuffers { impl DeduplicatedBuffers { // Self::push enforces len < u32::max - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn len(&self) -> u32 { self.buffers.len() as u32 } diff --git a/vortex-array/src/compute/conformance/search_sorted.rs b/vortex-array/src/compute/conformance/search_sorted.rs index a112b7fe4d1..472dfec9dc1 100644 --- a/vortex-array/src/compute/conformance/search_sorted.rs +++ b/vortex-array/src/compute/conformance/search_sorted.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] pub use rstest::rstest; pub use rstest_reuse; diff --git a/vortex-array/src/compute/conformance/take.rs b/vortex-array/src/compute/conformance/take.rs index 214262e5572..9fec410ac35 100644 --- a/vortex-array/src/compute/conformance/take.rs +++ b/vortex-array/src/compute/conformance/take.rs @@ -102,7 +102,7 @@ fn test_take_none(array: &ArrayRef) { assert_eq!(result.dtype(), array.dtype()); } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn test_take_selective(array: &ArrayRef) { let len = array.len(); @@ -155,7 +155,7 @@ fn test_take_first_and_last(array: &ArrayRef) { ); } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn test_take_with_nullable_indices(array: &ArrayRef) { let len = array.len(); @@ -281,7 +281,7 @@ fn test_take_single_middle(array: &ArrayRef) { ); } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn test_take_random_unsorted(array: &ArrayRef) { let len = array.len(); @@ -339,7 +339,7 @@ fn test_take_contiguous_range(array: &ArrayRef) { } } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn test_take_mixed_repeated(array: &ArrayRef) { let len = array.len(); @@ -375,7 +375,7 @@ fn test_take_mixed_repeated(array: &ArrayRef) { } } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn test_take_large_indices(array: &ArrayRef) { // Test with a large number of indices to stress test performance let len = array.len(); diff --git a/vortex-array/src/dtype/bigint/mod.rs b/vortex-array/src/dtype/bigint/mod.rs index 6846d202529..3ebf01425d6 100644 --- a/vortex-array/src/dtype/bigint/mod.rs +++ b/vortex-array/src/dtype/bigint/mod.rs @@ -330,7 +330,7 @@ define_as_primitive!(i64); define_as_primitive!(i128); #[cfg(test)] -#[allow(clippy::many_single_char_names)] +#[expect(clippy::many_single_char_names)] mod tests { use num_traits::ToPrimitive; diff --git a/vortex-array/src/dtype/extension/typed.rs b/vortex-array/src/dtype/extension/typed.rs index 747b813afad..5b176bbb34d 100644 --- a/vortex-array/src/dtype/extension/typed.rs +++ b/vortex-array/src/dtype/extension/typed.rs @@ -48,7 +48,7 @@ impl ExtDType { } } -#[allow(clippy::same_name_method)] +#[expect(clippy::same_name_method)] impl ExtDType { /// Creates a new extension dtype with the given metadata and storage dtype. pub fn try_with_vtable( diff --git a/vortex-array/src/dtype/ptype.rs b/vortex-array/src/dtype/ptype.rs index 842f2d585a1..ab443618d66 100644 --- a/vortex-array/src/dtype/ptype.rs +++ b/vortex-array/src/dtype/ptype.rs @@ -891,7 +891,6 @@ macro_rules! try_from_bytes { ($T:ty) => { impl ToBytes for $T { #[inline] - #[allow(clippy::size_of_in_element_count)] fn to_le_bytes(&self) -> &[u8] { // NOTE(ngates): this assumes the platform is little-endian. Currently enforced // with a flag cfg(target_endian = "little") diff --git a/vortex-array/src/dtype/serde/mod.rs b/vortex-array/src/dtype/serde/mod.rs index 1dbeeb6cf8b..d0f43f0c4ac 100644 --- a/vortex-array/src/dtype/serde/mod.rs +++ b/vortex-array/src/dtype/serde/mod.rs @@ -7,7 +7,7 @@ pub(crate) mod flatbuffers; mod proto; -#[allow(clippy::module_inception)] +#[expect(clippy::module_inception)] #[cfg(feature = "serde")] mod serde; diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index be47f178467..6233295958b 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -62,7 +62,7 @@ pub trait Executable: Sized { fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; } -#[allow(clippy::same_name_method)] +#[expect(clippy::same_name_method)] impl ArrayRef { /// Execute this array to produce an instance of `E`. /// diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index daf121d3a56..d25d6446ce6 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -887,7 +887,7 @@ impl Patches { } } -#[allow(clippy::too_many_arguments)] // private function, can clean up one day +#[expect(clippy::too_many_arguments)] // private function, can clean up one day fn take_map, T: NativePType>( indices: &[I], take_indices: &[T], diff --git a/vortex-array/src/scalar/truncation.rs b/vortex-array/src/scalar/truncation.rs index cc1d7a231a7..3a2b562d7aa 100644 --- a/vortex-array/src/scalar/truncation.rs +++ b/vortex-array/src/scalar/truncation.rs @@ -14,7 +14,7 @@ use crate::scalar::Scalar; use crate::scalar::StringLike; /// A trait for truncating [`Scalar`]s to a given length in bytes. -#[allow(clippy::len_without_is_empty)] +#[expect(clippy::len_without_is_empty)] pub trait ScalarTruncation: Send + Sized { /// Unwrap a Scalar into a ScalarTruncation object /// diff --git a/vortex-array/src/scalar/typed_view/decimal/tests.rs b/vortex-array/src/scalar/typed_view/decimal/tests.rs index fa6ccfad96b..ca5b06778d9 100644 --- a/vortex-array/src/scalar/typed_view/decimal/tests.rs +++ b/vortex-array/src/scalar/typed_view/decimal/tests.rs @@ -3,8 +3,6 @@ //! Tests for decimal scalar casting functionality. -#![allow(clippy::disallowed_types, clippy::panic)] - use rstest::rstest; use vortex_utils::aliases::hash_set::HashSet; diff --git a/vortex-array/src/scalar/typed_view/utf8.rs b/vortex-array/src/scalar/typed_view/utf8.rs index 7e6dcba8635..b07201a483d 100644 --- a/vortex-array/src/scalar/typed_view/utf8.rs +++ b/vortex-array/src/scalar/typed_view/utf8.rs @@ -159,7 +159,7 @@ mod private { impl Sealed for BufferString {} impl StringLike for BufferString { - #[allow(clippy::unwrap_in_result, clippy::expect_used)] + #[expect(clippy::unwrap_in_result, clippy::expect_used)] fn increment(self) -> Result { if self.is_empty() { return Err(self); diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric.rs b/vortex-array/src/scalar_fn/fns/binary/numeric.rs index 360c65ef6a3..d48b3e230ff 100644 --- a/vortex-array/src/scalar_fn/fns/binary/numeric.rs +++ b/vortex-array/src/scalar_fn/fns/binary/numeric.rs @@ -70,7 +70,6 @@ fn constant_numeric( } #[cfg(test)] -#[allow(deprecated)] mod test { use vortex_buffer::buffer; use vortex_error::VortexResult; diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index 099b51bd289..6486a53735f 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -14,7 +14,7 @@ use crate::arrays::bool::BoolArrayExt; /// Check that a named metadata matches its previous versioning. /// /// Goldenfile takes care of checking for equality against a checked-in file. -#[allow(clippy::unwrap_used)] +#[expect(clippy::unwrap_used)] pub fn check_metadata(name: &str, metadata: &[u8]) { let mut mint = Mint::new("goldenfiles/"); let mut f = mint diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index a1b7c46bb16..af0d3fdef30 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::expect_used)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::expect_used)] use std::clone::Clone; use std::fmt::Display; diff --git a/vortex-bench/src/polarsignals/data.rs b/vortex-bench/src/polarsignals/data.rs index eb2db956ee4..f95049f59b4 100644 --- a/vortex-bench/src/polarsignals/data.rs +++ b/vortex-bench/src/polarsignals/data.rs @@ -85,7 +85,7 @@ fn generate_sorted_label_sets() -> LabelSets { .iter() .map(|&idx| { let (_, fill, distinct) = LABELS[idx]; - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let null_count = ((1.0 - fill) * NUM_LABEL_SETS as f64).round() as usize; if s < null_count || distinct == 0 { None @@ -191,7 +191,7 @@ pub fn generate_polarsignals_parquet(n_rows: usize, output_path: &Path) -> Resul Ok(()) } -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] fn build_batch( schema: &Arc, n: usize, @@ -414,7 +414,7 @@ fn build_locations( } #[cfg(test)] -#[allow(clippy::disallowed_types)] +#[expect(clippy::disallowed_types)] mod tests { use std::collections::HashSet; diff --git a/vortex-bench/src/statpopgen/builder.rs b/vortex-bench/src/statpopgen/builder.rs index 7c46d1ff2b8..4e11b4e43e4 100644 --- a/vortex-bench/src/statpopgen/builder.rs +++ b/vortex-bench/src/statpopgen/builder.rs @@ -28,8 +28,7 @@ use vortex::utils::aliases::hash_set::HashSet; use super::vcf_conversion::*; -#[allow(dead_code)] -#[allow(non_snake_case)] +#[expect(non_snake_case)] pub struct GnomADBuilder<'a> { /// The schema of the to-be-generated Parquet file. schema: SchemaRef, @@ -240,7 +239,6 @@ impl InfoArrayBuilder { } impl<'a> GnomADBuilder<'a> { - #[allow(non_snake_case)] pub fn new(header: &'a Header, schema: SchemaRef) -> Self { let info_builder: HashMap<&'a str, InfoArrayBuilder> = header .infos() diff --git a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs index dc5fa6448ad..0fa7f488b25 100644 --- a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs +++ b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs @@ -129,7 +129,7 @@ impl Benchmark for StatPopGenBenchmark { Ok(()) } - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn expected_row_counts(&self) -> Option> { let n_rows = self.n_rows as usize; match self.scale_factor { diff --git a/vortex-bench/src/tpch/tpchgen.rs b/vortex-bench/src/tpch/tpchgen.rs index 1d7273f3ccc..5dbeb380a7e 100644 --- a/vortex-bench/src/tpch/tpchgen.rs +++ b/vortex-bench/src/tpch/tpchgen.rs @@ -269,7 +269,7 @@ fn generate_table_files( } /// Calculate the number of partitions without creating expensive iterators -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn calculate_num_parts(generator: TableGenerator, options: &TpchGenOptions) -> Result { let scale_factor = options.scale_factor.parse::()?; @@ -277,7 +277,7 @@ fn calculate_num_parts(generator: TableGenerator, options: &TpchGenOptions) -> R .uncompressed_data_size() .zip(options.max_file_size_mb) { - #[allow(clippy::cast_precision_loss)] + #[expect(clippy::cast_precision_loss)] let file_size = (data_size as f64 * scale_factor).ceil() as u64; file_size.div_ceil(max_file_size) } else { @@ -288,7 +288,7 @@ fn calculate_num_parts(generator: TableGenerator, options: &TpchGenOptions) -> R } /// Create a single batch iterator for a specific partition -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn create_single_batch_iterator( generator: TableGenerator, options: &TpchGenOptions, diff --git a/vortex-btrblocks/benches/compress.rs b/vortex-btrblocks/benches/compress.rs index 18a48881678..8b6832ea630 100644 --- a/vortex-btrblocks/benches/compress.rs +++ b/vortex-btrblocks/benches/compress.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(unexpected_cfgs)] +#![expect(clippy::unwrap_used)] #[cfg(not(codspeed))] mod benchmarks { diff --git a/vortex-btrblocks/benches/compress_listview.rs b/vortex-btrblocks/benches/compress_listview.rs index 11f5f92437e..c3d0f8f6aa8 100644 --- a/vortex-btrblocks/benches/compress_listview.rs +++ b/vortex-btrblocks/benches/compress_listview.rs @@ -1,9 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] -#![allow(unexpected_cfgs)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] #[cfg(not(codspeed))] mod benchmarks { diff --git a/vortex-buffer/benches/vortex_bitbuffer.rs b/vortex-buffer/benches/vortex_bitbuffer.rs index 807f890b706..67ce88da889 100644 --- a/vortex-buffer/benches/vortex_bitbuffer.rs +++ b/vortex-buffer/benches/vortex_bitbuffer.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] - use std::iter::Iterator; use arrow_buffer::BooleanBuffer; diff --git a/vortex-buffer/benches/vortex_buffer.rs b/vortex-buffer/benches/vortex_buffer.rs index 1fc28023c75..5c97cbb3915 100644 --- a/vortex-buffer/benches/vortex_buffer.rs +++ b/vortex-buffer/benches/vortex_buffer.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] - use std::iter::Iterator; use arrow_buffer::ArrowNativeType; diff --git a/vortex-buffer/src/lib.rs b/vortex-buffer/src/lib.rs index 8f6d0cf24c3..8319fffa387 100644 --- a/vortex-buffer/src/lib.rs +++ b/vortex-buffer/src/lib.rs @@ -2,8 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![deny(missing_docs)] -// cudarc HostSlice has len and is_empty methods that duplicate BufferMut methods. -#![allow(clippy::same_name_method)] //! A library for working with custom aligned buffers of sized values. //! diff --git a/vortex-compressor/benches/dict_encode.rs b/vortex-compressor/benches/dict_encode.rs index a1a22636e18..fee2e42e1f8 100644 --- a/vortex-compressor/benches/dict_encode.rs +++ b/vortex-compressor/benches/dict_encode.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] use divan::Bencher; use vortex_array::IntoArray; diff --git a/vortex-compressor/benches/stats_calc.rs b/vortex-compressor/benches/stats_calc.rs index 5675c8de434..e435350851c 100644 --- a/vortex-compressor/benches/stats_calc.rs +++ b/vortex-compressor/benches/stats_calc.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation, clippy::use_debug)] -#![allow(unexpected_cfgs)] - #[cfg(not(codspeed))] #[divan::bench_group(items_count = 64_000u32, bytes_count = 256_000u32)] mod benchmarks { diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index eca60891568..6a188108612 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -210,7 +210,7 @@ macro_rules! impl_encode { ($typ:ty, $utyp:ty, $($ityp:ty),+) => { $( impl Encode<$typ, $ityp> for DictEncoder { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn encode(distinct: &[$typ], values: &[$typ]) -> Buffer<$ityp> { let mut codes = vortex_utils::aliases::hash_map::HashMap::<$utyp, $ityp>::with_capacity( diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 1fcf9c62903..d956a90990a 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -210,7 +210,7 @@ macro_rules! impl_encode { ($typ:ty, $($ityp:ty),+) => { $( impl Encode<$typ, $ityp> for DictEncoder { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn encode(distinct: &[$typ], values: &[$typ]) -> Buffer<$ityp> { let mut codes = vortex_utils::aliases::hash_map::HashMap::<$typ, $ityp>::with_capacity( diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index a0cc4985ea5..8337f5062b0 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for bit unpacking. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] mod common; diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index 53f303d0b2c..638daa77538 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for DateTimeParts decoding. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] mod common; diff --git a/vortex-cuda/benches/dict_cuda.rs b/vortex-cuda/benches/dict_cuda.rs index db650eacadd..33eff05584d 100644 --- a/vortex-cuda/benches/dict_cuda.rs +++ b/vortex-cuda/benches/dict_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for dictionary decoding. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] mod common; diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 3a6083e8bba..54ae6090f08 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] -#![allow(clippy::expect_used)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::expect_used)] use std::mem::size_of; use std::sync::Arc; diff --git a/vortex-cuda/benches/filter_cuda.rs b/vortex-cuda/benches/filter_cuda.rs index 3eb623ac357..a08fe6797ba 100644 --- a/vortex-cuda/benches/filter_cuda.rs +++ b/vortex-cuda/benches/filter_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for GPU filtering using CUB DeviceSelect::Flagged. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use std::ffi::c_void; use std::fmt::Debug; diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index cc5fab5e22c..df59481e6c0 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for FoR decompression. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] mod common; diff --git a/vortex-cuda/benches/runend_cuda.rs b/vortex-cuda/benches/runend_cuda.rs index c073f60634c..973a69ee914 100644 --- a/vortex-cuda/benches/runend_cuda.rs +++ b/vortex-cuda/benches/runend_cuda.rs @@ -3,8 +3,8 @@ //! CUDA benchmarks for run-end decoding. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] mod common; diff --git a/vortex-cuda/benches/throughput_cuda.rs b/vortex-cuda/benches/throughput_cuda.rs index b911263ec26..dc9eb82a4f4 100644 --- a/vortex-cuda/benches/throughput_cuda.rs +++ b/vortex-cuda/benches/throughput_cuda.rs @@ -6,8 +6,7 @@ //! //! Cases: 100% input only, 100% output only, and 50/50 mixed. -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] use std::time::Duration; diff --git a/vortex-cuda/benches/transpose_patches.rs b/vortex-cuda/benches/transpose_patches.rs index a4a4dfaa099..fee0edba43a 100644 --- a/vortex-cuda/benches/transpose_patches.rs +++ b/vortex-cuda/benches/transpose_patches.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::unwrap_used)] use std::time::Duration; diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index a5d285d2d2a..8862f18915b 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] - use std::time::Duration; use criterion::BenchmarkId; diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index fd035398b14..eb82c6a9563 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::expect_used)] -#![allow(clippy::use_debug)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::expect_used)] +#![expect(clippy::use_debug)] use std::env; use std::fs::File; diff --git a/vortex-cuda/cub/src/filter.rs b/vortex-cuda/cub/src/filter.rs index 5fdf5e474ed..fff38857706 100644 --- a/vortex-cuda/cub/src/filter.rs +++ b/vortex-cuda/cub/src/filter.rs @@ -29,7 +29,7 @@ pub trait CubFilterable: Copy + 'static { /// - `d_flags` must have at least `num_items` bytes (one per element, 0 or 1) /// - `d_out` must have enough space for selected elements /// - `d_num_selected` must point to a valid i64 on the device - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] unsafe fn filter_bytemask( d_temp: *mut c_void, temp_bytes: usize, @@ -55,7 +55,7 @@ pub trait CubFilterable: Copy + 'static { /// - `d_bitmask` must contain enough bytes to hold `bit_offset + num_items` bits /// - `d_out` must have enough space for selected elements /// - `d_num_selected` must point to a valid i64 on the device - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] unsafe fn filter_bitmask( d_temp: *mut c_void, temp_bytes: usize, @@ -135,7 +135,7 @@ macro_rules! impl_filter { /// - `d_flags` must have at least `num_items` bytes (one per element, 0 or 1) /// - `d_out` must have enough space for selected elements /// - `d_num_selected` must point to a valid i64 on the device - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] pub unsafe fn []( d_temp: *mut c_void, temp_bytes: usize, @@ -175,7 +175,7 @@ macro_rules! impl_filter { /// - `d_bitmask` must contain enough bytes to hold `bit_offset + num_items` bits /// - `d_out` must have enough space for selected elements /// - `d_num_selected` must point to a valid i64 on the device - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] pub unsafe fn []( d_temp: *mut c_void, temp_bytes: usize, diff --git a/vortex-cuda/cub/src/lib.rs b/vortex-cuda/cub/src/lib.rs index 706373255aa..c0532576604 100644 --- a/vortex-cuda/cub/src/lib.rs +++ b/vortex-cuda/cub/src/lib.rs @@ -18,13 +18,7 @@ use std::path::PathBuf; use std::sync::OnceLock; /// Raw FFI type definitions and dynamically-loaded function pointers from bindgen. -#[allow( - non_upper_case_globals, - non_camel_case_types, - non_snake_case, - dead_code, - clippy::all -)] +#[allow(non_camel_case_types, dead_code, clippy::all)] pub mod sys; mod error; diff --git a/vortex-cuda/nvcomp/src/lib.rs b/vortex-cuda/nvcomp/src/lib.rs index 2f9fd00deb9..3ab502baa63 100644 --- a/vortex-cuda/nvcomp/src/lib.rs +++ b/vortex-cuda/nvcomp/src/lib.rs @@ -22,11 +22,10 @@ use std::path::PathBuf; use std::sync::OnceLock; /// Raw FFI type definitions and dynamically-loaded function pointers from bindgen. -#[allow( +#[expect( non_upper_case_globals, non_camel_case_types, non_snake_case, - dead_code, clippy::all )] pub mod sys; diff --git a/vortex-cuda/nvcomp/src/zstd.rs b/vortex-cuda/nvcomp/src/zstd.rs index 7a326d3fb25..44111901071 100644 --- a/vortex-cuda/nvcomp/src/zstd.rs +++ b/vortex-cuda/nvcomp/src/zstd.rs @@ -134,7 +134,7 @@ pub fn get_decompress_temp_size_with_opts( /// - Each output buffer must have at least the corresponding `device_uncompressed_bytes` size /// - `device_temp_ptr` must have at least `temp_bytes` allocated /// - The stream must be valid -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] pub unsafe fn decompress_async( device_compressed_ptrs: *const *const c_void, device_compressed_bytes: *const usize, @@ -170,7 +170,7 @@ pub unsafe fn decompress_async( /// # Safety /// /// Same requirements as [`decompress_async`]. -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] pub unsafe fn decompress_async_with_opts( device_compressed_ptrs: *const *const c_void, device_compressed_bytes: *const usize, diff --git a/vortex-cuda/src/arrow/mod.rs b/vortex-cuda/src/arrow/mod.rs index 27533a986dd..89580290a6a 100644 --- a/vortex-cuda/src/arrow/mod.rs +++ b/vortex-cuda/src/arrow/mod.rs @@ -95,7 +95,6 @@ pub(crate) struct ArrowArray { } impl ArrowArray { - #[allow(unused)] pub fn empty() -> Self { Self { length: 0, diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 1d6d1db76c3..2118d370d85 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -350,7 +350,7 @@ pub trait CudaArrayExt { #[async_trait] impl CudaArrayExt for ArrayRef { - #[allow(clippy::unwrap_in_result, clippy::unwrap_used)] + #[expect(clippy::unwrap_used)] async fn execute_cuda(self, ctx: &mut CudaExecutionCtx) -> VortexResult { if self.encoding_id() == Struct::ID { let len = self.len(); diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index 421a759e374..53742dc11de 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -130,7 +130,7 @@ impl CudaExecute for DateTimePartsExecutor { } } -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] async fn decode_datetimeparts_typed( days: PrimitiveArray, seconds: PrimitiveArray, diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 206ac8bc1cd..9aed85580fd 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -4,7 +4,7 @@ pub mod types; #[rustfmt::skip] -#[allow(warnings, clippy::all, clippy::pedantic, clippy::nursery)] +#[expect(warnings, clippy::all, clippy::pedantic, clippy::nursery)] pub mod gpu { include!(concat!(env!("OUT_DIR"), "/patches.rs")); } diff --git a/vortex-cuda/src/kernel/patches/types.rs b/vortex-cuda/src/kernel/patches/types.rs index 78fa0b8429e..380fefb2d94 100644 --- a/vortex-cuda/src/kernel/patches/types.rs +++ b/vortex-cuda/src/kernel/patches/types.rs @@ -123,7 +123,7 @@ impl HostPatches { } /// Transpose a set of patches from the default sorted layout into the data parallel layout. -#[allow(clippy::cognitive_complexity)] +#[expect(clippy::cognitive_complexity)] pub async fn transpose_patches( patches: &Patches, ctx: &mut CudaExecutionCtx, @@ -161,7 +161,7 @@ pub async fn transpose_patches( }) } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn transpose( indices_in: &[I], values_in: &[V], @@ -305,7 +305,7 @@ mod tests { } #[test] - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] fn test_transpose_complex() -> VortexResult<()> { test_case(1024, 0, &[0], &[0f32])?; test_case(512, 512, &[512, 513, 514], &[10i8, 20, 30])?; diff --git a/vortex-cuda/src/pinned.rs b/vortex-cuda/src/pinned.rs index 7939bc588ec..80c57ee99b6 100644 --- a/vortex-cuda/src/pinned.rs +++ b/vortex-cuda/src/pinned.rs @@ -27,7 +27,7 @@ pub(crate) struct PinnedByteBuffer { logical_len: usize, } -#[allow(clippy::same_name_method)] +#[expect(clippy::same_name_method)] impl PinnedByteBuffer { /// Allocate a pinned host buffer with a given capacity and logical length. /// @@ -70,7 +70,6 @@ impl PinnedByteBuffer { } } -#[allow(clippy::same_name_method)] impl HostSlice for PinnedByteBuffer { fn len(&self) -> usize { self.len() @@ -277,7 +276,6 @@ pub struct PooledPinnedBuffer { pool: Arc, } -#[allow(clippy::same_name_method)] impl PooledPinnedBuffer { /// Create a new pooled buffer. pub(crate) fn new(inner: PinnedByteBuffer, pool: Arc) -> Self { diff --git a/vortex-cxx/src/lib.rs b/vortex-cxx/src/lib.rs index 1cc63071ebc..50d345be648 100644 --- a/vortex-cxx/src/lib.rs +++ b/vortex-cxx/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![allow(clippy::boxed_local)] + mod dtype; mod expr; mod read; diff --git a/vortex-datafusion/src/persistent/cache.rs b/vortex-datafusion/src/persistent/cache.rs index 037b4b1cd0a..de632a7597d 100644 --- a/vortex-datafusion/src/persistent/cache.rs +++ b/vortex-datafusion/src/persistent/cache.rs @@ -38,7 +38,7 @@ impl FileMetadata for CachedVortexMetadata { .unwrap_or(1024 * 64) } - #[allow(clippy::disallowed_types)] + #[expect(clippy::disallowed_types)] fn extra_info(&self) -> std::collections::HashMap { Default::default() } diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index ed216bab432..d93a849e75a 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -130,7 +130,7 @@ impl FileOpener for VortexOpener { let projection_pushdown = self.projection_pushdown; // Replace column access for partition columns with literals - #[allow(clippy::disallowed_types)] + #[expect(clippy::disallowed_types)] let literal_value_cols = self .table_schema .table_partition_cols() diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index b2530c445bb..ee7f6d88f26 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -212,7 +212,7 @@ pub struct VortexDataSource { /// The initial Vortex projection expression (e.g. column selection from the builder). initial_projection: Expression, /// Column statistics for the initial projection columns. - #[allow(dead_code)] + #[expect(dead_code)] initial_statistics: Vec, // --- Phase 2: Projected (pushed into the Vortex scan) --- diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs index a96bd1cbaca..3ef95f3224a 100644 --- a/vortex-duckdb/build.rs +++ b/vortex-duckdb/build.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] // exit(1) + cargo:error= doesn't provide a double-traceback like panic!() -#![allow(clippy::exit)] +#![expect(clippy::exit)] use std::env; use std::fs; diff --git a/vortex-duckdb/src/convert/expr.rs b/vortex-duckdb/src/convert/expr.rs index 80d92ede449..471b9b41302 100644 --- a/vortex-duckdb/src/convert/expr.rs +++ b/vortex-duckdb/src/convert/expr.rs @@ -43,7 +43,6 @@ fn like_pattern_str(value: &duckdb::ExpressionRef) -> VortexResult VortexResult> { diff --git a/vortex-duckdb/src/copy.rs b/vortex-duckdb/src/copy.rs index c1af6869d39..fa6b25b5ce7 100644 --- a/vortex-duckdb/src/copy.rs +++ b/vortex-duckdb/src/copy.rs @@ -54,7 +54,7 @@ pub struct GlobalState { // Pool of background workers helping to drive the write task. // Note that this is optional and without it, we would only drive the task when DuckDB calls // into us, and we call `RUNTIME.block_on`. - #[allow(dead_code)] + #[expect(dead_code)] worker_pool: CurrentThreadWorkerPool, } diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index fe6336d3856..47a1eba2b4f 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -484,7 +484,7 @@ fn extract_projection_expr( }; // duckdb index is u64 (size_t) but in Rust u64 and usize are different things. - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let names = projection_ids .iter() .filter(|p| **p != EMPTY_COLUMN_IDX) @@ -493,7 +493,7 @@ fn extract_projection_expr( idx = &column_ids[*idx as usize]; } - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] column_names .get(*idx as usize) .vortex_expect("prune idx in column names") diff --git a/vortex-duckdb/src/duckdb/logical_type.rs b/vortex-duckdb/src/duckdb/logical_type.rs index 851ea2824c3..06f17222a34 100644 --- a/vortex-duckdb/src/duckdb/logical_type.rs +++ b/vortex-duckdb/src/duckdb/logical_type.rs @@ -371,7 +371,6 @@ floating_type!(Float, f32); floating_type!(Double, f64); #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] mod tests { use super::*; @@ -401,7 +400,6 @@ mod tests { fn test_clone_decimal_logical_type() { let decimal_type = LogicalType::decimal_type(10, 2).vortex_expect("Failed to create decimal type"); - #[allow(clippy::redundant_clone)] let cloned = decimal_type.clone(); assert_eq!(decimal_type.as_type_id(), cloned.as_type_id()); @@ -419,7 +417,6 @@ mod tests { let list_type = LogicalType::list_type(int_type).vortex_expect("Failed to create list type"); - #[allow(clippy::redundant_clone)] let cloned = list_type.clone(); assert_eq!(list_type.as_type_id(), cloned.as_type_id()); @@ -440,7 +437,6 @@ mod tests { let array_type = LogicalType::array_type(LogicalType::new(DUCKDB_TYPE::DUCKDB_TYPE_VARCHAR), 5) .vortex_expect("Failed to create array type"); - #[allow(clippy::redundant_clone)] let cloned = array_type.clone(); assert_eq!(array_type.as_type_id(), cloned.as_type_id()); @@ -473,7 +469,6 @@ mod tests { )) }; - #[allow(clippy::redundant_clone)] let cloned = map_type.clone(); assert_eq!(map_type.as_type_id(), cloned.as_type_id()); @@ -504,7 +499,6 @@ mod tests { let struct_type = LogicalType::struct_type(member_types, member_names) .vortex_expect("Failed to create struct type"); - #[allow(clippy::redundant_clone)] let cloned = struct_type.clone(); assert_eq!(struct_type.as_type_id(), cloned.as_type_id()); @@ -548,7 +542,6 @@ mod tests { )) }; - #[allow(clippy::redundant_clone)] let cloned = union_type.clone(); assert_eq!(union_type.as_type_id(), cloned.as_type_id()); diff --git a/vortex-duckdb/src/duckdb/table_function/init.rs b/vortex-duckdb/src/duckdb/table_function/init.rs index edfd8e324b6..f58e558d0b2 100644 --- a/vortex-duckdb/src/duckdb/table_function/init.rs +++ b/vortex-duckdb/src/duckdb/table_function/init.rs @@ -39,7 +39,6 @@ pub(crate) unsafe extern "C-unwind" fn init_global_callback( } /// Native callback for the local initialization of a table function. -#[allow(deref_nullptr)] pub(crate) unsafe extern "C-unwind" fn init_local_callback( init_input: *const cpp::duckdb_vx_tfunc_init_input, global_init_data: *mut c_void, diff --git a/vortex-duckdb/src/lib.rs b/vortex-duckdb/src/lib.rs index 5e321fa2040..2e45685d8d1 100644 --- a/vortex-duckdb/src/lib.rs +++ b/vortex-duckdb/src/lib.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::missing_safety_doc)] +#![expect(clippy::missing_safety_doc)] use std::ffi::CStr; use std::ffi::c_char; diff --git a/vortex-duckdb/src/multi_file.rs b/vortex-duckdb/src/multi_file.rs index e0d04d6c2b9..84a843e09cf 100644 --- a/vortex-duckdb/src/multi_file.rs +++ b/vortex-duckdb/src/multi_file.rs @@ -92,7 +92,6 @@ impl DataSourceTableFunction for VortexMultiFileScan { #[cfg(test)] mod tests { - #[allow(clippy::wildcard_imports)] use rstest::rstest; use super::*; diff --git a/vortex-error/tests/fmt.rs b/vortex-error/tests/fmt.rs index 43a81508333..8aba428a3ff 100644 --- a/vortex-error/tests/fmt.rs +++ b/vortex-error/tests/fmt.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors // This seems to be a bug in the lint - https://github.com/rust-lang/rust-clippy/issues/11024 -#![allow(clippy::tests_outside_test_module)] +#![expect(clippy::tests_outside_test_module)] use serial_test::serial; use vortex_error::VortexError; diff --git a/vortex-ffi/build.rs b/vortex-ffi/build.rs index c9869522788..909ca578057 100644 --- a/vortex-ffi/build.rs +++ b/vortex-ffi/build.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::exit)] +#![expect(clippy::unwrap_used)] use std::env; use std::path::PathBuf; use std::process::Command; diff --git a/vortex-ffi/examples/hello_vortex.rs b/vortex-ffi/examples/hello_vortex.rs index a87d0338292..e33c8e1f2bb 100644 --- a/vortex-ffi/examples/hello_vortex.rs +++ b/vortex-ffi/examples/hello_vortex.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: CC-BY-4.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used, clippy::expect_used, clippy::use_debug)] +#![expect(clippy::unwrap_used, clippy::use_debug)] //! This example shows usage of the Vortex C FFI to read a Vortex file written by a Rust client. //! //! You can invoke this example from a checkout by running diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 776a91f3216..5f684622ee6 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(non_camel_case_types)] +#![expect(non_camel_case_types)] //! FFI interface for working with Vortex Arrays. use std::ffi::c_void; diff --git a/vortex-ffi/src/dtype.rs b/vortex-ffi/src/dtype.rs index f404187bb9f..57bc3560d8e 100644 --- a/vortex-ffi/src/dtype.rs +++ b/vortex-ffi/src/dtype.rs @@ -28,7 +28,7 @@ arc_wrapper!( ); /// The variant tag for a Vortex data type. -#[allow(non_camel_case_types)] +#[expect(non_camel_case_types)] #[non_exhaustive] #[repr(C)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -324,7 +324,7 @@ pub unsafe extern "C-unwind" fn vx_dtype_time_zone(dtype: *const DType) -> *cons } #[cfg(test)] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] mod tests { use std::slice; diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 1f7dcf67195..00a9df35675 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(non_camel_case_types)] +#![expect(non_camel_case_types)] use std::ffi::CStr; use std::ffi::c_char; diff --git a/vortex-ffi/src/file.rs b/vortex-ffi/src/file.rs index 61e49b1d624..4ba04eda836 100644 --- a/vortex-ffi/src/file.rs +++ b/vortex-ffi/src/file.rs @@ -59,7 +59,6 @@ arc_wrapper!( /// Options supplied for opening a file. #[repr(C)] -#[allow(non_camel_case_types)] // FIXME(ngates): we cannot have transparent structs in FFI since we cannot break them. pub struct vx_file_open_options { /// URI for opening the file. @@ -76,7 +75,6 @@ pub struct vx_file_open_options { /// Scan options provided by an FFI client calling the `vx_file_scan` function. #[repr(C)] -#[allow(non_camel_case_types)] // FIXME(ngates): we cannot have transparent structs in FFI since we cannot break them. pub struct vx_file_scan_options { /// Column names to project out in the scan. These must be null-terminated C strings. diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs index db387172150..1d11fc168dc 100644 --- a/vortex-ffi/src/lib.rs +++ b/vortex-ffi/src/lib.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::missing_safety_doc)] #![deny(missing_docs)] //! Native interface to Vortex arrays, types, files and streams. diff --git a/vortex-ffi/src/log.rs b/vortex-ffi/src/log.rs index cbd7bb5ae7c..6bde4419715 100644 --- a/vortex-ffi/src/log.rs +++ b/vortex-ffi/src/log.rs @@ -23,7 +23,7 @@ fn to_level_filter(level: vx_log_level) -> LevelFilter { /// Log levels for the Vortex library. #[repr(C)] -#[allow(non_camel_case_types)] +#[expect(non_camel_case_types)] pub enum vx_log_level { /// No logging will be performed. LOG_LEVEL_OFF = 0, diff --git a/vortex-ffi/src/macros.rs b/vortex-ffi/src/macros.rs index 9c4e6f2c342..4b191f1c7f6 100644 --- a/vortex-ffi/src/macros.rs +++ b/vortex-ffi/src/macros.rs @@ -176,10 +176,10 @@ macro_rules! box_dyn_wrapper { ($(#[$meta:meta])* $T:ty, $ffi_ident:ident) => { paste::paste! { $(#[$meta])* - #[allow(non_camel_case_types)] + #[expect(non_camel_case_types)] pub struct $ffi_ident(Box<$T>); - #[allow(dead_code)] + #[expect(dead_code)] impl $ffi_ident { /// Wrap an owned object into a raw pointer. pub(crate) fn new(obj: Box<$T>) -> *mut $ffi_ident { @@ -239,10 +239,10 @@ macro_rules! box_wrapper { ($(#[$meta:meta])* $T:ty, $ffi_ident:ident) => { paste::paste! { $(#[$meta])* - #[allow(non_camel_case_types)] + #[expect(non_camel_case_types)] pub struct $ffi_ident($T); - #[allow(dead_code)] + #[expect(dead_code)] impl $ffi_ident { /// Wrap an owned object into a raw pointer. pub(crate) fn new_box(obj: Box<$T>) -> *mut $ffi_ident { diff --git a/vortex-ffi/src/ptype.rs b/vortex-ffi/src/ptype.rs index 47af686582a..513571e737a 100644 --- a/vortex-ffi/src/ptype.rs +++ b/vortex-ffi/src/ptype.rs @@ -6,7 +6,7 @@ use vortex::dtype::PType; /// Variant enum for Vortex primitive types. #[non_exhaustive] #[repr(C)] -#[allow(non_camel_case_types)] +#[expect(non_camel_case_types)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum vx_ptype { /// Unsigned 8-bit integer diff --git a/vortex-ffi/src/sink.rs b/vortex-ffi/src/sink.rs index 7b0bbe69ca0..639d22196e3 100644 --- a/vortex-ffi/src/sink.rs +++ b/vortex-ffi/src/sink.rs @@ -27,7 +27,7 @@ use crate::error::try_or_default; use crate::error::vx_error; use crate::session::vx_session; -#[allow(non_camel_case_types)] +#[expect(non_camel_case_types)] /// The `sink` interface is used to collect array chunks and place them into a resource /// (e.g. an array stream or file (`vx_array_sink_open_file`)). /// diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 000df4a7bcb..3cc3b6d0858 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] #![doc(html_logo_url = "/vortex/docs/_static/vortex_spiral_logo.svg")] //! Read and write Vortex layouts, a serialization of Vortex arrays. //! diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 84840b2f5bf..41dbb5c169c 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] use std::iter; use std::sync::Arc; use std::sync::LazyLock; diff --git a/vortex-file/tests/test_write_table.rs b/vortex-file/tests/test_write_table.rs index d0522c5a78c..aa924ae44b9 100644 --- a/vortex-file/tests/test_write_table.rs +++ b/vortex-file/tests/test_write_table.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::tests_outside_test_module)] +#![expect(clippy::tests_outside_test_module)] use std::sync::Arc; use std::sync::LazyLock; diff --git a/vortex-io/src/limit.rs b/vortex-io/src/limit.rs index f49ea7a3661..cd6928f8aa9 100644 --- a/vortex-io/src/limit.rs +++ b/vortex-io/src/limit.rs @@ -277,7 +277,7 @@ mod tests { // Push many small items for i in 0..10 { - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] stream.push(async move { vec![i as u8; 5] }, 5).await; } diff --git a/vortex-io/src/runtime/current.rs b/vortex-io/src/runtime/current.rs index c49d9e010a5..c8ff91b4aec 100644 --- a/vortex-io/src/runtime/current.rs +++ b/vortex-io/src/runtime/current.rs @@ -152,7 +152,7 @@ impl Iterator for ThreadSafeIterator { } } -#[allow(clippy::if_then_some_else_none)] // Clippy is wrong when if/else has await. +#[expect(clippy::if_then_some_else_none)] // Clippy is wrong when if/else has await. #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/vortex-io/src/runtime/handle.rs b/vortex-io/src/runtime/handle.rs index f04b897b29e..dfbdc7fa77f 100644 --- a/vortex-io/src/runtime/handle.rs +++ b/vortex-io/src/runtime/handle.rs @@ -165,7 +165,7 @@ impl Task { impl Future for Task { type Output = T; - #[allow(clippy::panic)] + #[expect(clippy::panic)] fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { match ready!(self.recv.poll_unpin(cx)) { Ok(result) => Poll::Ready(result), diff --git a/vortex-io/src/runtime/tests.rs b/vortex-io/src/runtime/tests.rs index 8b342f4242d..e35d9db6f78 100644 --- a/vortex-io/src/runtime/tests.rs +++ b/vortex-io/src/runtime/tests.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #![cfg(feature = "tokio")] -#![allow(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_truncation)] use std::sync::Arc; use std::sync::atomic::AtomicUsize; diff --git a/vortex-mask/benches/intersect_by_rank.rs b/vortex-mask/benches/intersect_by_rank.rs index fd02d5ece7b..e7f495d6be7 100644 --- a/vortex-mask/benches/intersect_by_rank.rs +++ b/vortex-mask/benches/intersect_by_rank.rs @@ -3,8 +3,6 @@ //! Benchmarks for `intersect_by_rank`. -#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] - use divan::Bencher; use vortex_buffer::BitBuffer; use vortex_mask::Mask; @@ -39,6 +37,7 @@ const DENSITY_MATRIX_ARGS: &[(f64, f64, &str)] = &[ fn create_random_mask(len: usize, selectivity: f64) -> Mask { Mask::from_buffer(BitBuffer::from_iter((0..len).map(|i| { + #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let threshold = (selectivity * 1000.0) as usize; (i * 7 + 13) % 1000 < threshold }))) diff --git a/vortex-mask/src/bitops.rs b/vortex-mask/src/bitops.rs index 3ad681db161..94d7ef02980 100644 --- a/vortex-mask/src/bitops.rs +++ b/vortex-mask/src/bitops.rs @@ -82,7 +82,7 @@ impl Not for &Mask { } #[cfg(test)] -#[allow(clippy::many_single_char_names)] +#[expect(clippy::many_single_char_names)] mod tests { use vortex_buffer::BitBuffer; diff --git a/vortex-mask/src/tests.rs b/vortex-mask/src/tests.rs index a13827abbc3..ff0bf04f76a 100644 --- a/vortex-mask/src/tests.rs +++ b/vortex-mask/src/tests.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::panic)] -#![allow(clippy::many_single_char_names)] - use rstest::rstest; use vortex_buffer::BitBuffer; diff --git a/vortex-metrics/src/histogram.rs b/vortex-metrics/src/histogram.rs index 5172369fe08..d39e7816b11 100644 --- a/vortex-metrics/src/histogram.rs +++ b/vortex-metrics/src/histogram.rs @@ -31,8 +31,8 @@ impl Histogram { /// Returns the estimated quantile value, which must be in the [0.0, 1.0] range, will panic otherwise. /// Returns `None` if the histogram is empty. - #[allow(clippy::expect_used)] - #[allow(clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] + #[expect(clippy::unwrap_in_result)] pub fn quantile(&self, quantile: f64) -> Option { assert!( (0.0..=1.0).contains(&quantile), diff --git a/vortex-metrics/src/timer.rs b/vortex-metrics/src/timer.rs index 0bf5545ec77..20be2f7d69c 100644 --- a/vortex-metrics/src/timer.rs +++ b/vortex-metrics/src/timer.rs @@ -42,8 +42,8 @@ impl Timer { /// Returns the estimated quantile value, which must be in the [0.0, 1.0] range, will panic otherwise. /// Returns `None` if the timer is empty. - #[allow(clippy::expect_used)] - #[allow(clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] + #[expect(clippy::unwrap_in_result)] pub fn quantile(&self, quantile: f64) -> Option { assert!( (0.0..=1.0).contains(&quantile), diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index baabe693614..f0cc68e2f34 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -203,7 +203,7 @@ pub struct PyArray; impl PyArray { #[new] #[pyo3(signature = (*args, **kwargs))] - #[allow(unused_variables)] + #[expect(unused_variables)] fn new(args: &Bound<'_, PyAny>, kwargs: Option<&Bound<'_, PyAny>>) -> Self { Self } diff --git a/vortex-python/src/arrow.rs b/vortex-python/src/arrow.rs index 1ca4be5981f..fefd174c176 100644 --- a/vortex-python/src/arrow.rs +++ b/vortex-python/src/arrow.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileComment: Derived from upstream file arrow-pyarrow/src/main at commit 549709fb at https://github.com/apache/arrow-rs // SPDX-FileNotice: https://github.com/apache/arrow-rs/blob/549709fbdf91cd1f6c263a7e4540c542b6fecf6b/NOTICE.txt -#![allow(clippy::same_name_method)] +#![expect(clippy::same_name_method)] use std::convert::From; use std::convert::TryFrom; diff --git a/vortex-python/src/object_store/registry.rs b/vortex-python/src/object_store/registry.rs index 13a16e3c36c..c9544e0558e 100644 --- a/vortex-python/src/object_store/registry.rs +++ b/vortex-python/src/object_store/registry.rs @@ -7,7 +7,7 @@ //! //! See also -#![allow(clippy::disallowed_types)] +#![expect(clippy::disallowed_types)] use std::collections::HashMap; use std::sync::Arc; diff --git a/vortex-python/src/scalar/factory.rs b/vortex-python/src/scalar/factory.rs index ea07d3494a9..90e1c2031f7 100644 --- a/vortex-python/src/scalar/factory.rs +++ b/vortex-python/src/scalar/factory.rs @@ -26,7 +26,6 @@ use crate::error::PyVortexResult; use crate::scalar::PyScalar; use crate::scalar::bool; -#[allow(unused_variables)] #[pyfunction(name = "scalar")] #[pyo3(signature = (value, *, dtype=None))] pub fn scalar<'py>( diff --git a/vortex-sqllogictest/build.rs b/vortex-sqllogictest/build.rs index a62b3e39458..177aadb6cd5 100644 --- a/vortex-sqllogictest/build.rs +++ b/vortex-sqllogictest/build.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] +#![expect(clippy::unwrap_used)] fn main() { // Propagate DuckDB rpath from vortex-duckdb diff --git a/vortex-sqllogictest/src/utils.rs b/vortex-sqllogictest/src/utils.rs index 5d4e1bf2d93..f5ee0e6b12c 100644 --- a/vortex-sqllogictest/src/utils.rs +++ b/vortex-sqllogictest/src/utils.rs @@ -34,7 +34,7 @@ fn list_files_impl(file_paths: &mut Vec, path: impl AsRef) -> any Ok(()) } -#[allow(clippy::unwrap_used)] +#[expect(clippy::unwrap_used)] pub fn pb_style() -> ProgressStyle { ProgressStyle::with_template("[{elapsed_precise}] {wide_bar} {pos:>7}/{len:7} {msg}") .unwrap() diff --git a/vortex-test/compat-gen/src/fixtures/arrays/datasets/mod.rs b/vortex-test/compat-gen/src/fixtures/arrays/datasets/mod.rs index c799bc7a380..929e416fe14 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/datasets/mod.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/datasets/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod clickbench; -#[allow(clippy::cast_possible_truncation)] mod tpch; use crate::fixtures::DatasetFixture; diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/mod.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/mod.rs index 3c3ffcb80d6..dd707779aea 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/mod.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod arrays; -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] mod encodings; use crate::fixtures::FlatLayoutFixture; diff --git a/vortex-test/e2e-cuda/src/lib.rs b/vortex-test/e2e-cuda/src/lib.rs index 106a298157c..6ae3b2fccf7 100644 --- a/vortex-test/e2e-cuda/src/lib.rs +++ b/vortex-test/e2e-cuda/src/lib.rs @@ -11,7 +11,7 @@ //! * run some operations on the loaded column view //! * call `array->release()` to drop the data allocated from the Rust side -#![allow(clippy::unwrap_used, clippy::expect_used)] +#![expect(clippy::unwrap_used, clippy::expect_used)] use std::mem; use std::sync::Arc; diff --git a/vortex-tui/src/browse/mod.rs b/vortex-tui/src/browse/mod.rs index fb3d279bfa0..a5746e4c61b 100644 --- a/vortex-tui/src/browse/mod.rs +++ b/vortex-tui/src/browse/mod.rs @@ -56,7 +56,6 @@ fn navigate_layout_down(app: &mut AppState, amount: usize) { /// Handle a key event in normal input mode. /// /// Returns [`HandleResult::Exit`] if the user pressed the quit key. -#[allow(clippy::cognitive_complexity)] pub(crate) fn handle_normal_mode(app: &mut AppState, event: InputEvent) -> HandleResult { // Check if we're in Query tab with SQL input focus - handle text input first #[cfg(feature = "native")] diff --git a/vortex-tui/src/browse/ui/query.rs b/vortex-tui/src/browse/ui/query.rs index 3c626b47207..e93886a9155 100644 --- a/vortex-tui/src/browse/ui/query.rs +++ b/vortex-tui/src/browse/ui/query.rs @@ -481,7 +481,7 @@ async fn get_row_count( { use arrow_array::Int64Array; if let Some(arr) = batch.column(0).as_any().downcast_ref::() { - #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + #[expect(clippy::cast_sign_loss, clippy::cast_possible_truncation)] return Ok(arr.value(0) as usize); } } @@ -637,7 +637,7 @@ fn render_results_table(app: &mut AppState, area: Rect, buf: &mut Buffer) { let rows = get_all_rows(results, &app.query_state); // Calculate column widths - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] let widths: Vec = results .column_names .iter() diff --git a/vortex-tui/src/datafusion_helper.rs b/vortex-tui/src/datafusion_helper.rs index 9205ca0bede..d4ca3714009 100644 --- a/vortex-tui/src/datafusion_helper.rs +++ b/vortex-tui/src/datafusion_helper.rs @@ -76,7 +76,7 @@ pub async fn create_vortex_context( /// /// Panics if the array type doesn't match the expected Arrow array type during downcast. /// This should not happen for well-formed Arrow arrays. -#[allow(clippy::unwrap_used)] +#[expect(clippy::unwrap_used)] pub fn arrow_value_to_json(array: &dyn ArrowArray, idx: usize) -> serde_json::Value { use arrow_array::*; use arrow_schema::DataType; diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 381e54a1f55..54fe1b61926 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(unexpected_cfgs)] +#![expect(clippy::unwrap_used)] use std::fmt; use std::ops::Deref; @@ -122,7 +121,7 @@ mod setup { } /// Create Dict <- VarBinView encoding tree for strings with BitPacked codes - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub fn dict_varbinview_string() -> ArrayRef { let mut rng = StdRng::seed_from_u64(42); @@ -156,7 +155,7 @@ mod setup { } /// Create RunEnd <- FoR <- BitPacked encoding tree for u32 - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub fn runend_for_bp_u32() -> ArrayRef { let mut rng = StdRng::seed_from_u64(42); // Create data with runs of repeated values @@ -198,7 +197,7 @@ mod setup { } /// Create Dict <- FSST <- VarBin encoding tree for strings - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub fn dict_fsst_varbin_string() -> ArrayRef { let mut rng = StdRng::seed_from_u64(43); @@ -235,7 +234,7 @@ mod setup { /// Create Dict <- FSST <- VarBin <- BitPacked encoding tree for strings /// Compress the VarBin offsets inside FSST with BitPacked - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub fn dict_fsst_varbin_bp_string() -> ArrayRef { let mut rng = StdRng::seed_from_u64(45); diff --git a/vortex/benches/pipeline.rs b/vortex/benches/pipeline.rs index d6c668e0607..5fa3f646c50 100644 --- a/vortex/benches/pipeline.rs +++ b/vortex/benches/pipeline.rs @@ -108,11 +108,7 @@ //! [1] //! [2] -#![allow( - clippy::unwrap_used, - clippy::uninit_vec, - clippy::cast_possible_truncation -)] +#![expect(clippy::unwrap_used, clippy::uninit_vec)] use divan::Bencher; use fastlanes::BitPacking; diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 656d1a6137d..b3bcd8334bc 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -1,9 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#![allow(clippy::unwrap_used)] -#![allow(clippy::cast_possible_truncation)] -#![allow(unexpected_cfgs)] +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] use std::sync::LazyLock; @@ -85,7 +84,7 @@ fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) (uint_array, int_array, float_array) } -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn gen_varbin_words(len: usize, uniqueness: f64) -> Vec { let mut rng = StdRng::seed_from_u64(0); let uniq_cnt = (len as f64 * uniqueness) as usize; diff --git a/vortex/examples/compression_showcase.rs b/vortex/examples/compression_showcase.rs index 7560cc377c8..e7f80fff657 100644 --- a/vortex/examples/compression_showcase.rs +++ b/vortex/examples/compression_showcase.rs @@ -236,7 +236,7 @@ fn compress_structured_data() -> Result<(), Box> { } /// Estimate the size of an array in bytes (approximation) -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn estimate_size(array: &ArrayRef) -> usize { array.nbytes() as usize } diff --git a/vortex/examples/tracing_vortex.rs b/vortex/examples/tracing_vortex.rs index 8d7dd6b2701..7fd0267988e 100644 --- a/vortex/examples/tracing_vortex.rs +++ b/vortex/examples/tracing_vortex.rs @@ -8,7 +8,7 @@ //! //! Run with: cargo run --example tracing_vortex --features tokio -#![allow( +#![expect( clippy::disallowed_types, clippy::unwrap_used, clippy::cast_possible_truncation @@ -92,7 +92,7 @@ async fn main() -> Result<(), Box> { } /// Simulates application activity with various log levels and spans -#[allow(clippy::cognitive_complexity)] +#[expect(clippy::cognitive_complexity)] async fn simulate_application_activity(user_id: u32) { // Simulate HTTP request handling let request_span = span!( From 360b41d79da0a1ac373b33e6d81f0648ba0efca7 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Fri, 10 Apr 2026 17:36:56 +0100 Subject: [PATCH 009/250] Remove portable_simd take implementation and remove unused cargo flags (#7388) First of all we never use nightly so portable simd never gets enabled, second of all it looks like avx512 doesn't bring any benefits beyond avx2 for simd gather per gcc mailing list and Intel GDS mailing list. AVX2 on modern intel machines is comparable to Zen 5 machines and on Zen 5 machines AVX512, AVX2 and scalar versions behave the same. Signed-off-by: Robert Kruszewski --- .github/AGENTS.md | 1 - .github/scripts/close_fixed_fuzzer_issues.py | 2 - .github/workflows/ci.yml | 7 +- .github/workflows/fuzz-coverage.yml | 3 +- .github/workflows/fuzzer-fix-automation.yml | 4 +- .../minimize_fuzz_corpus_workflow.yml | 3 +- .github/workflows/run-fuzzer.yml | 4 +- Cargo.toml | 2 - .../src/arrays/primitive/compute/take/mod.rs | 25 +- .../arrays/primitive/compute/take/portable.rs | 318 ------------------ vortex-array/src/lib.rs | 2 - vortex-scan/README.md | 18 - 12 files changed, 18 insertions(+), 371 deletions(-) delete mode 100644 vortex-array/src/arrays/primitive/compute/take/portable.rs diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 5d43ab24300..dd85b76afdd 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -5,7 +5,6 @@ Nightly is required for: - `-Z` flags: sanitizers (`-Zsanitizer=address`), miri (`-Zmiri-*`), publish (`-Zpublish-timeout`) - `cargo-fuzz` (requires nightly) - `public-api` xtask (nightly rustdoc JSON) -- `--cfg vortex_nightly` (enables `portable_simd` feature gate) Everything else (build, clippy, tests, docs, benchmarks, packaging) should use stable. diff --git a/.github/scripts/close_fixed_fuzzer_issues.py b/.github/scripts/close_fixed_fuzzer_issues.py index ab637e725de..48513a3ceb2 100755 --- a/.github/scripts/close_fixed_fuzzer_issues.py +++ b/.github/scripts/close_fixed_fuzzer_issues.py @@ -162,7 +162,6 @@ def build_fuzz_target(target: str) -> bool: """Build the fuzz target once. Returns True on success.""" print(f"\nBuilding fuzz target: {target}") env = os.environ.copy() - env["RUSTFLAGS"] = "--cfg vortex_nightly" result = run( ["cargo", "+nightly", "fuzz", "build", "--dev", "--sanitizer=none", target], env=env, @@ -173,7 +172,6 @@ def build_fuzz_target(target: str) -> bool: def retest_crash(target: str, crash_path: str, timeout_secs: int = 120) -> str: """Run the fuzz target with the crash file. Returns 'fixed', 'reproduces', or 'timeout'.""" env = os.environ.copy() - env["RUSTFLAGS"] = "--cfg vortex_nightly" try: result = run( [ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b8fe2bfaf4..fb7b86d204c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -409,7 +409,7 @@ jobs: VORTEX_SKIP_SLOW_TESTS: "1" # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins # unset -Zsanitizer flag and we should allow that. - RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 -C opt-level=0 -C strip=none" + RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -492,8 +492,7 @@ jobs: run: | # TODO(myrrc): remove --no-default-features RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ - --cfg disable_loom --cfg vortex_nightly -C debuginfo=2 \ - -C opt-level=0 -C strip=none -Zexternal-clangrt \ + -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ ${{ matrix.sanitizer_flags }}" \ cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ --target x86_64-unknown-linux-gnu -Zbuild-std \ @@ -850,7 +849,7 @@ jobs: timeout-minutes: 40 env: MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE - RUSTFLAGS: "-A warnings --cfg vortex_nightly" + RUSTFLAGS: "-A warnings" RUST_BACKTRACE: full steps: - uses: runs-on/action@v2 diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index 8db58c1a5f6..ee4ed865b4b 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -75,8 +75,7 @@ jobs: - name: Generate coverage data run: | - RUSTFLAGS="--cfg vortex_nightly" \ - cargo +$NIGHTLY_TOOLCHAIN fuzz coverage --release --debug-assertions \ + cargo +$NIGHTLY_TOOLCHAIN fuzz coverage --release --debug-assertions \ ${{ matrix.fuzz_target }} \ -- -rss_limit_mb=4096 diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index 08618237c95..40691fec51a 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -163,7 +163,7 @@ jobs: echo "Building fuzzer target: ${{ steps.extract.outputs.target }} (debug mode for faster build)" # Build the fuzzer target in debug mode (faster than release) - if RUSTFLAGS="--cfg vortex_nightly" cargo +$NIGHTLY_TOOLCHAIN fuzz build --dev --sanitizer=none "${{ steps.extract.outputs.target }}" 2>&1 | tee fuzzer_build.log; then + if cargo +$NIGHTLY_TOOLCHAIN fuzz build --dev --sanitizer=none "${{ steps.extract.outputs.target }}" 2>&1 | tee fuzzer_build.log; then echo "✅ Fuzzer target built successfully" echo "build_success=true" >> $GITHUB_OUTPUT else @@ -183,7 +183,7 @@ jobs: echo "Attempting to reproduce crash with fuzzer (debug mode)..." # Run fuzzer with crash file (debug mode, no sanitizer, full backtrace) - RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=full timeout 30s cargo +$NIGHTLY_TOOLCHAIN fuzz run --dev --sanitizer=none "${{ steps.extract.outputs.target }}" "${{ steps.download.outputs.crash_file_path }}" -- -runs=1 -rss_limit_mb=0 2>&1 | tee crash_reproduction.log + RUST_BACKTRACE=full timeout 30s cargo +$NIGHTLY_TOOLCHAIN fuzz run --dev --sanitizer=none "${{ steps.extract.outputs.target }}" "${{ steps.download.outputs.crash_file_path }}" -- -runs=1 -rss_limit_mb=0 2>&1 | tee crash_reproduction.log FUZZ_EXIT_CODE=${PIPESTATUS[0]} diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index f4fcdbc39cb..784d1e059d1 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -91,8 +91,7 @@ jobs: CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" MINIMIZED_DIR="${CORPUS_DIR}_minimized" mkdir -p "$MINIMIZED_DIR" - RUSTFLAGS="--cfg vortex_nightly" \ - cargo +$NIGHTLY_TOOLCHAIN fuzz cmin $FEATURES_FLAG \ + cargo +$NIGHTLY_TOOLCHAIN fuzz cmin $FEATURES_FLAG \ ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- "$MINIMIZED_DIR" rm -rf "$CORPUS_DIR" mv "$MINIMIZED_DIR" "$CORPUS_DIR" diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index 251ee31a5b7..82f0dd0ace4 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -124,7 +124,7 @@ jobs: if [ "${{ inputs.jobs }}" -gt 1 ]; then FORK_FLAG="-fork=${{ inputs.jobs }}" fi - ${{ inputs.extra_env }} RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 \ + ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ $FEATURES_FLAG \ ${{ inputs.fuzz_target }} -- \ @@ -163,7 +163,7 @@ jobs: if [ -n "${{ inputs.extra_features }}" ]; then FEATURES_FLAG="--features ${{ inputs.extra_features }}" fi - RUSTFLAGS="--cfg vortex_nightly" RUST_BACKTRACE=1 \ + RUST_BACKTRACE=1 \ cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ $FEATURES_FLAG \ ${{ inputs.fuzz_target }} \ diff --git a/Cargo.toml b/Cargo.toml index cc0c7c45a0c..9853cf94ed9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -316,8 +316,6 @@ unused_lifetimes = "deny" unused_qualifications = "deny" unexpected_cfgs = { level = "deny", check-cfg = [ "cfg(codspeed)", - "cfg(disable_loom)", - "cfg(vortex_nightly)", 'cfg(target_os, values("unknown"))', ] } warnings = "warn" diff --git a/vortex-array/src/arrays/primitive/compute/take/mod.rs b/vortex-array/src/arrays/primitive/compute/take/mod.rs index 4a0c87ac800..aedda05b6b6 100644 --- a/vortex-array/src/arrays/primitive/compute/take/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/take/mod.rs @@ -4,9 +4,6 @@ #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] mod avx2; -#[cfg(vortex_nightly)] -mod portable; - use std::sync::LazyLock; use vortex_buffer::Buffer; @@ -32,23 +29,19 @@ use crate::validity::Validity; // Kernel selection happens on the first call to `take` and uses a combination of compile-time // and runtime feature detection to infer the best kernel for the platform. static PRIMITIVE_TAKE_KERNEL: LazyLock<&'static dyn TakeImpl> = LazyLock::new(|| { - cfg_if::cfg_if! { - if #[cfg(vortex_nightly)] { - // nightly codepath: use portable_simd kernel - &portable::TakeKernelPortableSimd - } else if #[cfg(target_arch = "x86_64")] { - // stable x86_64 path: use the optimized AVX2 kernel when available, falling - // back to scalar when not. - if is_x86_feature_detected!("avx2") { - &avx2::TakeKernelAVX2 - } else { - &TakeKernelScalar - } + #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] + { + if is_x86_feature_detected!("avx2") { + &avx2::TakeKernelAVX2 } else { - // stable all other platforms: scalar kernel &TakeKernelScalar } } + + #[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))] + { + &TakeKernelScalar + } }); trait TakeImpl: Send + Sync { diff --git a/vortex-array/src/arrays/primitive/compute/take/portable.rs b/vortex-array/src/arrays/primitive/compute/take/portable.rs deleted file mode 100644 index 2a9fec9cfb8..00000000000 --- a/vortex-array/src/arrays/primitive/compute/take/portable.rs +++ /dev/null @@ -1,318 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! An implementation of the Take kernel for primitive Arrays that uses -//! the nightly-only `portable_simd` feature. -//! -//! This is only enabled on non-x86_64 platforms and when using the nightly compiler for builds. - -#![allow(unused)] - -use std::mem::MaybeUninit; -use std::mem::size_of; -use std::mem::transmute; -use std::simd; -use std::simd::num::SimdUint; - -use multiversion::multiversion; -use num_traits::AsPrimitive; -use vortex_buffer::Alignment; -use vortex_buffer::Buffer; -use vortex_buffer::BufferMut; -use vortex_error::VortexResult; - -use crate::ArrayRef; -use crate::IntoArray; -use crate::array::ArrayView; -use crate::arrays::PrimitiveArray; -use crate::arrays::primitive::PrimitiveArrayExt; -use crate::arrays::primitive::compute::take::TakeImpl; -use crate::arrays::primitive::vtable::Primitive; -use crate::dtype::NativePType; -use crate::dtype::PType; -use crate::dtype::UnsignedPType; -use crate::dtype::half::f16; -use crate::match_each_native_simd_ptype; -use crate::match_each_unsigned_integer_ptype; -use crate::validity::Validity; - -pub(super) struct TakeKernelPortableSimd; - -impl TakeImpl for TakeKernelPortableSimd { - fn take( - &self, - array: ArrayView<'_, Primitive>, - unsigned_indices: ArrayView<'_, Primitive>, - validity: Validity, - ) -> VortexResult { - if array.ptype() == PType::F16 { - // Special handling for f16 to treat as opaque u16. - let decoded = match_each_unsigned_integer_ptype!(unsigned_indices.ptype(), |C| { - take_portable_simd::( - array.reinterpret_cast(PType::U16).as_slice(), - unsigned_indices.as_slice(), - ) - }); - Ok(PrimitiveArray::new(decoded, validity) - .reinterpret_cast(PType::F16) - .into_array()) - } else { - match_each_unsigned_integer_ptype!(unsigned_indices.ptype(), |C| { - match_each_native_simd_ptype!(array.ptype(), |V| { - let decoded = take_portable_simd::( - array.as_slice(), - unsigned_indices.as_slice(), - ); - Ok(PrimitiveArray::new(decoded, validity).into_array()) - }) - }) - } - } -} - -// --------------------------------------------------------------------------- -// Portable SIMD take algorithm -// --------------------------------------------------------------------------- - -/// SIMD types larger than the SIMD register size are beneficial for performance as this leads to -/// better instruction level parallelism. -const SIMD_WIDTH: usize = 64; - -/// Takes the specified indices into a new [`Buffer`] using portable SIMD. -/// -/// This function handles the type matching required to satisfy [`simd::SimdElement`] bounds. For -/// `f16` values, it reinterprets them as `u16` since `f16` doesn't implement `SimdElement`. -fn take_portable(buffer: &[T], indices: &[I]) -> Buffer { - if T::PTYPE == PType::F16 { - assert_eq!(size_of::(), size_of::()); - - // Since Rust does not actually support 16-bit floats, we first reinterpret the data as - // `u16` integers. - // SAFETY: We know that f16 has the same bit pattern as u16, so this transmute is fine. - let u16_slice: &[u16] = - unsafe { std::slice::from_raw_parts(buffer.as_ptr().cast(), buffer.len()) }; - return unsafe { take_with_indices(u16_slice, indices).transmute::() }; - } - - match_each_native_simd_ptype!(T::PTYPE, |TC| { - assert_eq!(size_of::(), size_of::()); - - // SAFETY: This is essentially a no-op that tricks the compiler into adding the - // `simd::SimdElement` bound we need to call `take_with_indices`. - let buffer: &[TC] = - unsafe { std::slice::from_raw_parts(buffer.as_ptr().cast::(), buffer.len()) }; - unsafe { take_with_indices(buffer, indices).transmute::() } - }) -} - -/// Helper that matches on index type and calls [`take_portable_simd`]. -/// -/// We separate this code out from above to add the [`simd::SimdElement`] constraint. -fn take_with_indices( - buffer: &[T], - indices: &[I], -) -> Buffer { - match_each_unsigned_integer_ptype!(I::PTYPE, |IC| { - let indices: &[IC] = - unsafe { std::slice::from_raw_parts(indices.as_ptr().cast::(), indices.len()) }; - take_portable_simd::(buffer, indices) - }) -} - -/// Takes elements from an array using SIMD indexing. -/// -/// Performs a gather operation that takes values at specified indices and returns them in a new -/// buffer. Uses SIMD instructions to process `LANE_COUNT` indices in parallel. -/// -/// Returns a [`Buffer`] where each element corresponds to `values[indices[i]]`. -#[multiversion(targets("x86_64+avx2", "x86_64+avx", "aarch64+neon"))] -fn take_portable_simd(values: &[T], indices: &[I]) -> Buffer -where - T: NativePType + simd::SimdElement, - I: UnsignedPType + simd::SimdElement, - simd::Simd: SimdUint = simd::Simd>, -{ - let indices_len = indices.len(); - - let mut buffer = BufferMut::::with_capacity_aligned( - indices_len, - Alignment::of::>(), - ); - - let buf_slice = buffer.spare_capacity_mut(); - - for chunk_idx in 0..(indices_len / LANE_COUNT) { - let offset = chunk_idx * LANE_COUNT; - let mask = simd::Mask::from_bitmask(u64::MAX); - let codes_chunk = simd::Simd::::from_slice(&indices[offset..]); - - let selection = simd::Simd::gather_select( - values, - mask, - codes_chunk.cast::(), - simd::Simd::::default(), - ); - - unsafe { - selection.store_select_unchecked( - transmute::<&mut [MaybeUninit], &mut [T]>(&mut buf_slice[offset..][..64]), - mask.cast(), - ); - } - } - - for idx in ((indices_len / LANE_COUNT) * LANE_COUNT)..indices_len { - unsafe { - buf_slice - .get_unchecked_mut(idx) - .write(values[indices[idx].as_()]); - } - } - - unsafe { - buffer.set_len(indices_len); - } - - // NOTE: if we don't do this, we pass back a Buffer which is over-aligned to the SIMD - // register width. The caller expects that this memory should be aligned to the value type - // so that we can slice it at value boundaries. - buffer = buffer.aligned(Alignment::of::()); - - buffer.freeze() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_take_out_of_bounds() { - let indices = vec![2_000_000u32; 64]; - let values = vec![1i32]; - - let result = take_portable_simd::(&values, &indices); - assert_eq!(result.as_slice(), [0i32; 64]); - } - - /// Tests SIMD gather with a mix of sequential, strided, and repeated indices. This exercises - /// irregular access patterns that stress the gather operation. - #[test] - fn test_take_mixed_access_patterns() { - // Create a values array with distinct elements. - let values: Vec = (0..256).map(|i| i * 100).collect(); - - // Build indices with mixed patterns: - // - Sequential access (0, 1, 2, ...). - // - Strided access (0, 4, 8, ...). - // - Repeated indices (same index multiple times). - // - Reverse order. - let mut indices: Vec = Vec::with_capacity(200); - - // Sequential: indices 0..64. - indices.extend(0u32..64); - // Strided by 4: 0, 4, 8, ..., 252. - indices.extend((0u32..64).map(|i| i * 4)); - // Repeated: index 42 repeated 32 times. - indices.extend(std::iter::repeat(42u32).take(32)); - // Reverse: 255, 254, ..., 216. - indices.extend((216u32..256).rev()); - - let result = take_portable_simd::(&values, &indices); - let result_slice = result.as_slice(); - - // Verify sequential portion. - for i in 0..64 { - assert_eq!(result_slice[i], (i as i64) * 100, "sequential at index {i}"); - } - - // Verify strided portion. - for i in 0..64 { - assert_eq!( - result_slice[64 + i], - (i as i64) * 4 * 100, - "strided at index {i}" - ); - } - - // Verify repeated portion. - for i in 0..32 { - assert_eq!(result_slice[128 + i], 42 * 100, "repeated at index {i}"); - } - - // Verify reverse portion. - for i in 0..40 { - assert_eq!( - result_slice[160 + i], - (255 - i as i64) * 100, - "reverse at index {i}" - ); - } - } - - /// Tests that the scalar remainder path works correctly when the number of indices is not - /// evenly divisible by the SIMD lane count. - #[test] - fn test_take_with_remainder() { - let values: Vec = (0..1000).collect(); - - // Use 64 + 37 = 101 indices to test both the SIMD loop (64 elements) and the scalar - // remainder (37 elements). - let indices: Vec = (0u8..101).collect(); - - let result = take_portable_simd::(&values, &indices); - let result_slice = result.as_slice(); - - assert_eq!(result_slice.len(), 101); - - // Verify all elements. - for i in 0..101 { - assert_eq!(result_slice[i], i as u16, "mismatch at index {i}"); - } - - // Also test with exactly 1 remainder element. - let indices_one_remainder: Vec = (0u8..65).collect(); - let result_one = take_portable_simd::(&values, &indices_one_remainder); - assert_eq!(result_one.as_slice().len(), 65); - assert_eq!(result_one.as_slice()[64], 64); - } - - /// Tests gather with large 64-bit values and various index types to ensure no truncation - /// occurs during the operation. - #[test] - fn test_take_large_values_no_truncation() { - // Create values near the edges of i64 range. - let values: Vec = vec![ - i64::MIN, - i64::MIN + 1, - -1_000_000_000_000i64, - -1, - 0, - 1, - 1_000_000_000_000i64, - i64::MAX - 1, - i64::MAX, - ]; - - // Indices that access each value multiple times in different orders. - let indices: Vec = vec![ - 0, 8, 1, 7, 2, 6, 3, 5, 4, // Forward-backward interleaved. - 8, 8, 8, 0, 0, 0, // Repeated extremes. - 4, 4, 4, 4, 4, 4, 4, 4, // Repeated zero. - 0, 1, 2, 3, 4, 5, 6, 7, 8, // Sequential. - 8, 7, 6, 5, 4, 3, 2, 1, 0, // Reverse. - // Pad to 64 to ensure we hit the SIMD path. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, - ]; - - let result = take_portable_simd::(&values, &indices); - let result_slice = result.as_slice(); - - // Verify each result matches the expected value. - for (i, &idx) in indices.iter().enumerate() { - assert_eq!( - result_slice[i], values[idx as usize], - "mismatch at position {i} for index {idx}" - ); - } - } -} diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index 463cb75a162..0869eac3ff4 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -1,7 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors - -#![cfg_attr(vortex_nightly, feature(portable_simd))] //! Vortex crate containing core logic for encoding and memory representation of [arrays](ArrayRef). //! //! At the heart of Vortex are [arrays](ArrayRef). diff --git a/vortex-scan/README.md b/vortex-scan/README.md index 2473604853f..31621370f50 100644 --- a/vortex-scan/README.md +++ b/vortex-scan/README.md @@ -121,7 +121,6 @@ Filters are automatically optimized using: All concurrent code has been verified using: -- **Loom Testing**: Exhaustive verification of all possible thread interleavings - **Address Sanitizer**: Memory safety verification in CI - **Debug Assertions**: Runtime checks for invariants in debug builds @@ -135,23 +134,6 @@ Run the standard test suite: cargo test -p vortex-scan --all-features ``` -### Loom Concurrency Tests - -The crate includes comprehensive Loom tests that exhaustively verify concurrent behavior. -These tests run by default but can be disabled if need be: - -```rust -# Skip Loom tests when using incompatible tools like address sanitizer -RUSTFLAGS="--cfg disable_loom" cargo test -p vortex-scan -``` - -Loom tests verify: - -- Memory ordering correctness in the work-stealing queue -- Absence of data races in filter expression evaluation -- Proper synchronization in concurrent task factories -- Thread termination conditions and cleanup - ## Performance Considerations ### Concurrency Level From 8d9052ee5a8184c9694edc227603261327276e38 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 10 Apr 2026 13:35:03 -0400 Subject: [PATCH 010/250] Decompose TurboQuant into 2 `ScalarFnArray`s and `DictArray` (#7374) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Decomposes `TurboQuant` into: ```text ScalarFnArray(L2Denorm, [ ScalarFnArray(SorfTransform, [ Extension( FixedSizeListArray( DictArray(codes=Primitive, values=Primitive), padded_dim ) ) ]), norms ]) ``` This makes the implementation more modular and turns the TurboQuant-specific pieces into reusable building blocks. Also defines SORF sign generation with a frozen local `SplitMix64` implementation and cleans up related vector compute code. ## API Changes - add `SorfTransform` - remove the `TurboQuant` encoding/array type - keep `TurboQuantScheme` as the compressor entry point - simplify a all scalar fn APIs as we no longer need `ApproxOptions` Note that `ApproxOptions` doesn't actually make sense here because we are doing exact compute here, it is the encoding itself that is lossy. Until we figure out what the exact semantics are of a lossy encoding I will remove this. ## Testing More tests: - TurboQuant roundtrip / structural coverage - SORF roundtrip + norm preservation - deterministic SplitMix64 / sign generation coverage - readthrough behavior (when children are normalized) for vector similarity ops --------- Signed-off-by: Connor Tsui Co-authored-by: Claude --- vortex-file/src/strategy.rs | 4 - vortex-tensor/Cargo.toml | 2 +- vortex-tensor/public-api.lock | 244 ++-- vortex-tensor/src/encodings/mod.rs | 1 - .../src/encodings/turboquant/array/data.rs | 261 ---- .../src/encodings/turboquant/array/mod.rs | 11 - .../src/encodings/turboquant/array/slots.rs | 38 - .../turboquant/{array => }/centroids.rs | 18 +- .../turboquant/{scheme => }/compress.rs | 223 ++- .../src/encodings/turboquant/compute/mod.rs | 22 - .../src/encodings/turboquant/compute/ops.rs | 28 - .../src/encodings/turboquant/compute/rules.rs | 15 - .../src/encodings/turboquant/compute/slice.rs | 32 - .../src/encodings/turboquant/compute/take.rs | 33 - .../src/encodings/turboquant/metadata.rs | 89 -- vortex-tensor/src/encodings/turboquant/mod.rs | 91 +- .../turboquant/{scheme/mod.rs => scheme.rs} | 125 +- .../encodings/turboquant/scheme/decompress.rs | 141 -- .../src/encodings/turboquant/tests.rs | 1260 ----------------- .../src/encodings/turboquant/tests/compute.rs | 214 +++ .../src/encodings/turboquant/tests/mod.rs | 217 +++ .../encodings/turboquant/tests/nullable.rs | 178 +++ .../encodings/turboquant/tests/roundtrip.rs | 343 +++++ .../encodings/turboquant/tests/structural.rs | 333 +++++ .../src/encodings/turboquant/vtable.rs | 382 ----- vortex-tensor/src/lib.rs | 6 +- .../src/scalar_fns/cosine_similarity.rs | 72 +- vortex-tensor/src/scalar_fns/inner_product.rs | 63 +- vortex-tensor/src/scalar_fns/l2_denorm.rs | 129 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 36 +- vortex-tensor/src/scalar_fns/mod.rs | 34 +- .../src/scalar_fns/sorf_transform/mod.rs | 146 ++ .../sorf_transform}/rotation.rs | 212 ++- .../scalar_fns/sorf_transform/splitmix64.rs | 73 + .../src/scalar_fns/sorf_transform/tests.rs | 438 ++++++ .../src/scalar_fns/sorf_transform/vtable.rs | 229 +++ vortex-tensor/src/utils.rs | 36 + vortex/benches/single_encoding_throughput.rs | 3 +- 38 files changed, 2836 insertions(+), 2946 deletions(-) delete mode 100644 vortex-tensor/src/encodings/turboquant/array/data.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/array/mod.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/array/slots.rs rename vortex-tensor/src/encodings/turboquant/{array => }/centroids.rs (96%) rename vortex-tensor/src/encodings/turboquant/{scheme => }/compress.rs (52%) delete mode 100644 vortex-tensor/src/encodings/turboquant/compute/mod.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/compute/ops.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/compute/rules.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/compute/slice.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/compute/take.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/metadata.rs rename vortex-tensor/src/encodings/turboquant/{scheme/mod.rs => scheme.rs} (58%) delete mode 100644 vortex-tensor/src/encodings/turboquant/scheme/decompress.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/tests.rs create mode 100644 vortex-tensor/src/encodings/turboquant/tests/compute.rs create mode 100644 vortex-tensor/src/encodings/turboquant/tests/mod.rs create mode 100644 vortex-tensor/src/encodings/turboquant/tests/nullable.rs create mode 100644 vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs create mode 100644 vortex-tensor/src/encodings/turboquant/tests/structural.rs delete mode 100644 vortex-tensor/src/encodings/turboquant/vtable.rs create mode 100644 vortex-tensor/src/scalar_fns/sorf_transform/mod.rs rename vortex-tensor/src/{encodings/turboquant/array => scalar_fns/sorf_transform}/rotation.rs (65%) create mode 100644 vortex-tensor/src/scalar_fns/sorf_transform/splitmix64.rs create mode 100644 vortex-tensor/src/scalar_fns/sorf_transform/tests.rs create mode 100644 vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index a30d28ab886..0a3e31f32e3 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -56,8 +56,6 @@ use vortex_pco::Pco; use vortex_runend::RunEnd; use vortex_sequence::Sequence; use vortex_sparse::Sparse; -#[cfg(feature = "unstable_encodings")] -use vortex_tensor::encodings::turboquant::TurboQuant; use vortex_utils::aliases::hash_map::HashMap; use vortex_utils::aliases::hash_set::HashSet; use vortex_zigzag::ZigZag; @@ -111,8 +109,6 @@ pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { allowed.insert(RunEnd.id()); allowed.insert(Sequence.id()); allowed.insert(Sparse.id()); - #[cfg(feature = "unstable_encodings")] - allowed.insert(TurboQuant.id()); allowed.insert(ZigZag.id()); #[cfg(feature = "zstd")] diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index 9f94a0c2d3d..3dd463bf4df 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -29,8 +29,8 @@ half = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } prost = { workspace = true } -rand = { workspace = true } [dev-dependencies] +rand = { workspace = true } rand_distr = { workspace = true } rstest = { workspace = true } diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 8a14e2204f0..bec8df1cb29 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -4,78 +4,6 @@ pub mod vortex_tensor::encodings pub mod vortex_tensor::encodings::turboquant -pub struct vortex_tensor::encodings::turboquant::TurboQuant - -impl vortex_tensor::encodings::turboquant::TurboQuant - -pub const vortex_tensor::encodings::turboquant::TurboQuant::ID: vortex_array::array::ArrayId - -pub const vortex_tensor::encodings::turboquant::TurboQuant::MAX_BIT_WIDTH: u8 - -pub const vortex_tensor::encodings::turboquant::TurboQuant::MAX_CENTROIDS: usize - -pub const vortex_tensor::encodings::turboquant::TurboQuant::MIN_DIMENSION: u32 - -pub unsafe fn vortex_tensor::encodings::turboquant::TurboQuant::new_array_unchecked(dtype: vortex_array::dtype::DType, codes: vortex_array::array::erased::ArrayRef, centroids: vortex_array::array::erased::ArrayRef, rotation_signs: vortex_array::array::erased::ArrayRef) -> vortex_tensor::encodings::turboquant::TurboQuantArray - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::try_new_array(dtype: vortex_array::dtype::DType, codes: vortex_array::array::erased::ArrayRef, centroids: vortex_array::array::erased::ArrayRef, rotation_signs: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::validate_dtype(dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult - -impl core::clone::Clone for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::clone(&self) -> vortex_tensor::encodings::turboquant::TurboQuant - -impl core::fmt::Debug for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::array::vtable::VTable for vortex_tensor::encodings::turboquant::TurboQuant - -pub type vortex_tensor::encodings::turboquant::TurboQuant::ArrayData = vortex_tensor::encodings::turboquant::TurboQuantData - -pub type vortex_tensor::encodings::turboquant::TurboQuant::OperationsVTable = vortex_tensor::encodings::turboquant::TurboQuant - -pub type vortex_tensor::encodings::turboquant::TurboQuant::ValidityVTable = vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::buffer(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::execute_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::id(&self) -> vortex_array::array::ArrayId - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::serialize(array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> - -impl vortex_array::array::vtable::operations::OperationsVTable for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_tensor::encodings::turboquant::TurboQuant>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - -impl vortex_array::array::vtable::validity::ValidityVTable for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::validity(_array: vortex_array::array::view::ArrayView<'_, vortex_tensor::encodings::turboquant::TurboQuant>) -> vortex_error::VortexResult - -impl vortex_array::arrays::dict::take::TakeExecute for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::take(array: vortex_array::array::view::ArrayView<'_, vortex_tensor::encodings::turboquant::TurboQuant>, indices: &vortex_array::array::erased::ArrayRef, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> - -impl vortex_array::arrays::slice::SliceReduce for vortex_tensor::encodings::turboquant::TurboQuant - -pub fn vortex_tensor::encodings::turboquant::TurboQuant::slice(array: vortex_array::array::view::ArrayView<'_, vortex_tensor::encodings::turboquant::TurboQuant>, range: core::ops::range::Range) -> vortex_error::VortexResult> - pub struct vortex_tensor::encodings::turboquant::TurboQuantConfig pub vortex_tensor::encodings::turboquant::TurboQuantConfig::bit_width: u8 @@ -96,44 +24,6 @@ impl core::fmt::Debug for vortex_tensor::encodings::turboquant::TurboQuantConfig pub fn vortex_tensor::encodings::turboquant::TurboQuantConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_tensor::encodings::turboquant::TurboQuantData - -impl vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::bit_width(&self) -> u8 - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::dimension(&self) -> u32 - -pub unsafe fn vortex_tensor::encodings::turboquant::TurboQuantData::new_unchecked(dimension: u32, bit_width: u8, num_rounds: u8) -> Self - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::num_rounds(&self) -> u8 - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::padded_dim(&self) -> u32 - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::try_new(dimension: u32, bit_width: u8, num_rounds: u8) -> vortex_error::VortexResult - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::validate(dtype: &vortex_array::dtype::DType, codes: &vortex_array::array::erased::ArrayRef, centroids: &vortex_array::array::erased::ArrayRef, rotation_signs: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult<()> - -impl core::clone::Clone for vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::clone(&self) -> vortex_tensor::encodings::turboquant::TurboQuantData - -impl core::fmt::Debug for vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::fmt::Display for vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::hash::ArrayEq for vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool - -impl vortex_array::hash::ArrayHash for vortex_tensor::encodings::turboquant::TurboQuantData - -pub fn vortex_tensor::encodings::turboquant::TurboQuantData::array_hash(&self, state: &mut H, _precision: vortex_array::hash::Precision) - pub struct vortex_tensor::encodings::turboquant::TurboQuantScheme impl core::clone::Clone for vortex_tensor::encodings::turboquant::TurboQuantScheme @@ -164,28 +54,18 @@ pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::matches(&self, ca pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::scheme_name(&self) -> &'static str -pub trait vortex_tensor::encodings::turboquant::TurboQuantArrayExt: vortex_array::array::typed::TypedArrayRef - -pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::centroids(&self) -> &vortex_array::array::erased::ArrayRef +pub const vortex_tensor::encodings::turboquant::MAX_BIT_WIDTH: u8 -pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::codes(&self) -> &vortex_array::array::erased::ArrayRef +pub const vortex_tensor::encodings::turboquant::MAX_CENTROIDS: usize -pub fn vortex_tensor::encodings::turboquant::TurboQuantArrayExt::rotation_signs(&self) -> &vortex_array::array::erased::ArrayRef +pub const vortex_tensor::encodings::turboquant::MIN_DIMENSION: u32 -impl> vortex_tensor::encodings::turboquant::TurboQuantArrayExt for T - -pub fn T::centroids(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn T::codes(&self) -> &vortex_array::array::erased::ArrayRef - -pub fn T::rotation_signs(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_tensor::encodings::turboquant::tq_validate_vector_dtype(dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult pub fn vortex_tensor::encodings::turboquant::turboquant_encode(ext: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::extension::vtable::Extension>, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub unsafe fn vortex_tensor::encodings::turboquant::turboquant_encode_unchecked(ext: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::extension::vtable::Extension>, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub type vortex_tensor::encodings::turboquant::TurboQuantArray = vortex_array::array::typed::Array - pub mod vortex_tensor::fixed_shape pub struct vortex_tensor::fixed_shape::AnyFixedShapeTensor @@ -360,9 +240,9 @@ pub struct vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity impl vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity -pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::new(options: &vortex_tensor::scalar_fns::ApproxOptions) -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::new() -> vortex_array::scalar_fn::typed::ScalarFn -pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::try_new_array(options: &vortex_tensor::scalar_fns::ApproxOptions, lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::try_new_array(lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity @@ -370,13 +250,13 @@ pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::clone(&se impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity -pub type vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::Options = vortex_tensor::scalar_fns::ApproxOptions +pub type vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::Options = vortex_array::scalar_fn::vtable::EmptyOptions pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::vtable::Arity pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::child_name(&self, _options: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::vtable::ChildName -pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::execute(&self, _options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::expression::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -396,9 +276,9 @@ pub struct vortex_tensor::scalar_fns::inner_product::InnerProduct impl vortex_tensor::scalar_fns::inner_product::InnerProduct -pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::new(options: &vortex_tensor::scalar_fns::ApproxOptions) -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::new() -> vortex_array::scalar_fn::typed::ScalarFn -pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::try_new_array(options: &vortex_tensor::scalar_fns::ApproxOptions, lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::try_new_array(lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_tensor::scalar_fns::inner_product::InnerProduct @@ -406,13 +286,13 @@ pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::clone(&self) -> v impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::inner_product::InnerProduct -pub type vortex_tensor::scalar_fns::inner_product::InnerProduct::Options = vortex_tensor::scalar_fns::ApproxOptions +pub type vortex_tensor::scalar_fns::inner_product::InnerProduct::Options = vortex_array::scalar_fn::vtable::EmptyOptions pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::vtable::Arity pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::child_name(&self, _options: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::vtable::ChildName -pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::execute(&self, _options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::expression::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -432,11 +312,11 @@ pub struct vortex_tensor::scalar_fns::l2_denorm::L2Denorm impl vortex_tensor::scalar_fns::l2_denorm::L2Denorm -pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new(options: &vortex_tensor::scalar_fns::ApproxOptions) -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new() -> vortex_array::scalar_fn::typed::ScalarFn -pub unsafe fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new_array_unchecked(options: &vortex_tensor::scalar_fns::ApproxOptions, normalized: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult +pub unsafe fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new_array_unchecked(normalized: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::try_new_array(options: &vortex_tensor::scalar_fns::ApproxOptions, normalized: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, len: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::try_new_array(normalized: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, len: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_tensor::scalar_fns::l2_denorm::L2Denorm @@ -444,7 +324,7 @@ pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::clone(&self) -> vortex_te impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::l2_denorm::L2Denorm -pub type vortex_tensor::scalar_fns::l2_denorm::L2Denorm::Options = vortex_tensor::scalar_fns::ApproxOptions +pub type vortex_tensor::scalar_fns::l2_denorm::L2Denorm::Options = vortex_array::scalar_fn::vtable::EmptyOptions pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::vtable::Arity @@ -464,9 +344,9 @@ pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::return_dtype(&self, _opti pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::validity(&self, _options: &Self::Options, expression: &vortex_array::expr::expression::Expression) -> vortex_error::VortexResult> -pub fn vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm(options: &vortex_tensor::scalar_fns::ApproxOptions, input: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm(input: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::l2_denorm::validate_l2_normalized_rows(input: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_tensor::scalar_fns::l2_denorm::validate_l2_normalized_rows(input: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> pub mod vortex_tensor::scalar_fns::l2_norm @@ -474,9 +354,9 @@ pub struct vortex_tensor::scalar_fns::l2_norm::L2Norm impl vortex_tensor::scalar_fns::l2_norm::L2Norm -pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::new(options: &vortex_tensor::scalar_fns::ApproxOptions) -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::new() -> vortex_array::scalar_fn::typed::ScalarFn -pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::try_new_array(options: &vortex_tensor::scalar_fns::ApproxOptions, child: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::try_new_array(child: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_tensor::scalar_fns::l2_norm::L2Norm @@ -484,7 +364,7 @@ pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::clone(&self) -> vortex_tensor impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::l2_norm::L2Norm -pub type vortex_tensor::scalar_fns::l2_norm::L2Norm::Options = vortex_tensor::scalar_fns::ApproxOptions +pub type vortex_tensor::scalar_fns::l2_norm::L2Norm::Options = vortex_array::scalar_fn::vtable::EmptyOptions pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::vtable::Arity @@ -504,45 +384,87 @@ pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::return_dtype(&self, _options: pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::validity(&self, _options: &Self::Options, expression: &vortex_array::expr::expression::Expression) -> vortex_error::VortexResult> -pub enum vortex_tensor::scalar_fns::ApproxOptions +pub mod vortex_tensor::scalar_fns::sorf_transform + +pub struct vortex_tensor::scalar_fns::sorf_transform::SorfMatrix + +impl vortex_tensor::scalar_fns::sorf_transform::SorfMatrix + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::inverse_rotate(&self, input: &[f32], output: &mut [f32]) + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::padded_dim(&self) -> usize + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::rotate(&self, input: &[f32], output: &mut [f32]) + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::try_new(seed: u64, dimension: usize, num_rounds: usize) -> vortex_error::VortexResult + +pub struct vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::dimension: u32 + +pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::element_ptype: vortex_array::dtype::ptype::PType + +pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::num_rounds: u8 + +pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::seed: u64 + +impl core::clone::Clone for vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfOptions::clone(&self) -> vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +impl core::cmp::Eq for vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +impl core::cmp::PartialEq for vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfOptions::eq(&self, other: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions) -> bool + +impl core::fmt::Debug for vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl core::fmt::Display for vortex_tensor::scalar_fns::sorf_transform::SorfOptions + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl core::hash::Hash for vortex_tensor::scalar_fns::sorf_transform::SorfOptions -pub vortex_tensor::scalar_fns::ApproxOptions::Approximate +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfOptions::hash<__H: core::hash::Hasher>(&self, state: &mut __H) -pub vortex_tensor::scalar_fns::ApproxOptions::Exact +impl core::marker::StructuralPartialEq for vortex_tensor::scalar_fns::sorf_transform::SorfOptions -impl vortex_tensor::scalar_fns::ApproxOptions +pub struct vortex_tensor::scalar_fns::sorf_transform::SorfTransform -pub fn vortex_tensor::scalar_fns::ApproxOptions::is_approx(&self) -> bool +impl vortex_tensor::scalar_fns::sorf_transform::SorfTransform -pub fn vortex_tensor::scalar_fns::ApproxOptions::is_exact(&self) -> bool +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::new(options: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions) -> vortex_array::scalar_fn::typed::ScalarFn -impl core::clone::Clone for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::try_new_array(options: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions, child: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::ApproxOptions::clone(&self) -> vortex_tensor::scalar_fns::ApproxOptions +impl core::clone::Clone for vortex_tensor::scalar_fns::sorf_transform::SorfTransform -impl core::cmp::Eq for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::clone(&self) -> vortex_tensor::scalar_fns::sorf_transform::SorfTransform -impl core::cmp::PartialEq for vortex_tensor::scalar_fns::ApproxOptions +impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::sorf_transform::SorfTransform -pub fn vortex_tensor::scalar_fns::ApproxOptions::eq(&self, other: &vortex_tensor::scalar_fns::ApproxOptions) -> bool +pub type vortex_tensor::scalar_fns::sorf_transform::SorfTransform::Options = vortex_tensor::scalar_fns::sorf_transform::SorfOptions -impl core::default::Default for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::vtable::Arity -pub fn vortex_tensor::scalar_fns::ApproxOptions::default() -> vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::child_name(&self, _options: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::vtable::ChildName -impl core::fmt::Debug for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::ApproxOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::expression::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::fmt::Display for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::id(&self) -> vortex_array::scalar_fn::ScalarFnId -pub fn vortex_tensor::scalar_fns::ApproxOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::is_fallible(&self, _options: &Self::Options) -> bool -impl core::hash::Hash for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::is_null_sensitive(&self, _options: &Self::Options) -> bool -pub fn vortex_tensor::scalar_fns::ApproxOptions::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::return_dtype(&self, options: &Self::Options, arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult -impl core::marker::StructuralPartialEq for vortex_tensor::scalar_fns::ApproxOptions +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::validity(&self, _options: &Self::Options, expression: &vortex_array::expr::expression::Expression) -> vortex_error::VortexResult> pub mod vortex_tensor::vector diff --git a/vortex-tensor/src/encodings/mod.rs b/vortex-tensor/src/encodings/mod.rs index 7c75269b632..084baf97e57 100644 --- a/vortex-tensor/src/encodings/mod.rs +++ b/vortex-tensor/src/encodings/mod.rs @@ -4,7 +4,6 @@ //! Encodings for the different tensor types. // TODO(connor): -// pub mod norm; // Unit-normalized vectors. // pub mod spherical; // Spherical transform on unit-normalized vectors. pub mod turboquant; diff --git a/vortex-tensor/src/encodings/turboquant/array/data.rs b/vortex-tensor/src/encodings/turboquant/array/data.rs deleted file mode 100644 index dd1867eb598..00000000000 --- a/vortex-tensor/src/encodings/turboquant/array/data.rs +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; -use std::sync::Arc; - -use vortex_array::ArrayRef; -use vortex_array::TypedArrayRef; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_ensure_eq; - -use crate::encodings::turboquant::array::slots::Slot; -use crate::encodings::turboquant::vtable::TurboQuant; - -/// TurboQuant array data. -/// -/// TurboQuant is a lossy vector quantization encoding for [`Vector`](crate::vector::Vector) -/// extension arrays. It stores quantized coordinate codes for unit-norm vectors, along with shared -/// codebook centroids and the parameters of the current structured rotation. -/// -/// Norms should be stored externally in the [`L2Denorm`](crate::scalar_fns::l2_denorm::L2Denorm) -/// `ScalarFnArray` wrapper. -/// -/// See the [module docs](crate::encodings::turboquant) for algorithmic details. -/// -/// Note that degenerate TurboQuant arrays have zero rows and `bit_width == 0`, with all slots -/// empty. -#[derive(Clone, Debug)] -pub struct TurboQuantData { - /// The vector dimension `d`, cached from the `FixedSizeList` storage dtype's list size. - /// - /// Stored as a convenience field to avoid repeatedly extracting it from `dtype`. - pub(crate) dimension: u32, - - /// The number of bits per coordinate (0-8), derived from `log2(centroids.len())`. - /// - /// This is 0 for degenerate empty arrays. - pub(crate) bit_width: u8, - - /// The number of sign-diagonal + WHT rounds in the structured rotation. - /// - /// This is 0 for degenerate empty arrays. - pub(crate) num_rounds: u8, -} - -impl Display for TurboQuantData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "dimension: {}, bit_width: {}, num_rounds: {}", - self.dimension, self.bit_width, self.num_rounds - ) - } -} - -impl TurboQuantData { - /// Build a `TurboQuantData` with validation. - /// - /// # Errors - /// - /// Returns an error if: - /// - `dimension` is less than [`MIN_DIMENSION`](TurboQuant::MIN_DIMENSION). - /// - `bit_width` is greater than [`MAX_BIT_WIDTH`](TurboQuant::MAX_BIT_WIDTH). - pub fn try_new(dimension: u32, bit_width: u8, num_rounds: u8) -> VortexResult { - vortex_ensure!( - dimension >= TurboQuant::MIN_DIMENSION, - "TurboQuant requires dimension >= {}, got {dimension}", - TurboQuant::MIN_DIMENSION - ); - vortex_ensure!( - bit_width <= TurboQuant::MAX_BIT_WIDTH, - "bit_width is expected to be between 0 and {}, got {bit_width}", - TurboQuant::MAX_BIT_WIDTH - ); - - Ok(Self { - dimension, - bit_width, - num_rounds, - }) - } - - /// Build a `TurboQuantData` without validation. - /// - /// # Safety - /// - /// The caller must ensure: - /// - /// - `dimension` is >= [`MIN_DIMENSION`](TurboQuant::MIN_DIMENSION). - /// - `bit_width` is in the range `[0, MAX_BIT_WIDTH]`. - /// - `num_rounds` is >= 1 (or 0 for degenerate empty arrays). - /// - /// Violating these invariants may produce incorrect results during decompression. - pub unsafe fn new_unchecked(dimension: u32, bit_width: u8, num_rounds: u8) -> Self { - Self { - dimension, - bit_width, - num_rounds, - } - } - - /// Validates the components that would be used to create a `TurboQuantData`. - /// - /// This function checks all the invariants required by [`new_unchecked`](Self::new_unchecked). - pub fn validate( - dtype: &DType, - codes: &ArrayRef, - centroids: &ArrayRef, - rotation_signs: &ArrayRef, - ) -> VortexResult<()> { - let vector_metadata = TurboQuant::validate_dtype(dtype)?; - let dimension = vector_metadata.dimensions(); - let padded_dim = dimension.next_power_of_two(); - - // TurboQuant arrays are always non-nullable. Nullability should be handled by the external - // L2Denorm ScalarFnArray wrapper. - vortex_ensure!( - !dtype.is_nullable(), - "TurboQuant dtype must be non-nullable, got {dtype}", - ); - - // Codes must be a non-nullable FixedSizeList with list_size == padded_dim. - let expected_codes_dtype = DType::FixedSizeList( - Arc::new(DType::Primitive(PType::U8, Nullability::NonNullable)), - padded_dim, - Nullability::NonNullable, - ); - vortex_ensure_eq!( - *codes.dtype(), - expected_codes_dtype, - "codes dtype does not match expected {expected_codes_dtype}", - ); - - // Centroids are always f32 regardless of element type. - let centroids_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - vortex_ensure_eq!( - *centroids.dtype(), - centroids_dtype, - "centroids dtype must be non-nullable f32", - ); - - // Rotation signs must be a FixedSizeList with list_size == padded_dim. The FSL length - // is the number of rotation rounds. - let expected_signs_dtype = DType::FixedSizeList( - Arc::new(DType::Primitive(PType::U8, Nullability::NonNullable)), - padded_dim, - Nullability::NonNullable, - ); - vortex_ensure_eq!( - *rotation_signs.dtype(), - expected_signs_dtype, - "rotation_signs dtype does not match expected {expected_signs_dtype}", - ); - // Degenerate (empty) case: all children must be empty, and bit_width is 0. - let num_rows = codes.len(); - if num_rows == 0 { - vortex_ensure!( - centroids.is_empty(), - "degenerate TurboQuant must have empty centroids, got length {}", - centroids.len() - ); - vortex_ensure!( - rotation_signs.is_empty(), - "degenerate TurboQuant must have empty rotation_signs, got length {}", - rotation_signs.len() - ); - return Ok(()); - } - - vortex_ensure!( - !rotation_signs.is_empty(), - "rotation_signs must have at least 1 round" - ); - - // Non-degenerate: derive and validate bit_width from centroids. - let num_centroids = centroids.len(); - vortex_ensure!( - num_centroids.is_power_of_two() - && (2..=TurboQuant::MAX_CENTROIDS).contains(&num_centroids), - "centroids length must be a power of 2 in [2, {}], got {num_centroids}", - TurboQuant::MAX_CENTROIDS - ); - - #[expect( - clippy::cast_possible_truncation, - reason = "Guaranteed to be [1,8] by the preceding power-of-2 and range checks." - )] - let bit_width = num_centroids.trailing_zeros() as u8; - vortex_ensure!( - (1..=TurboQuant::MAX_BIT_WIDTH).contains(&bit_width), - "derived bit_width must be 1-{}, got {bit_width}", - TurboQuant::MAX_BIT_WIDTH - ); - - Ok(()) - } - - pub(crate) fn make_slots( - codes: ArrayRef, - centroids: ArrayRef, - rotation_signs: ArrayRef, - ) -> Vec> { - let mut slots = vec![None; Slot::COUNT]; - slots[Slot::Codes as usize] = Some(codes); - slots[Slot::Centroids as usize] = Some(centroids); - slots[Slot::RotationSigns as usize] = Some(rotation_signs); - slots - } - - /// The vector dimension `d`, as stored in the [`Vector`](crate::vector::Vector) extension - /// dtype's `FixedSizeList` storage. - pub fn dimension(&self) -> u32 { - self.dimension - } - - /// MSE bits per coordinate (1-MAX_BIT_WIDTH for non-empty arrays, 0 for degenerate empty arrays). - pub fn bit_width(&self) -> u8 { - self.bit_width - } - - /// The number of sign-diagonal + WHT rounds in the structured rotation. - pub fn num_rounds(&self) -> u8 { - self.num_rounds - } - - /// Padded dimension (next power of 2 >= [`dimension`](Self::dimension)). - /// - /// The current Walsh-Hadamard-based structured rotation requires power-of-2 input, so - /// non-power-of-2 dimensions are zero-padded to this value. - pub fn padded_dim(&self) -> u32 { - self.dimension.next_power_of_two() - } -} - -pub trait TurboQuantArrayExt: TypedArrayRef { - fn codes(&self) -> &ArrayRef { - self.as_ref().slots()[Slot::Codes as usize] - .as_ref() - .vortex_expect("TurboQuantArray codes slot") - } - - fn centroids(&self) -> &ArrayRef { - self.as_ref().slots()[Slot::Centroids as usize] - .as_ref() - .vortex_expect("TurboQuantArray centroids slot") - } - - fn rotation_signs(&self) -> &ArrayRef { - self.as_ref().slots()[Slot::RotationSigns as usize] - .as_ref() - .vortex_expect("TurboQuantArray rotation_signs slot") - } -} - -impl> TurboQuantArrayExt for T {} diff --git a/vortex-tensor/src/encodings/turboquant/array/mod.rs b/vortex-tensor/src/encodings/turboquant/array/mod.rs deleted file mode 100644 index e82313f1dc6..00000000000 --- a/vortex-tensor/src/encodings/turboquant/array/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! TurboQuant array definition: stores quantized coordinate codes, norms, centroids (codebook), -//! and rotation signs. - -pub(crate) mod data; -pub(crate) mod slots; - -pub(crate) mod centroids; -pub(crate) mod rotation; diff --git a/vortex-tensor/src/encodings/turboquant/array/slots.rs b/vortex-tensor/src/encodings/turboquant/array/slots.rs deleted file mode 100644 index c4fe7e0c5bf..00000000000 --- a/vortex-tensor/src/encodings/turboquant/array/slots.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -/// Slot positions for TurboQuantArray children. -/// -/// Norms are not stored in the TurboQuantArray. They live in the external [`L2Denorm`] -/// ScalarFnArray wrapper returned by [`turboquant_encode`]. -/// -/// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm -/// [`turboquant_encode`]: crate::encodings::turboquant::turboquant_encode -#[repr(usize)] -#[derive(Clone, Copy, Debug)] -pub(crate) enum Slot { - Codes = 0, - Centroids = 1, - RotationSigns = 2, -} - -impl Slot { - pub(crate) const COUNT: usize = 3; - - pub(crate) fn name(self) -> &'static str { - match self { - Self::Codes => "codes", - Self::Centroids => "centroids", - Self::RotationSigns => "rotation_signs", - } - } - - pub(crate) fn from_index(idx: usize) -> Self { - match idx { - 0 => Self::Codes, - 1 => Self::Centroids, - 2 => Self::RotationSigns, - _ => vortex_error::vortex_panic!("invalid slot index {idx}"), - } - } -} diff --git a/vortex-tensor/src/encodings/turboquant/array/centroids.rs b/vortex-tensor/src/encodings/turboquant/centroids.rs similarity index 96% rename from vortex-tensor/src/encodings/turboquant/array/centroids.rs rename to vortex-tensor/src/encodings/turboquant/centroids.rs index e3027d0a58a..cd7b8b889ce 100644 --- a/vortex-tensor/src/encodings/turboquant/array/centroids.rs +++ b/vortex-tensor/src/encodings/turboquant/centroids.rs @@ -15,7 +15,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_utils::aliases::dash_map::DashMap; -use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::MAX_BIT_WIDTH; +use crate::encodings::turboquant::MIN_DIMENSION; /// The maximum iterations for Max-Lloyd algorithm when computing centroids. const MAX_ITERATIONS: usize = 200; @@ -37,14 +38,14 @@ static CENTROID_CACHE: LazyLock>> = LazyLock::new(Da /// `dimension`-dimensional space. pub fn get_centroids(dimension: u32, bit_width: u8) -> VortexResult> { vortex_ensure!( - (1..=TurboQuant::MAX_BIT_WIDTH).contains(&bit_width), + (1..=MAX_BIT_WIDTH).contains(&bit_width), "TurboQuant bit_width must be 1-{}, got {bit_width}", - TurboQuant::MAX_BIT_WIDTH + MAX_BIT_WIDTH ); vortex_ensure!( - dimension >= TurboQuant::MIN_DIMENSION, + dimension >= MIN_DIMENSION, "TurboQuant dimension must be >= {}, got {dimension}", - TurboQuant::MIN_DIMENSION + MIN_DIMENSION ); if let Some(centroids) = CENTROID_CACHE.get(&(dimension, bit_width)) { @@ -92,7 +93,7 @@ impl HalfIntExponent { /// `f(x) = C_d * (1 - x^2)^((d-3)/2)` on `[-1, 1]` /// where `C_d` is the normalizing constant. fn max_lloyd_centroids(dimension: u32, bit_width: u8) -> Vec { - debug_assert!((1..=TurboQuant::MAX_BIT_WIDTH).contains(&bit_width)); + debug_assert!((1..=MAX_BIT_WIDTH).contains(&bit_width)); let num_centroids = 1usize << bit_width; // For the marginal distribution on [-1, 1], we use the exponent (d-3)/2. @@ -175,7 +176,6 @@ fn mean_between_centroids(lo: f64, hi: f64, exponent: HalfIntExponent) -> f64 { /// Uses `powi` + `sqrt` instead of `powf` for the half-integer exponents that arise from `(d-3)/2`. /// This is significantly faster than the general `powf` which goes through /// `exp(exponent * ln(base))`. -#[inline] fn pdf_unnormalized(x_val: f64, exponent: HalfIntExponent) -> f64 { let base = (1.0 - x_val * x_val).max(0.0); @@ -199,7 +199,7 @@ pub fn compute_centroid_boundaries(centroids: &[f32]) -> Vec { /// Find the index of the nearest centroid using precomputed decision boundaries. /// -/// `boundaries` must be the output of [`compute_boundaries`] for the corresponding +/// `boundaries` must be the output of [`compute_centroid_boundaries`] for the corresponding /// centroids. Uses binary search on the midpoints, avoiding distance comparisons /// in the inner loop. #[inline] @@ -210,7 +210,7 @@ pub fn find_nearest_centroid(value: f32, boundaries: &[f32]) -> u8 { ); debug_assert!( boundaries.len() <= 256, // 1 << 8 - "boundaries must be sorted" + "too many boundaries" ); #[expect( diff --git a/vortex-tensor/src/encodings/turboquant/scheme/compress.rs b/vortex-tensor/src/encodings/turboquant/compress.rs similarity index 52% rename from vortex-tensor/src/encodings/turboquant/scheme/compress.rs rename to vortex-tensor/src/encodings/turboquant/compress.rs index f787a80cbf7..b0173bbe36c 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme/compress.rs +++ b/vortex-tensor/src/encodings/turboquant/compress.rs @@ -6,35 +6,42 @@ //! The input to [`turboquant_encode`] must be a non-nullable [`Vector`](crate::vector::Vector) //! extension array whose rows are already L2-normalized (unit norm). Normalization is handled //! externally by [`normalize_as_l2_denorm`](crate::scalar_fns::l2_denorm::normalize_as_l2_denorm), -//! which the [`TurboQuantScheme`](super::TurboQuantScheme) calls before invoking this function. +//! which the [`TurboQuantScheme`] calls before invoking this function. +//! +//! [`TurboQuantScheme`]: crate::encodings::turboquant::TurboQuantScheme use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Extension; +use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArray; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::extension::EmptyMetadata; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_fastlanes::bitpack_compress::bitpack_encode; - -use crate::encodings::turboquant::TurboQuant; -use crate::encodings::turboquant::array::centroids::compute_centroid_boundaries; -use crate::encodings::turboquant::array::centroids::find_nearest_centroid; -use crate::encodings::turboquant::array::centroids::get_centroids; -use crate::encodings::turboquant::array::rotation::RotationMatrix; -use crate::encodings::turboquant::vtable::TurboQuantArray; + +use crate::encodings::turboquant::MAX_BIT_WIDTH; +use crate::encodings::turboquant::MIN_DIMENSION; +use crate::encodings::turboquant::centroids::compute_centroid_boundaries; +use crate::encodings::turboquant::centroids::find_nearest_centroid; +use crate::encodings::turboquant::centroids::get_centroids; use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows; +use crate::scalar_fns::sorf_transform::SorfMatrix; +use crate::scalar_fns::sorf_transform::SorfOptions; +use crate::scalar_fns::sorf_transform::SorfTransform; +use crate::utils::cast_to_f32; +use crate::vector::AnyVector; +use crate::vector::Vector; /// Configuration for TurboQuant encoding. #[derive(Clone, Debug)] @@ -50,50 +57,15 @@ pub struct TurboQuantConfig { impl Default for TurboQuantConfig { fn default() -> Self { Self { - bit_width: TurboQuant::MAX_BIT_WIDTH, + bit_width: MAX_BIT_WIDTH, seed: Some(42), num_rounds: 3, } } } -/// Extract elements from a FixedSizeListArray as a flat f32 PrimitiveArray for quantization. -/// -/// All quantization (rotation, centroid lookup) happens in f32. f16 is upcast; f64 is truncated. -fn extract_f32_elements( - fsl: &FixedSizeListArray, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let elements = fsl.elements(); - let primitive = elements.clone().execute::(ctx)?; - let ptype = primitive.ptype(); - - match ptype { - PType::F16 => Ok(primitive - .as_slice::() - .iter() - .map(|&v| f32::from(v)) - .collect()), - PType::F32 => Ok(primitive), - PType::F64 => Ok(primitive - .as_slice::() - .iter() - .map(|&v| { - #[expect( - clippy::cast_possible_truncation, - reason = "TurboQuant quantization operates in f32, so f64 inputs are intentionally downcast" - )] - let v = v as f32; - v - }) - .collect()), - _ => vortex_bail!("TurboQuant requires float elements, got {ptype:?}"), - } -} - /// Shared intermediate results from the quantization loop. struct QuantizationResult { - rotation: RotationMatrix, centroids: Vec, all_indices: BufferMut, padded_dim: usize, @@ -115,12 +87,13 @@ fn turboquant_quantize_core( usize::try_from(fsl.list_size()).vortex_expect("u32 FixedSizeList dimension fits in usize"); let num_rows = fsl.len(); - let rotation = RotationMatrix::try_new(seed, dimension, num_rounds as usize)?; + let rotation = SorfMatrix::try_new(seed, dimension, num_rounds as usize)?; let padded_dim = rotation.padded_dim(); let padded_dim_u32 = u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); - let f32_elements = extract_f32_elements(fsl, ctx)?; + let elements_prim: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + let f32_elements = cast_to_f32(elements_prim)?; let centroids = get_centroids(padded_dim_u32, bit_width)?; let boundaries = compute_centroid_boundaries(¢roids); @@ -129,7 +102,7 @@ fn turboquant_quantize_core( let mut padded = vec![0.0f32; padded_dim]; let mut rotated = vec![0.0f32; padded_dim]; - let f32_slice = f32_elements.as_slice::(); + let f32_slice = f32_elements.as_slice(); for row in 0..num_rows { let x = &f32_slice[row * dimension..(row + 1) * dimension]; @@ -145,48 +118,44 @@ fn turboquant_quantize_core( } Ok(QuantizationResult { - rotation, centroids, all_indices, padded_dim, }) } -/// Build a `TurboQuantArray` from quantization results. +/// Build a quantized representation: `FSL(DictArray(codes, centroids), padded_dim)`. /// -/// The `ext_dtype` must be a non-nullable [`Vector`](crate::vector::Vector) extension dtype. -fn build_turboquant( +/// This is a Dict-encoded FixedSizeList where each row of `padded_dim` u8 codes +/// indexes into the centroid codebook. The Dict can be independently sliced, taken, +/// or executed (dequantized) without knowledge of the rotation. +fn build_quantized_fsl( num_rows: usize, - core: QuantizationResult, - ext_dtype: &DType, -) -> VortexResult { - let padded_dim = core.padded_dim; + all_indices: BufferMut, + centroids: &[f32], + padded_dim: usize, +) -> VortexResult { + let codes = PrimitiveArray::new::(all_indices.freeze(), Validity::NonNullable); + + let mut centroids_buf = BufferMut::::with_capacity(centroids.len()); + centroids_buf.extend_from_slice(centroids); + let centroids_array = PrimitiveArray::new::(centroids_buf.freeze(), Validity::NonNullable); + + let dict = DictArray::try_new(codes.into_array(), centroids_array.into_array())?; + let padded_dim_u32 = u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); - let codes_elements = - PrimitiveArray::new::(core.all_indices.freeze(), Validity::NonNullable); - let codes = FixedSizeListArray::try_new( - codes_elements.into_array(), + Ok(FixedSizeListArray::try_new( + dict.into_array(), padded_dim_u32, Validity::NonNullable, num_rows, )? - .into_array(); - - // TODO(perf): `get_centroids` returns Vec; could avoid the copy by - // supporting Buffer::from(Vec) or caching as Buffer directly. - let mut centroids_buf = BufferMut::::with_capacity(core.centroids.len()); - centroids_buf.extend_from_slice(&core.centroids); - let centroids_array = - PrimitiveArray::new::(centroids_buf.freeze(), Validity::NonNullable).into_array(); - - let rotation_signs = bitpack_rotation_signs(&core.rotation)?; - - TurboQuant::try_new_array(ext_dtype.clone(), codes, centroids_array, rotation_signs) + .into_array()) } /// Encode a non-nullable, L2-normalized [`Vector`](crate::vector::Vector) extension array into a -/// [`TurboQuantArray`]. +/// `ScalarFnArray(SorfTransform, [FSL(Dict(codes, centroids))])`. /// /// The input must be a non-nullable Vector extension array whose rows are already unit-norm. /// **Null vectors are not supported.** The caller must normalize and strip nullability before @@ -196,9 +165,9 @@ fn build_turboquant( /// [`turboquant_encode_unchecked`] to skip this check when the caller has just performed /// normalization. /// -/// The returned array is a plain [`TurboQuantArray`] that decompresses to unit-norm vectors. -/// The caller is responsible for wrapping it in an [`L2Denorm`] ScalarFnArray if the original -/// magnitudes need to be restored. +/// The returned array is a `SorfTransform` ScalarFnArray wrapping `FSL(Dict)` that decompresses +/// to unit-norm vectors. The caller is responsible for wrapping it in an [`L2Denorm`] ScalarFnArray +/// if the original magnitudes need to be restored. /// /// [`normalize_as_l2_denorm`]: crate::scalar_fns::l2_denorm::normalize_as_l2_denorm /// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm @@ -214,14 +183,15 @@ pub fn turboquant_encode( "TurboQuant input must be non-nullable (normalize first via L2Denorm), got {ext_dtype}", ); - validate_l2_normalized_rows(ext.as_ref().clone(), ctx)?; + validate_l2_normalized_rows(ext.as_ref(), ctx)?; // SAFETY: We just validated that the input is non-nullable and all rows are unit-norm. unsafe { turboquant_encode_unchecked(ext, config, ctx) } } /// Encode a non-nullable, L2-normalized [`Vector`](crate::vector::Vector) extension array into a -/// [`TurboQuantArray`], without validating the unit-norm precondition. +/// `ScalarFnArray(SorfTransform, [FSL(Dict(codes, centroids))])`, without validating the unit-norm +/// precondition. /// /// # Safety /// @@ -242,71 +212,64 @@ pub unsafe fn turboquant_encode_unchecked( let fsl = storage.clone().execute::(ctx)?; vortex_ensure!( - config.bit_width >= 1 && config.bit_width <= TurboQuant::MAX_BIT_WIDTH, - "bit_width must be 1-{}, got {}", - TurboQuant::MAX_BIT_WIDTH, + config.bit_width >= 1 && config.bit_width <= MAX_BIT_WIDTH, + "bit_width must be 1-{MAX_BIT_WIDTH}, got {}", config.bit_width ); let dimension = fsl.list_size(); vortex_ensure!( - dimension >= TurboQuant::MIN_DIMENSION, - "TurboQuant requires dimension >= {}, got {dimension}", - TurboQuant::MIN_DIMENSION + dimension >= MIN_DIMENSION, + "TurboQuant requires dimension >= {MIN_DIMENSION}, got {dimension}", ); + let vector_metadata = ext_dtype.as_extension().metadata::(); + let element_ptype = vector_metadata.element_ptype(); + + let seed = config.seed.unwrap_or(42); + let num_rows = fsl.len(); + if fsl.is_empty() { let padded_dim = dimension.next_power_of_two(); - let empty_codes = FixedSizeListArray::try_new( - PrimitiveArray::empty::(Nullability::NonNullable).into_array(), - padded_dim, - Validity::NonNullable, - 0, - )?; - + let empty_codes = PrimitiveArray::empty::(Nullability::NonNullable); let empty_centroids = PrimitiveArray::empty::(Nullability::NonNullable); - let empty_signs = FixedSizeListArray::try_new( - PrimitiveArray::empty::(Nullability::NonNullable).into_array(), + let empty_dict = + DictArray::try_new(empty_codes.into_array(), empty_centroids.into_array())?; + let empty_fsl = FixedSizeListArray::try_new( + empty_dict.into_array(), padded_dim, Validity::NonNullable, 0, )?; - - return Ok(TurboQuant::try_new_array( - ext_dtype, - empty_codes.into_array(), - empty_centroids.into_array(), - empty_signs.into_array(), - )? - .into_array()); + let empty_padded_vector = wrap_padded_as_vector(empty_fsl.into_array())?; + + let sorf_options = SorfOptions { + seed, + num_rounds: config.num_rounds, + dimension, + element_ptype, + }; + return Ok( + SorfTransform::try_new_array(&sorf_options, empty_padded_vector, 0)?.into_array(), + ); } - let seed = config.seed.unwrap_or(42); - let num_rows = fsl.len(); let core = turboquant_quantize_core(&fsl, seed, config.bit_width, config.num_rounds, ctx)?; - - Ok(build_turboquant(num_rows, core, &ext_dtype)?.into_array()) + let quantized_fsl = + build_quantized_fsl(num_rows, core.all_indices, &core.centroids, core.padded_dim)?; + let padded_vector = wrap_padded_as_vector(quantized_fsl)?; + + let sorf_options = SorfOptions { + seed, + num_rounds: config.num_rounds, + dimension, + element_ptype, + }; + Ok(SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array()) } -/// Export rotation signs as a `FixedSizeListArray` wrapping a 1-bit [`BitPackedArray`]. -/// -/// The rotation matrix's `num_rounds * padded_dim` sign values are exported as 0/1 u8 values in -/// inverse application order, bitpacked to 1 bit per sign, then wrapped in a -/// `FixedSizeListArray` with `list_size = padded_dim` and `len = num_rounds`. -fn bitpack_rotation_signs(rotation: &RotationMatrix) -> VortexResult { - let signs_u8 = rotation.export_inverse_signs_u8(); - let num_rounds = rotation.num_rounds(); - let padded_dim = u32::try_from(rotation.padded_dim()).vortex_expect("padded_dim fits in u32"); - - let mut buf = BufferMut::::with_capacity(signs_u8.len()); - buf.extend_from_slice(&signs_u8); - let prim = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let bitpacked = bitpack_encode(&prim, 1, None)?; - - let fsl = FixedSizeListArray::try_new( - bitpacked.into_array(), - padded_dim, - Validity::NonNullable, - num_rounds, - )?; - Ok(fsl.into_array()) +/// Wrap an `FSL` in a [`Vector`](crate::vector::Vector) extension so it can be +/// passed as the child of [`SorfTransform`], which expects a `Vector` input. +fn wrap_padded_as_vector(fsl: ArrayRef) -> VortexResult { + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl).into_array()) } diff --git a/vortex-tensor/src/encodings/turboquant/compute/mod.rs b/vortex-tensor/src/encodings/turboquant/compute/mod.rs deleted file mode 100644 index eab759ef0b9..00000000000 --- a/vortex-tensor/src/encodings/turboquant/compute/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Compute pushdown implementations for TurboQuant. - -mod ops; -mod slice; -mod take; - -pub(crate) mod rules; - -use num_traits::Float; -use num_traits::FromPrimitive; -use vortex_error::VortexExpect; - -/// Convert an f32 value to a float type `T`. -/// -/// `FromPrimitive::from_f32` is infallible for all Vortex float types: f16 saturates via the -/// inherent `f16::from_f32()`, f32 is identity, f64 is lossless widening. -pub(crate) fn float_from_f32(v: f32) -> T { - FromPrimitive::from_f32(v).vortex_expect("f32-to-float conversion is infallible") -} diff --git a/vortex-tensor/src/encodings/turboquant/compute/ops.rs b/vortex-tensor/src/encodings/turboquant/compute/ops.rs deleted file mode 100644 index 4999816319b..00000000000 --- a/vortex-tensor/src/encodings/turboquant/compute/ops.rs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_array::ArrayView; -use vortex_array::ExecutionCtx; -use vortex_array::arrays::ExtensionArray; -use vortex_array::arrays::slice::SliceReduce; -use vortex_array::scalar::Scalar; -use vortex_array::vtable::OperationsVTable; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; - -use crate::encodings::turboquant::TurboQuant; - -impl OperationsVTable for TurboQuant { - fn scalar_at( - array: ArrayView<'_, TurboQuant>, - index: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult { - // Slice to single row, decompress that one row. - let Some(sliced) = ::slice(array, index..index + 1)? else { - vortex_bail!("slice returned None for index {index}") - }; - let decoded = sliced.execute::(ctx)?; - decoded.scalar_at(0) - } -} diff --git a/vortex-tensor/src/encodings/turboquant/compute/rules.rs b/vortex-tensor/src/encodings/turboquant/compute/rules.rs deleted file mode 100644 index 39919a8c1ec..00000000000 --- a/vortex-tensor/src/encodings/turboquant/compute/rules.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_array::arrays::dict::TakeExecuteAdaptor; -use vortex_array::arrays::slice::SliceReduceAdaptor; -use vortex_array::kernel::ParentKernelSet; -use vortex_array::optimizer::rules::ParentRuleSet; - -use crate::encodings::turboquant::TurboQuant; - -pub(crate) static RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(TurboQuant))]); - -pub(crate) static PARENT_KERNELS: ParentKernelSet = - ParentKernelSet::new(&[ParentKernelSet::lift(&TakeExecuteAdaptor(TurboQuant))]); diff --git a/vortex-tensor/src/encodings/turboquant/compute/slice.rs b/vortex-tensor/src/encodings/turboquant/compute/slice.rs deleted file mode 100644 index c19f604e36a..00000000000 --- a/vortex-tensor/src/encodings/turboquant/compute/slice.rs +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::ops::Range; - -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::IntoArray; -use vortex_array::arrays::slice::SliceReduce; -use vortex_error::VortexResult; - -use crate::encodings::turboquant::TurboQuant; -use crate::encodings::turboquant::TurboQuantArrayExt; - -impl SliceReduce for TurboQuant { - fn slice( - array: ArrayView<'_, TurboQuant>, - range: Range, - ) -> VortexResult> { - let sliced_codes = array.codes().slice(range)?; - - Ok(Some( - TurboQuant::try_new_array( - array.dtype().clone(), - sliced_codes, - array.centroids().clone(), - array.rotation_signs().clone(), - )? - .into_array(), - )) - } -} diff --git a/vortex-tensor/src/encodings/turboquant/compute/take.rs b/vortex-tensor/src/encodings/turboquant/compute/take.rs deleted file mode 100644 index 19a2e65e393..00000000000 --- a/vortex-tensor/src/encodings/turboquant/compute/take.rs +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::dict::TakeExecute; -use vortex_error::VortexResult; - -use crate::encodings::turboquant::TurboQuant; -use crate::encodings::turboquant::TurboQuantArrayExt; - -impl TakeExecute for TurboQuant { - fn take( - array: ArrayView<'_, TurboQuant>, - indices: &ArrayRef, - _ctx: &mut ExecutionCtx, - ) -> VortexResult> { - // FSL children handle per-row take natively. - let taken_codes = array.codes().take(indices.clone())?; - - Ok(Some( - TurboQuant::try_new_array( - array.dtype().clone(), - taken_codes, - array.centroids().clone(), - array.rotation_signs().clone(), - )? - .into_array(), - )) - } -} diff --git a/vortex-tensor/src/encodings/turboquant/metadata.rs b/vortex-tensor/src/encodings/turboquant/metadata.rs deleted file mode 100644 index b4a7a8aef08..00000000000 --- a/vortex-tensor/src/encodings/turboquant/metadata.rs +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Protobuf-backed metadata for TurboQuant encoding. - -use prost::Message; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_err; - -use crate::encodings::turboquant::TurboQuant; - -/// Serialized metadata for TurboQuant arrays. -#[derive(Clone, PartialEq, Message)] -pub(super) struct TurboQuantMetadata { - /// The number of bits per coordinate, which must be <= [`TurboQuant::MAX_BIT_WIDTH`]. - #[prost(uint32, required, tag = "1")] - bit_width: u32, - - /// The number of sign-diagonal + WHT rounds in the structured rotation. - #[prost(uint32, required, tag = "2")] - num_rounds: u32, -} - -impl TurboQuantMetadata { - /// Creates metadata for the given bit width and number of rotation rounds. - pub(super) fn new(bit_width: u8, num_rounds: u8) -> Self { - Self { - bit_width: u32::from(bit_width), - num_rounds: u32::from(num_rounds), - } - } - - /// Returns the validated TurboQuant bit width. - pub(super) fn bit_width(&self) -> VortexResult { - let bit_width = u8::try_from(self.bit_width).map_err(|_| { - vortex_err!( - "TurboQuant bit_width must fit into u8, got {}", - self.bit_width - ) - })?; - vortex_ensure!( - bit_width <= TurboQuant::MAX_BIT_WIDTH, - "bit_width is expected to be between 0 and {}, got {bit_width}", - TurboQuant::MAX_BIT_WIDTH - ); - - Ok(bit_width) - } - - /// Returns the validated number of rotation rounds. - /// - /// Returns 0 for degenerate (empty) arrays, which is validated at a higher level. - pub(super) fn num_rounds(&self) -> VortexResult { - u8::try_from(self.num_rounds).map_err(|_| { - vortex_err!( - "TurboQuant num_rounds must fit into u8, got {}", - self.num_rounds - ) - }) - } -} - -#[cfg(test)] -mod tests { - use prost::Message; - use rstest::rstest; - use vortex_error::VortexResult; - - use super::TurboQuantMetadata; - - #[rstest] - #[case(0, 0)] - #[case(0, 3)] - #[case(3, 1)] - #[case(8, 3)] - #[case(8, 5)] - fn protobuf_metadata_roundtrip( - #[case] bit_width: u8, - #[case] num_rounds: u8, - ) -> VortexResult<()> { - let bytes = TurboQuantMetadata::new(bit_width, num_rounds).encode_to_vec(); - let decoded = TurboQuantMetadata::decode(bytes.as_slice())?; - assert_eq!(decoded.bit_width()?, bit_width); - assert_eq!(decoded.num_rounds()?, num_rounds); - - Ok(()) - } -} diff --git a/vortex-tensor/src/encodings/turboquant/mod.rs b/vortex-tensor/src/encodings/turboquant/mod.rs index ff3259788e9..49e53effd0d 100644 --- a/vortex-tensor/src/encodings/turboquant/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/mod.rs @@ -17,25 +17,34 @@ //! TurboQuant minimizes mean-squared reconstruction error (1-8 bits per coordinate) //! using MSE-optimal scalar quantization on coordinates of a rotated unit vector. //! -//! The `TurboQuantArray` stores only the quantized unit-norm vector data (codes, centroids, -//! rotation signs). Per-vector L2 norms are stored separately in an [`L2Denorm`] ScalarFnArray -//! wrapper. The [`turboquant_encode`] function returns this wrapper: +//! The encoding is decomposed into independently swappable layers: +//! +//! - **Normalization**: [`L2Denorm`] stores per-vector norms and wraps the compressed child. +//! - **Orthogonal transform**: [`SorfTransform`] records the SORF structured orthogonal +//! transform and applies the inverse at decode time. +//! - **Quantization**: `DictArray(codes, centroids)` wrapped in `FixedSizeListArray` stores +//! the per-coordinate codebook indices. +//! +//! The full encoded tree is: //! //! ```text -//! ScalarFnArray(L2Denorm, [TurboQuantArray, norms]) +//! ScalarFnArray(L2Denorm, [ +//! ScalarFnArray(SorfTransform, [FSL(Dict(codes, centroids))]), +//! norms +//! ]) //! ``` //! -//! When executed, the TQ array decompresses to unit-norm vectors, and the [`L2Denorm`] function -//! lazily re-applies the stored norms to reconstruct the original magnitudes. +//! When executed, the tree automatically decompresses: Dict dequantizes codes → SorfTransform +//! inverse-rotates → L2Denorm re-applies norms → original vectors (approximately). //! //! [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm -//! [`turboquant_encode`]: crate::encodings::turboquant::turboquant_encode +//! [`SorfTransform`]: crate::scalar_fns::sorf_transform::SorfTransform //! //! The TurboQuant paper analyzes a full random orthogonal rotation. The current Vortex //! implementation instead uses a fixed 3-round Walsh-Hadamard-based structured transform with -//! random sign diagonals. This is a practical approximation chosen for encode/decode efficiency, -//! and should be understood as an implementation choice rather than the exact construction used in -//! the paper's proofs. +//! random sign diagonals generated by Vortex's frozen local SplitMix64 stream. This is a practical +//! approximation chosen for encode/decode efficiency, and should be understood as an +//! implementation choice rather than the exact construction used in the paper's proofs. //! //! The current encoding is also intentionally MSE-only. It does not yet implement the paper's QJL //! residual correction for unbiased inner-product estimation, and it still uses internal @@ -95,7 +104,6 @@ //! use vortex_array::session::ArraySession; //! use vortex_session::VortexSession; //! use vortex_tensor::encodings::turboquant::{TurboQuantConfig, turboquant_encode_unchecked}; -//! use vortex_tensor::scalar_fns::ApproxOptions; //! use vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm; //! use vortex_tensor::vector::Vector; //! @@ -117,9 +125,7 @@ //! // Normalize, then quantize the normalized child at 2 bits per coordinate. //! let session = VortexSession::empty().with::(); //! let mut ctx = session.create_execution_ctx(); -//! let l2_denorm = normalize_as_l2_denorm( -//! &ApproxOptions::Exact, ext.into_array(), &mut ctx, -//! ).unwrap(); +//! let l2_denorm = normalize_as_l2_denorm(ext.into_array(), &mut ctx).unwrap(); //! let normalized = l2_denorm.child_at(0).clone(); //! //! let normalized_ext = normalized.as_opt::().unwrap(); @@ -133,24 +139,55 @@ //! assert!(tq.nbytes() < 51200); //! ``` -mod array; -pub use array::data::TurboQuantArrayExt; -pub use array::data::TurboQuantData; +pub(crate) mod centroids; +pub(crate) mod compress; -pub(crate) mod compute; +mod scheme; +pub use compress::TurboQuantConfig; +pub use compress::turboquant_encode; +pub use compress::turboquant_encode_unchecked; +pub use scheme::TurboQuantScheme; -mod metadata; +/// Minimum vector dimension for TurboQuant encoding. +/// +/// Note that this is not a theoretical minimum, it is mostly a practical one to limit the total +/// amount of distortion. +pub const MIN_DIMENSION: u32 = 128; -mod vtable; +/// Maximum supported number of bits per quantized coordinate. +pub const MAX_BIT_WIDTH: u8 = 8; -pub use vtable::TurboQuant; -pub use vtable::TurboQuantArray; +/// Maximum supported number of centroids in the scalar quantizer codebook. +pub const MAX_CENTROIDS: usize = 1usize << (MAX_BIT_WIDTH as usize); -mod scheme; -pub use scheme::TurboQuantScheme; -pub use scheme::compress::TurboQuantConfig; -pub use scheme::compress::turboquant_encode; -pub use scheme::compress::turboquant_encode_unchecked; +use vortex_array::dtype::DType; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; + +use crate::vector::AnyVector; +use crate::vector::VectorMatcherMetadata; + +/// Validates that `dtype` is a [`Vector`](crate::vector::Vector) extension type with +/// dimension >= [`MIN_DIMENSION`]. +/// +/// Returns the validated vector metadata on success. +pub fn tq_validate_vector_dtype(dtype: &DType) -> VortexResult { + let vector_metadata = dtype + .as_extension_opt() + .and_then(|ext| ext.metadata_opt::()) + .ok_or_else(|| { + vortex_err!("TurboQuant dtype must be a Vector extension type, got {dtype}") + })?; + + let dimensions = vector_metadata.dimensions(); + vortex_ensure!( + dimensions >= MIN_DIMENSION, + "TurboQuant requires dimension >= {MIN_DIMENSION}, got {dimensions}", + ); + + Ok(vector_metadata) +} #[cfg(test)] mod tests; diff --git a/vortex-tensor/src/encodings/turboquant/scheme/mod.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs similarity index 58% rename from vortex-tensor/src/encodings/turboquant/scheme/mod.rs rename to vortex-tensor/src/encodings/turboquant/scheme.rs index c3beb2a6993..ba9b95dd1e0 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -1,16 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! TurboQuant compression scheme and decompression. +//! TurboQuant compression scheme. //! //! The scheme first normalizes the input via [`normalize_as_l2_denorm`], then encodes the -//! normalized child via [`turboquant_encode`]. The result is: +//! normalized child via [`turboquant_encode_unchecked`]. The result is: //! //! ```text -//! ScalarFnArray(L2Denorm, [TurboQuantArray, norms]) +//! ScalarFnArray(L2Denorm, [ +//! ScalarFnArray( +//! SorfTransform, +//! FSL(Dict(codes, centroids)) +//! ), +//! norms +//! ]) //! ``` //! +//! Decompression is automatic: executing the outer array walks the ScalarFn tree. +//! //! [`normalize_as_l2_denorm`]: crate::scalar_fns::l2_denorm::normalize_as_l2_denorm +//! [`turboquant_encode_unchecked`]: crate::encodings::turboquant::turboquant_encode_unchecked use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -25,22 +34,20 @@ use vortex_compressor::stats::ArrayAndStats; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::encodings::turboquant::TurboQuant; +use crate::encodings::turboquant::MAX_CENTROIDS; use crate::encodings::turboquant::TurboQuantConfig; +use crate::encodings::turboquant::tq_validate_vector_dtype; use crate::encodings::turboquant::turboquant_encode_unchecked; -use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; -pub(super) mod compress; -pub(super) mod decompress; - /// TurboQuant compression scheme for [`Vector`] extension types. /// -/// Applies lossy vector quantization to [`Vector`] extension arrays using the TurboQuant -/// algorithm with MSE-optimal encoding. +/// Applies lossy vector quantization to [`Vector`] extension arrays using the TurboQuant algorithm +/// with MSE-optimal encoding. /// /// Register this scheme with the compressor builder via `with_scheme`: +/// /// ```ignore /// use vortex_btrblocks::BtrBlocksCompressorBuilder; /// use vortex_tensor::encodings::turboquant::TurboQuantScheme; @@ -64,7 +71,7 @@ impl Scheme for TurboQuantScheme { return false; }; - TurboQuant::validate_dtype(ext.dtype()).is_ok() + tq_validate_vector_dtype(ext.dtype()).is_ok() } fn expected_compression_ratio( @@ -76,15 +83,19 @@ impl Scheme for TurboQuantScheme { let dtype = data.array().dtype(); let vector_metadata = - TurboQuant::validate_dtype(dtype).vortex_expect("invalid dtype for TurboQuant"); + tq_validate_vector_dtype(dtype).vortex_expect("invalid dtype for TurboQuant"); let element_ptype = vector_metadata.element_ptype(); - let bit_width: u8 = element_ptype + let element_bit_width: u8 = element_ptype .bit_width() .try_into() .vortex_expect("invalid bit width for TurboQuant"); let dimension = vector_metadata.dimensions(); - CompressionEstimate::Ratio(estimate_compression_ratio(bit_width, dimension, len)) + CompressionEstimate::Ratio(estimate_compression_ratio( + element_bit_width, + dimension, + len, + )) } fn compress( @@ -100,50 +111,50 @@ impl Scheme for TurboQuantScheme { let mut ctx = compressor.execution_ctx(); - // Normalize first: produces L2Denorm(normalized_vectors, norms). - let l2_denorm = - normalize_as_l2_denorm(&ApproxOptions::Exact, ext_array.as_ref().clone(), &mut ctx)?; + // 1. Normalize: produces L2Denorm(normalized_vectors, norms). + let l2_denorm = normalize_as_l2_denorm(ext_array.as_ref().clone(), &mut ctx)?; let normalized = l2_denorm.child_at(0).clone(); let norms = l2_denorm.child_at(1).clone(); let num_rows = l2_denorm.len(); - // Quantize the normalized child. + // 2. Quantize the normalized child: SorfTransform(FSL(Dict)). let normalized_ext = normalized .as_opt::() .vortex_expect("normalized child should be an Extension array"); + let config = TurboQuantConfig::default(); // SAFETY: We just normalized the input via `normalize_as_l2_denorm`, so all rows are // guaranteed to be unit-norm (or zero for originally-null rows). - let tq = unsafe { turboquant_encode_unchecked(normalized_ext, &config, &mut ctx)? }; + let sorf_dict = unsafe { turboquant_encode_unchecked(normalized_ext, &config, &mut ctx)? }; + // 3. Wrap back in L2Denorm: the SorfTransform is the "normalized" child. // SAFETY: TurboQuant is a lossy approximation of the normalized child, so we intentionally // bypass the strict normalized-row validation when reattaching the stored norms. - Ok( - unsafe { L2Denorm::new_array_unchecked(&ApproxOptions::Exact, tq, norms, num_rows) }? - .into_array(), - ) + Ok(unsafe { L2Denorm::new_array_unchecked(sorf_dict, norms, num_rows) }?.into_array()) } } +// TODO(connor): If we ever add scheme vtables with metadata, we would need to pass in the config as +// a parameter here. /// Estimate the compression ratio for TurboQuant MSE encoding with the default config. -fn estimate_compression_ratio(bits_per_element: u8, dimensions: u32, num_vectors: usize) -> f64 { +fn estimate_compression_ratio(element_bit_width: u8, dimensions: u32, num_vectors: usize) -> f64 { let config = TurboQuantConfig::default(); let padded_dim = dimensions.next_power_of_two() as usize; - // Per-vector: MSE codes per padded coordinate, plus one f32 norm. - let compressed_bits_per_vector = 32 // norm is always f32 - + (config.bit_width as usize) * padded_dim; // MSE codes + // Per-vector: MSE codes per padded coordinate, plus one stored norm in the input element + // float width. + let compressed_bits_per_vector = + usize::from(element_bit_width) + usize::from(config.bit_width) * padded_dim; - // Shared overhead: codebook centroids (2^bit_width f32 values) and - // rotation signs (num_rounds * padded_dim bits). + // Shared overhead: codebook centroids (2^bit_width f32 values). + // Note: rotation signs are no longer stored — rotation is deterministic from seed. let num_centroids = 1usize << config.bit_width; - debug_assert!(num_centroids <= TurboQuant::MAX_CENTROIDS); - let overhead_bits = num_centroids * 32 // centroids are always f32 - + config.num_rounds as usize * padded_dim; // rotation signs, 1 bit each + debug_assert!(num_centroids <= MAX_CENTROIDS); + let overhead_bits = num_centroids * 32; // centroids are always f32 let compressed_size_bits = compressed_bits_per_vector * num_vectors + overhead_bits; - let uncompressed_size_bits = bits_per_element as usize * dimensions as usize * num_vectors; + let uncompressed_size_bits = usize::from(element_bit_width) * dimensions as usize * num_vectors; uncompressed_size_bits as f64 / compressed_size_bits as f64 } @@ -155,27 +166,27 @@ mod tests { /// Verify compression ratio for typical embedding dimensions. /// - /// f32 input at 768-d (padded to 1024) with 1000 vectors should give ~4-6x. - /// f32 input at 1024-d (no padding) should give higher ratio since no waste. + /// f32 input at 768-d (padded to 1024) with 1000 vectors should give ~3x. + /// f32 input at 1024-d (no padding) should give ~4x since no padding waste. #[rstest] - #[case::f32_768d(32, 768, 1000, 2.5, 4.0)] + #[case::f32_768d(32, 768, 1000, 2.5, 4.5)] #[case::f32_1024d(32, 1024, 1000, 3.5, 5.0)] - #[case::f32_1536d(32, 1536, 1000, 2.5, 4.0)] + #[case::f32_1536d(32, 1536, 1000, 2.5, 4.5)] #[case::f32_128d(32, 128, 1000, 3.0, 5.0)] - #[case::f64_768d(64, 768, 1000, 5.0, 7.0)] - #[case::f16_768d(16, 768, 1000, 1.2, 2.0)] + #[case::f64_768d(64, 768, 1000, 5.0, 9.0)] + #[case::f16_768d(16, 768, 1000, 1.2, 2.5)] fn compression_ratio_in_expected_range( - #[case] bits_per_element: u8, + #[case] element_bit_width: u8, #[case] dim: u32, #[case] num_vectors: usize, #[case] min_ratio: f64, #[case] max_ratio: f64, ) { - let ratio = estimate_compression_ratio(bits_per_element, dim, num_vectors); + let ratio = estimate_compression_ratio(element_bit_width, dim, num_vectors); assert!( ratio > min_ratio && ratio < max_ratio, "ratio {ratio:.2} not in [{min_ratio}, {max_ratio}] for \ - {bits_per_element}-bit elements, dim={dim}, n={num_vectors}" + {element_bit_width}-bit elements, dim={dim}, n={num_vectors}" ); } @@ -186,14 +197,38 @@ mod tests { #[case(32, 768, 10)] #[case(64, 256, 50)] fn ratio_always_greater_than_one( - #[case] bits_per_element: u8, + #[case] element_bit_width: u8, #[case] dim: u32, #[case] num_vectors: usize, ) { - let ratio = estimate_compression_ratio(bits_per_element, dim, num_vectors); + let ratio = estimate_compression_ratio(element_bit_width, dim, num_vectors); assert!( ratio > 1.0, - "ratio {ratio:.4} <= 1.0 for {bits_per_element}-bit, dim={dim}, n={num_vectors}" + "ratio {ratio:.4} <= 1.0 for {element_bit_width}-bit, dim={dim}, n={num_vectors}" + ); + } + + #[rstest] + #[case(16)] + #[case(32)] + #[case(64)] + fn ratio_accounts_for_norm_storage_width(#[case] element_bit_width: u8) { + let dim = 128u32; + let num_vectors = 1usize; + let padded_dim = dim.next_power_of_two() as usize; + let config = TurboQuantConfig::default(); + let num_centroids = 1usize << config.bit_width; + + let expected_compressed_bits = usize::from(element_bit_width) + + usize::from(config.bit_width) * padded_dim + + num_centroids * 32; + let expected_uncompressed_bits = + usize::from(element_bit_width) * dim as usize * num_vectors; + let expected = expected_uncompressed_bits as f64 / expected_compressed_bits as f64; + + assert_eq!( + estimate_compression_ratio(element_bit_width, dim, num_vectors), + expected ); } diff --git a/vortex-tensor/src/encodings/turboquant/scheme/decompress.rs b/vortex-tensor/src/encodings/turboquant/scheme/decompress.rs deleted file mode 100644 index 2e92ced88e1..00000000000 --- a/vortex-tensor/src/encodings/turboquant/scheme/decompress.rs +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! TurboQuant decoding (dequantization) logic. -//! -//! Decompression produces unit-norm vectors. The original magnitudes are restored externally -//! by the [`L2Denorm`](crate::scalar_fns::l2_denorm::L2Denorm) ScalarFnArray wrapper. - -use num_traits::Float; -use num_traits::FromPrimitive; -use vortex_array::Array; -use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::arrays::ExtensionArray; -use vortex_array::arrays::FixedSizeListArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex_array::dtype::NativePType; -use vortex_array::dtype::Nullability; -use vortex_array::match_each_float_ptype; -use vortex_array::validity::Validity; -use vortex_buffer::BufferMut; -use vortex_error::VortexResult; - -use crate::encodings::turboquant::TurboQuant; -use crate::encodings::turboquant::TurboQuantArrayExt; -use crate::encodings::turboquant::array::rotation::RotationMatrix; -use crate::encodings::turboquant::compute::float_from_f32; -use crate::vector::AnyVector; - -/// Decompress a `TurboQuantArray` into a unit-norm [`Vector`] extension array. -/// -/// The returned array is an [`ExtensionArray`] with the (non-nullable) Vector dtype wrapping a -/// `FixedSizeListArray` of the original vector element type. Each vector has unit L2 norm; the -/// original magnitudes are restored by the [`L2Denorm`](crate::scalar_fns::l2_denorm::L2Denorm) -/// ScalarFnArray wrapper. -/// -/// [`Vector`]: crate::vector::Vector -pub fn execute_decompress( - array: Array, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let dim = array.dimension() as usize; - let padded_dim = array.padded_dim() as usize; - let num_rows = array.len(); - let ext_dtype = array.dtype().as_extension().clone(); - let element_ptype = ext_dtype.metadata::().element_ptype(); - - if num_rows == 0 { - match_each_float_ptype!(element_ptype, |T| { - let elements = PrimitiveArray::empty::(Nullability::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - array.dimension(), - Validity::NonNullable, - 0, - )?; - - return Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()); - }) - } - - // Read stored centroids (always f32). - let centroids_prim = array.centroids().clone().execute::(ctx)?; - let centroids = centroids_prim.as_slice::(); - - // The rotation signs are stored as a FixedSizeListArray wrapping bitpacked u8 values. - // We unwrap to the flat elements, then FastLanes SIMD-unpacks the 1-bit values into u8 0/1. - // These are expanded to u32 XOR masks once (amortized over all rows), enabling branchless - // XOR-based sign application in the per-row structured-rotation hot loop. - let num_rounds = array.num_rounds() as usize; - let signs_fsl = array - .rotation_signs() - .clone() - .execute::(ctx)?; - let signs_prim = signs_fsl - .elements() - .clone() - .execute::(ctx)?; - let rotation = RotationMatrix::from_u8_slice(signs_prim.as_slice::(), dim, num_rounds)?; - - // Unpack codes from FixedSizeListArray -> flat u8 elements. - let codes_fsl = array.codes().clone().execute::(ctx)?; - let codes_prim = codes_fsl - .elements() - .clone() - .execute::(ctx)?; - let indices = codes_prim.as_slice::(); - - // MSE decode: dequantize (f32) -> inverse rotate (f32) -> cast to T. - // The rotation and centroid lookup always happen in f32. The final output is cast to the - // Vector's element type to match the original storage dtype. No norm scaling is applied here; - // that is handled by the external L2Denorm wrapper. - match_each_float_ptype!(element_ptype, |T| { - decompress_typed::(centroids, &rotation, indices, dim, padded_dim, num_rows).and_then( - |elements| { - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - array.dimension(), - Validity::NonNullable, - num_rows, - )?; - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) - }, - ) - }) -} - -/// Typed decompress: dequantizes in f32 and produces unit-norm output as `T`. -fn decompress_typed( - centroids: &[f32], - rotation: &RotationMatrix, - indices: &[u8], - dim: usize, - padded_dim: usize, - num_rows: usize, -) -> VortexResult { - let mut output = BufferMut::::with_capacity(num_rows * dim); - let mut dequantized = vec![0.0f32; padded_dim]; - let mut unrotated = vec![0.0f32; padded_dim]; - - for row in 0..num_rows { - let row_indices = &indices[row * padded_dim..(row + 1) * padded_dim]; - - for idx in 0..padded_dim { - dequantized[idx] = centroids[row_indices[idx] as usize]; - } - - rotation.inverse_rotate(&dequantized, &mut unrotated); - - for idx in 0..dim { - output.push(float_from_f32::(unrotated[idx])); - } - } - - Ok(PrimitiveArray::new::( - output.freeze(), - Validity::NonNullable, - )) -} diff --git a/vortex-tensor/src/encodings/turboquant/tests.rs b/vortex-tensor/src/encodings/turboquant/tests.rs deleted file mode 100644 index 8e633469a3d..00000000000 --- a/vortex-tensor/src/encodings/turboquant/tests.rs +++ /dev/null @@ -1,1260 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::sync::LazyLock; - -use rand::SeedableRng; -use rand::rngs::StdRng; -use rand_distr::Distribution; -use rand_distr::Normal; -use rstest::rstest; -use vortex_array::ArrayRef; -use vortex_array::IntoArray; -use vortex_array::VortexSessionExecute; -use vortex_array::arrays::Extension; -use vortex_array::arrays::ExtensionArray; -use vortex_array::arrays::FixedSizeListArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::ScalarFnVTable; -use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::extension::ExtDType; -use vortex_array::extension::EmptyMetadata; -use vortex_array::session::ArraySession; -use vortex_array::validity::Validity; -use vortex_buffer::BufferMut; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_session::VortexSession; - -use crate::encodings::turboquant::TurboQuant; -use crate::encodings::turboquant::TurboQuantArrayExt; -use crate::encodings::turboquant::TurboQuantConfig; -use crate::encodings::turboquant::array::rotation::RotationMatrix; -use crate::encodings::turboquant::turboquant_encode; -use crate::encodings::turboquant::turboquant_encode_unchecked; -use crate::scalar_fns::ApproxOptions; -use crate::scalar_fns::l2_denorm::L2Denorm; -use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; -use crate::scalar_fns::l2_norm::L2Norm; -use crate::vector::Vector; - -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - -/// Create a FixedSizeListArray of random f32 vectors (i.i.d. standard normal) with the given -/// validity. -fn make_fsl_with_validity( - num_rows: usize, - dim: usize, - seed: u64, - validity: Validity, -) -> FixedSizeListArray { - let mut rng = StdRng::seed_from_u64(seed); - let normal = Normal::new(0.0f32, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(normal.sample(&mut rng)); - } - - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - validity, - num_rows, - ) - .unwrap() -} - -/// Create a non-nullable FixedSizeListArray of random f32 vectors (i.i.d. standard normal). -fn make_fsl(num_rows: usize, dim: usize, seed: u64) -> FixedSizeListArray { - let mut rng = StdRng::seed_from_u64(seed); - let normal = Normal::new(0.0f32, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(normal.sample(&mut rng)); - } - - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - num_rows, - ) - .unwrap() -} - -/// Wrap a `FixedSizeListArray` in a `Vector` extension array. -fn make_vector_ext(fsl: &FixedSizeListArray) -> ExtensionArray { - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) - .unwrap() - .erased(); - ExtensionArray::new(ext_dtype, fsl.clone().into_array()) -} - -/// Full encode pipeline: normalize, then TQ-encode, then wrap in L2Denorm. -/// -/// This mirrors what `TurboQuantScheme::compress()` does: normalize via `normalize_as_l2_denorm`, -/// then quantize the normalized child via `turboquant_encode_unchecked`, then reassemble. -fn normalize_and_encode( - ext: &ExtensionArray, - config: &TurboQuantConfig, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - let l2_denorm = normalize_as_l2_denorm(&ApproxOptions::Exact, ext.as_ref().clone(), ctx)?; - let normalized = l2_denorm.child_at(0).clone(); - let norms = l2_denorm.child_at(1).clone(); - let num_rows = l2_denorm.len(); - - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - // SAFETY: We just normalized the input via `normalize_as_l2_denorm`. - let tq = unsafe { turboquant_encode_unchecked(normalized_ext, config, ctx)? }; - - Ok( - unsafe { L2Denorm::new_array_unchecked(&ApproxOptions::Exact, tq, norms, num_rows) }? - .into_array(), - ) -} - -/// Unwrap an L2Denorm ScalarFnArray into its TQ child and norms child. -fn unwrap_l2denorm(encoded: &ArrayRef) -> (ArrayRef, ArrayRef) { - let sfn = encoded - .as_opt::() - .expect("expected ScalarFnArray"); - let tq_child = sfn.child_at(0).clone(); - let norms_child = sfn.child_at(1).clone(); - (tq_child, norms_child) -} - -fn theoretical_mse_bound(bit_width: u8) -> f32 { - let sqrt3_pi_over_2 = (3.0f32).sqrt() * std::f32::consts::PI / 2.0; - sqrt3_pi_over_2 / (4.0f32).powi(bit_width as i32) -} - -fn per_vector_normalized_mse( - original: &[f32], - reconstructed: &[f32], - dim: usize, - num_rows: usize, -) -> f32 { - let mut total = 0.0f32; - for row in 0..num_rows { - let orig = &original[row * dim..(row + 1) * dim]; - let recon = &reconstructed[row * dim..(row + 1) * dim]; - let norm_sq: f32 = orig.iter().map(|&v| v * v).sum(); - if norm_sq < 1e-10 { - continue; - } - let err_sq: f32 = orig - .iter() - .zip(recon.iter()) - .map(|(&a, &b)| (a - b) * (a - b)) - .sum(); - total += err_sq / norm_sq; - } - total / num_rows as f32 -} - -/// Normalize, encode, and decode, returning (original, decoded) flat f32 slices. -fn encode_decode( - fsl: &FixedSizeListArray, - config: &TurboQuantConfig, -) -> VortexResult<(Vec, Vec)> { - let mut ctx = SESSION.create_execution_ctx(); - let original: Vec = { - let prim = fsl.elements().clone().execute::(&mut ctx)?; - prim.as_slice::().to_vec() - }; - let ext = make_vector_ext(fsl); - let encoded = normalize_and_encode(&ext, config, &mut ctx)?; - let decoded_ext = encoded.execute::(&mut ctx)?; - let decoded_fsl = decoded_ext - .storage_array() - .clone() - .execute::(&mut ctx)?; - let decoded_elements: Vec = { - let prim = decoded_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - prim.as_slice::().to_vec() - }; - Ok((original, decoded_elements)) -} - -fn empty_turboquant_parts( - dim: u32, -) -> VortexResult<(vortex_array::dtype::DType, ArrayRef, ArrayRef, ArrayRef)> { - let fsl = make_fsl(0, dim as usize, 42); - let ext = make_vector_ext(&fsl); - - let codes = FixedSizeListArray::try_new( - PrimitiveArray::empty::(Nullability::NonNullable).into_array(), - dim, - Validity::NonNullable, - 0, - )? - .into_array(); - let centroids = PrimitiveArray::empty::(Nullability::NonNullable).into_array(); - let rotation_signs = FixedSizeListArray::try_new( - PrimitiveArray::empty::(Nullability::NonNullable).into_array(), - dim, - Validity::NonNullable, - 0, - )? - .into_array(); - - // TQ dtype is non-nullable. - Ok(( - ext.dtype().as_nonnullable(), - codes, - centroids, - rotation_signs, - )) -} - -fn normalized_child( - ext: &ExtensionArray, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - Ok( - normalize_as_l2_denorm(&ApproxOptions::Exact, ext.as_ref().clone(), ctx)? - .child_at(0) - .clone(), - ) -} - -// ----------------------------------------------------------------------- -// Roundtrip tests -// ----------------------------------------------------------------------- - -#[rstest] -#[case(128, 1)] -#[case(128, 2)] -#[case(128, 3)] -#[case(128, 4)] -#[case(128, 6)] -#[case(128, 8)] -#[case(256, 2)] -fn roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { - let fsl = make_fsl(10, dim, 42); - let config = TurboQuantConfig { - bit_width, - seed: Some(123), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - assert_eq!(decoded.len(), original.len()); - Ok(()) -} - -#[test] -fn empty_try_new_rejects_invalid_centroids_dtype() -> VortexResult<()> { - let (dtype, codes, _centroids, rotation_signs) = empty_turboquant_parts(128)?; - let wrong_centroids = PrimitiveArray::empty::(Nullability::NonNullable).into_array(); - - let err = TurboQuant::try_new_array(dtype, codes, wrong_centroids, rotation_signs).unwrap_err(); - - assert!( - err.to_string() - .contains("centroids dtype must be non-nullable f32") - ); - Ok(()) -} - -#[test] -fn empty_try_new_rejects_invalid_rotation_signs_dtype() -> VortexResult<()> { - let (dtype, codes, centroids, _rotation_signs) = empty_turboquant_parts(128)?; - let wrong_rotation_signs = PrimitiveArray::empty::(Nullability::NonNullable).into_array(); - - let err = TurboQuant::try_new_array(dtype, codes, centroids, wrong_rotation_signs).unwrap_err(); - - assert!( - err.to_string() - .contains("rotation_signs dtype does not match") - ); - Ok(()) -} - -// ----------------------------------------------------------------------- -// MSE quality tests -// ----------------------------------------------------------------------- - -#[rstest] -#[case(128, 1)] -#[case(128, 2)] -#[case(128, 3)] -#[case(128, 4)] -#[case(256, 2)] -#[case(256, 4)] -fn mse_within_theoretical_bound(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { - let num_rows = 200; - let fsl = make_fsl(num_rows, dim, 42); - let config = TurboQuantConfig { - bit_width, - seed: Some(123), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - - let normalized_mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); - let bound = theoretical_mse_bound(bit_width); - - assert!( - normalized_mse < bound, - "Normalized MSE {normalized_mse:.6} exceeds bound {bound:.6} \ - for dim={dim}, bits={bit_width}", - ); - Ok(()) -} - -#[rstest] -#[case(128, 6)] -#[case(128, 8)] -#[case(256, 6)] -#[case(256, 8)] -fn high_bitwidth_mse_is_small(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { - let num_rows = 200; - let fsl = make_fsl(num_rows, dim, 42); - - let config_4bit = TurboQuantConfig { - bit_width: 4, - seed: Some(123), - num_rounds: 3, - }; - let (original_4, decoded_4) = encode_decode(&fsl, &config_4bit)?; - let mse_4bit = per_vector_normalized_mse(&original_4, &decoded_4, dim, num_rows); - - let config = TurboQuantConfig { - bit_width, - seed: Some(123), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - let mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); - - assert!( - mse < mse_4bit, - "{bit_width}-bit MSE ({mse:.6}) should be < 4-bit MSE ({mse_4bit:.6})" - ); - assert!(mse < 0.01, "{bit_width}-bit MSE ({mse:.6}) should be < 1%"); - Ok(()) -} - -#[test] -fn mse_decreases_with_bits() -> VortexResult<()> { - let dim = 128; - let num_rows = 50; - let fsl = make_fsl(num_rows, dim, 99); - - let mut prev_mse = f32::MAX; - for bit_width in 1..=8u8 { - let config = TurboQuantConfig { - bit_width, - seed: Some(123), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - let mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); - assert!( - mse <= prev_mse * 1.01, - "MSE should decrease: {bit_width}-bit={mse:.6} > prev={prev_mse:.6}" - ); - prev_mse = mse; - } - Ok(()) -} - -// ----------------------------------------------------------------------- -// Edge cases -// ----------------------------------------------------------------------- - -#[rstest] -#[case(0)] -#[case(1)] -fn roundtrip_edge_cases(#[case] num_rows: usize) -> VortexResult<()> { - let fsl = make_fsl(num_rows, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 2, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let decoded = encoded.execute::(&mut ctx)?; - assert_eq!(decoded.len(), num_rows); - Ok(()) -} - -#[rstest] -#[case(1)] -#[case(64)] -#[case(127)] -fn rejects_dimension_below_128(#[case] dim: usize) { - let fsl = make_fsl_small(dim); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 2, - seed: Some(0), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - assert!(turboquant_encode(ext.as_view(), &config, &mut ctx).is_err()); -} - -#[test] -fn checked_encode_accepts_normalized_f16_input() -> VortexResult<()> { - let num_rows = 10; - let dim = 128; - let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f32, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(half::f16::from_f32(normal.sample(&mut rng))); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - num_rows, - )?; - - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(42), - num_rounds: 3, - }; - - let mut ctx = SESSION.create_execution_ctx(); - let normalized = normalized_child(&ext, &mut ctx)?; - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - - let encoded = turboquant_encode(normalized_ext, &config, &mut ctx)?; - assert_eq!(encoded.len(), num_rows); - Ok(()) -} - -fn make_fsl_small(dim: usize) -> FixedSizeListArray { - let mut buf = BufferMut::::with_capacity(dim); - for i in 0..dim { - buf.push(i as f32 + 1.0); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - 1, - ) - .unwrap() -} - -/// Verify that all-zero vectors roundtrip correctly (norm == 0 branch). -#[test] -fn all_zero_vectors_roundtrip() -> VortexResult<()> { - let num_rows = 10; - let dim = 128; - let buf = BufferMut::::full(0.0f32, num_rows * dim); - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - num_rows, - )?; - - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(42), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - // All-zero vectors should decode to all-zero (norm=0 -> 0 * anything = 0). - for (i, (&o, &d)) in original.iter().zip(decoded.iter()).enumerate() { - assert_eq!(o, 0.0, "original[{i}] not zero"); - assert_eq!(d, 0.0, "decoded[{i}] not zero for all-zero input"); - } - Ok(()) -} - -/// Verify that f64 input is accepted and encoded (converted to f32 internally). -#[test] -fn f64_input_encodes_successfully() -> VortexResult<()> { - let num_rows = 10; - let dim = 128; - let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f64, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(normal.sample(&mut rng)); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - num_rows, - )?; - - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(42), - num_rounds: 3, - }; - // Verify encoding succeeds with f64 input (f64->f32 conversion). - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, norms_child) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - assert_eq!(norms_child.len(), num_rows); - assert_eq!(tq.dimension() as usize, dim); - Ok(()) -} - -/// Verify that f16 input is accepted and encoded (upcast to f32 internally). -#[test] -fn f16_input_encodes_successfully() -> VortexResult<()> { - let num_rows = 10; - let dim = 128; - let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f32, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(half::f16::from_f32(normal.sample(&mut rng))); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - num_rows, - )?; - - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(42), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, norms_child) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - assert_eq!(norms_child.len(), num_rows); - assert_eq!(tq.dimension() as usize, dim); - - // Verify roundtrip: decode and check reconstruction is reasonable. - let decoded_ext = encoded.execute::(&mut ctx)?; - let decoded_fsl = decoded_ext - .storage_array() - .clone() - .execute::(&mut ctx)?; - assert_eq!(decoded_fsl.len(), num_rows); - Ok(()) -} - -// ----------------------------------------------------------------------- -// Verification tests for stored metadata -// ----------------------------------------------------------------------- - -/// Verify that the centroids stored in the array match what `get_centroids()` computes. -#[test] -fn stored_centroids_match_computed() -> VortexResult<()> { - let fsl = make_fsl(10, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, _norms) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - - let mut ctx = SESSION.create_execution_ctx(); - let stored_centroids_prim = tq.centroids().clone().execute::(&mut ctx)?; - let stored = stored_centroids_prim.as_slice::(); - - let padded_dim = tq.padded_dim(); - let computed = crate::encodings::turboquant::array::centroids::get_centroids(padded_dim, 3)?; - - assert_eq!(stored.len(), computed.len()); - for i in 0..stored.len() { - assert_eq!(stored[i], computed[i], "Centroid mismatch at {i}"); - } - Ok(()) -} - -/// Verify that stored rotation signs produce identical decode to seed-based decode. -#[test] -fn stored_rotation_signs_produce_correct_decode() -> VortexResult<()> { - let fsl = make_fsl(20, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 4, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, _norms) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - - // Decode via the full L2Denorm path (TQ decompress + norm scaling). - let mut ctx = SESSION.create_execution_ctx(); - let decoded_ext = encoded.execute::(&mut ctx)?; - let decoded_fsl = decoded_ext - .storage_array() - .clone() - .execute::(&mut ctx)?; - let decoded = decoded_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - let decoded_slice = decoded.as_slice::(); - - // Verify stored signs match seed-derived signs. - let rot_from_seed = RotationMatrix::try_new(123, 128, 4)?; - let expected_u8 = rot_from_seed.export_inverse_signs_u8(); - let stored_signs_fsl = tq - .rotation_signs() - .clone() - .execute::(&mut ctx)?; - let stored_signs = stored_signs_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - let stored_u8 = stored_signs.as_slice::(); - - assert_eq!(expected_u8.len(), stored_u8.len()); - for i in 0..expected_u8.len() { - assert_eq!(expected_u8[i], stored_u8[i], "Sign mismatch at index {i}"); - } - - // Also verify decode output is non-empty and has expected size. - assert_eq!(decoded_slice.len(), 20 * 128); - Ok(()) -} - -// ----------------------------------------------------------------------- -// Compute pushdown tests -// ----------------------------------------------------------------------- - -#[test] -fn slice_preserves_data() -> VortexResult<()> { - let fsl = make_fsl(20, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 4, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - // Full decompress then slice. - let mut ctx = SESSION.create_execution_ctx(); - let full_decoded = encoded.clone().execute::(&mut ctx)?; - let full_fsl = full_decoded - .storage_array() - .clone() - .execute::(&mut ctx)?; - let expected = full_fsl.slice(5..10)?; - let expected_fsl = expected.execute::(&mut ctx)?; - let expected_elements = expected_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - - // Slice then decompress. - let sliced = encoded.slice(5..10)?; - let sliced_decoded = sliced.execute::(&mut ctx)?; - let sliced_fsl = sliced_decoded - .storage_array() - .clone() - .execute::(&mut ctx)?; - let actual_elements = sliced_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - - assert_eq!( - expected_elements.as_slice::(), - actual_elements.as_slice::() - ); - Ok(()) -} - -#[test] -fn scalar_at_matches_decompress() -> VortexResult<()> { - let fsl = make_fsl(10, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 2, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - let full_decoded = encoded.clone().execute::(&mut ctx)?; - - for i in [0, 1, 5, 9] { - let expected = full_decoded.scalar_at(i)?; - let actual = encoded.scalar_at(i)?; - assert_eq!(expected, actual, "scalar_at mismatch at index {i}"); - } - Ok(()) -} - -#[test] -fn l2_norm_readthrough() -> VortexResult<()> { - let fsl = make_fsl(10, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 5, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (_tq_child, norms_child) = unwrap_l2denorm(&encoded); - - // Stored norms should match the actual L2 norms of the input. - let norms_prim = norms_child.execute::(&mut ctx)?; - let stored_norms = norms_prim.as_slice::(); - - let input_prim = fsl.elements().clone().execute::(&mut ctx)?; - let input_f32 = input_prim.as_slice::(); - for row in 0..10 { - let vec = &input_f32[row * 128..(row + 1) * 128]; - let actual_norm: f32 = vec.iter().map(|&v| v * v).sum::().sqrt(); - assert!( - (stored_norms[row] - actual_norm).abs() < 1e-5, - "norm mismatch at row {row}: stored={}, actual={}", - stored_norms[row], - actual_norm - ); - } - Ok(()) -} - -#[test] -fn cosine_similarity_quantized_accuracy() -> VortexResult<()> { - let fsl = make_fsl(20, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 4, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, norms_child) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - - // Compute exact cosine similarity from original data. - let input_prim = fsl.elements().clone().execute::(&mut ctx)?; - let input_f32 = input_prim.as_slice::(); - - // Read quantized codes, norms, and centroids for approximate computation. - let mut ctx = SESSION.create_execution_ctx(); - let pd = tq.padded_dim() as usize; - let norms_prim = norms_child.execute::(&mut ctx)?; - let norms = norms_prim.as_slice::(); - let codes_fsl = tq.codes().clone().execute::(&mut ctx)?; - let codes_prim = codes_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - let all_codes = codes_prim.as_slice::(); - let centroids_prim = tq.centroids().clone().execute::(&mut ctx)?; - let centroid_vals = centroids_prim.as_slice::(); - - for (row_a, row_b) in [(0, 1), (5, 10), (0, 19)] { - let vec_a = &input_f32[row_a * 128..(row_a + 1) * 128]; - let vec_b = &input_f32[row_b * 128..(row_b + 1) * 128]; - - let dot: f32 = vec_a.iter().zip(vec_b.iter()).map(|(&x, &y)| x * y).sum(); - let norm_a: f32 = vec_a.iter().map(|&v| v * v).sum::().sqrt(); - let norm_b: f32 = vec_b.iter().map(|&v| v * v).sum::().sqrt(); - let exact_cos = dot / (norm_a * norm_b); - - // Approximate cosine similarity in quantized domain. - let approx_cos = if norms[row_a] == 0.0 || norms[row_b] == 0.0 { - 0.0 - } else { - let codes_a = &all_codes[row_a * pd..(row_a + 1) * pd]; - let codes_b = &all_codes[row_b * pd..(row_b + 1) * pd]; - codes_a - .iter() - .zip(codes_b.iter()) - .map(|(&ca, &cb)| centroid_vals[ca as usize] * centroid_vals[cb as usize]) - .sum::() - }; - - // At 4-bit, the theoretical MSE bound per coordinate is ~0.0106 (Theorem 1). For cosine - // similarity (bounded [-1, 1]), the error is bounded roughly by 2*sqrt(MSE) ~ 0.2. We use - // 0.15 as a tighter empirical bound. - let error = (exact_cos - approx_cos).abs(); - assert!( - error < 0.15, - "cosine similarity error too large for ({row_a}, {row_b}): \ - exact={exact_cos:.4}, approx={approx_cos:.4}, error={error:.4}" - ); - } - Ok(()) -} - -/// Verify approximate dot product in the quantized domain. -/// -/// NOTE: The MSE quantizer (TurboQuant_mse) has inherent **multiplicative bias** for inner -/// products — the quantized dot product systematically over- or under-estimates the true value. -/// This is a fundamental property: the paper's `TurboQuant_prod` variant adds QJL specifically -/// to debias inner products, but we only implement the MSE-only variant. -/// -/// Even at 8-bit (near-lossless reconstruction, MSE ~4e-5), the quantized-domain dot product -/// can have ~10-15% relative error due to this bias. This tolerance is therefore intentionally -/// loose — we're testing that the approximation is in the right ballpark, not that it's precise. -/// -/// TODO(connor): Revisit these tolerances when we have TurboQuant_prod (QJL debiasing). -#[test] -fn dot_product_quantized_accuracy() -> VortexResult<()> { - let fsl = make_fsl(20, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 8, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, norms_child) = unwrap_l2denorm(&encoded); - let tq = tq_child.as_opt::().unwrap(); - - let input_prim = fsl.elements().clone().execute::(&mut ctx)?; - let input_f32 = input_prim.as_slice::(); - - let mut ctx = SESSION.create_execution_ctx(); - let pd = tq.padded_dim() as usize; - let norms_prim = norms_child.execute::(&mut ctx)?; - let norms = norms_prim.as_slice::(); - let codes_fsl = tq.codes().clone().execute::(&mut ctx)?; - let codes_prim = codes_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - let all_codes = codes_prim.as_slice::(); - let centroids_prim = tq.centroids().clone().execute::(&mut ctx)?; - let centroid_vals = centroids_prim.as_slice::(); - - for (row_a, row_b) in [(0, 1), (5, 10), (0, 19)] { - let vec_a = &input_f32[row_a * 128..(row_a + 1) * 128]; - let vec_b = &input_f32[row_b * 128..(row_b + 1) * 128]; - - let exact_dot: f32 = vec_a.iter().zip(vec_b.iter()).map(|(&x, &y)| x * y).sum(); - - let codes_a = &all_codes[row_a * pd..(row_a + 1) * pd]; - let codes_b = &all_codes[row_b * pd..(row_b + 1) * pd]; - let unit_dot: f32 = codes_a - .iter() - .zip(codes_b.iter()) - .map(|(&ca, &cb)| centroid_vals[ca as usize] * centroid_vals[cb as usize]) - .sum(); - let approx_dot = norms[row_a] * norms[row_b] * unit_dot; - - // See doc comment above: 15% relative error is expected due to MSE quantizer bias. - let scale = exact_dot.abs().max(1.0); - let rel_error = (exact_dot - approx_dot).abs() / scale; - assert!( - rel_error < 0.15, - "dot product error too large for ({row_a}, {row_b}): \ - exact={exact_dot:.4}, approx={approx_dot:.4}, rel_error={rel_error:.4}" - ); - } - Ok(()) -} - -/// Roundtrip at large embedding dimensions to validate padding and SRHT at common sizes. -/// -/// NOTE: The theoretical MSE bound (Theorem 1) is proved for Haar-distributed random orthogonal -/// matrices, not SRHT. The SRHT is a practical O(d log d) approximation that doesn't exactly -/// satisfy the Haar assumption, so empirical MSE can slightly exceed the theoretical bound. We -/// use a 2x multiplier to account for this gap. -/// -/// The 1024-d case uses 5-bit instead of 4-bit because at 4-bit the SRHT approximation error -/// at d=1024 pushes MSE ~20% above the 1x theoretical bound (0.0127 vs bound 0.0106). -/// -/// TODO(connor): Revisit after Stage 2 block decomposition — at d=768 with block_size=256, -/// the per-block SRHT will be lower-dimensional and may have different error characteristics. -#[rstest] -#[case(768, 4)] -#[case(1024, 5)] -fn large_dimension_roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { - let num_rows = 10; - let fsl = make_fsl(num_rows, dim, 42); - let config = TurboQuantConfig { - bit_width, - seed: Some(123), - num_rounds: 3, - }; - let (original, decoded) = encode_decode(&fsl, &config)?; - assert_eq!(decoded.len(), original.len()); - - let normalized_mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); - // 2x slack for the SRHT-vs-Haar gap (see doc comment above). - let bound = 2.0 * theoretical_mse_bound(bit_width); - assert!( - normalized_mse < bound, - "Normalized MSE {normalized_mse:.6} exceeds 2x bound {bound:.6} for dim={dim}, bits={bit_width}", - ); - Ok(()) -} - -/// Verify that the encoded array's dtype is a Vector extension type. -#[test] -fn encoded_dtype_is_vector_extension() -> VortexResult<()> { - let fsl = make_fsl(10, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 2, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - // The encoded TurboQuant array should claim a Vector extension dtype. - assert!( - encoded.dtype().is_extension(), - "TurboQuant dtype should be an extension type, got {}", - encoded.dtype() - ); - assert!( - encoded.dtype().as_extension().is::(), - "TurboQuant dtype should be a Vector extension type" - ); - Ok(()) -} - -// ----------------------------------------------------------------------- -// Nullable vector tests -// ----------------------------------------------------------------------- - -/// Encode a nullable Vector array and verify roundtrip preserves validity and non-null values. -#[test] -fn nullable_vectors_roundtrip() -> VortexResult<()> { - // Rows 2, 5, 7 are null. - let validity = Validity::from_iter([ - true, true, false, true, true, false, true, false, true, true, - ]); - let fsl = make_fsl_with_validity(10, 128, 42, validity); - let ext = make_vector_ext(&fsl); - - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 4, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - assert_eq!(encoded.len(), 10); - assert!(encoded.dtype().is_nullable()); - - // Check validity of the encoded array. - let encoded_validity = encoded.validity()?; - for i in 0..10 { - let expected = ![2, 5, 7].contains(&i); - assert_eq!( - encoded_validity.is_valid(i)?, - expected, - "validity mismatch at row {i}" - ); - } - - // Decode and verify non-null rows have correct data. - let decoded_ext = encoded.execute::(&mut ctx)?; - assert_eq!(decoded_ext.len(), 10); - - let decoded_fsl = decoded_ext - .storage_array() - .clone() - .execute::(&mut ctx)?; - let decoded_prim = decoded_fsl - .elements() - .clone() - .execute::(&mut ctx)?; - let decoded_f32 = decoded_prim.as_slice::(); - - // Original f32 elements for non-null row comparison. - let orig_prim = fsl.elements().clone().execute::(&mut ctx)?; - let orig_f32 = orig_prim.as_slice::(); - - // Non-null rows should have reasonable reconstruction (within MSE bounds). - for row in [0, 1, 3, 4, 6, 8, 9] { - let orig_vec = &orig_f32[row * 128..(row + 1) * 128]; - let dec_vec = &decoded_f32[row * 128..(row + 1) * 128]; - let norm_sq: f32 = orig_vec.iter().map(|&v| v * v).sum(); - let err_sq: f32 = orig_vec - .iter() - .zip(dec_vec.iter()) - .map(|(&a, &b)| (a - b) * (a - b)) - .sum(); - // 3-bit normalized MSE should be well under the theoretical bound. - assert!( - err_sq / norm_sq < 0.1, - "non-null row {row} has excessive reconstruction error" - ); - } - Ok(()) -} - -/// Verify that norms carry the validity: null vectors have null norms. -#[test] -fn nullable_norms_match_validity() -> VortexResult<()> { - let validity = Validity::from_iter([true, false, true, false, true]); - let fsl = make_fsl_with_validity(5, 128, 42, validity); - let ext = make_vector_ext(&fsl); - - let config = TurboQuantConfig { - bit_width: 2, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (_tq_child, norms_child) = unwrap_l2denorm(&encoded); - - let norms_validity = norms_child.validity()?; - for i in 0..5 { - let expected = i % 2 == 0; // rows 0, 2, 4 are valid - assert_eq!( - norms_validity.is_valid(i)?, - expected, - "norms validity mismatch at row {i}" - ); - } - Ok(()) -} - -/// Verify that L2Norm readthrough works correctly on nullable TurboQuant arrays. -#[test] -fn nullable_l2_norm_readthrough() -> VortexResult<()> { - let validity = Validity::from_iter([true, false, true, false, true]); - let fsl = make_fsl_with_validity(5, 128, 42, validity); - let ext = make_vector_ext(&fsl); - - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - // Compute L2Norm on the encoded array. - let norm_sfn = L2Norm::try_new_array(&ApproxOptions::Exact, encoded, 5)?; - let norms: PrimitiveArray = norm_sfn.into_array().execute(&mut ctx)?; - - // Null rows should have null norms, valid rows should have correct norms. - let orig_prim = fsl.elements().clone().execute::(&mut ctx)?; - let orig_f32 = orig_prim.as_slice::(); - for row in 0..5 { - if row % 2 == 0 { - assert!(norms.is_valid(row)?, "row {row} should be valid"); - let expected: f32 = orig_f32[row * 128..(row + 1) * 128] - .iter() - .map(|&v| v * v) - .sum::() - .sqrt(); - let actual = norms.as_slice::()[row]; - assert!( - (actual - expected).abs() < 1e-5, - "norm mismatch at valid row {row}: actual={actual}, expected={expected}" - ); - } else { - assert!(!norms.is_valid(row)?, "row {row} should be null"); - } - } - Ok(()) -} - -/// Verify that slicing a nullable TurboQuant array preserves validity. -#[test] -fn nullable_slice_preserves_validity() -> VortexResult<()> { - // Rows 2, 5, 7 are null. - let validity = Validity::from_iter([ - true, true, false, true, true, false, true, false, true, true, - ]); - let fsl = make_fsl_with_validity(10, 128, 42, validity); - let ext = make_vector_ext(&fsl); - - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 2, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - - // Slice rows 1..6 -> [true, false, true, true, false]. - let sliced = encoded.slice(1..6)?; - assert_eq!(sliced.len(), 5); - - let sliced_validity = sliced.validity()?; - let expected = [true, false, true, true, false]; - for (i, &exp) in expected.iter().enumerate() { - assert_eq!( - sliced_validity.is_valid(i)?, - exp, - "sliced validity mismatch at index {i}" - ); - } - Ok(()) -} - -// ----------------------------------------------------------------------- -// Serde roundtrip tests -// ----------------------------------------------------------------------- - -/// Verify that a TurboQuant array (extracted from the L2Denorm wrapper) survives -/// serialize/deserialize. -/// -/// TODO(connor): ScalarFnArray cannot be serialized yet, so we test the TQ child directly. -#[test] -fn serde_roundtrip() -> VortexResult<()> { - use vortex_array::ArrayContext; - use vortex_array::ArrayEq; - use vortex_array::Precision; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_array::session::ArraySessionExt; - use vortex_buffer::ByteBufferMut; - use vortex_fastlanes::BitPacked; - use vortex_session::registry::ReadContext; - - let fsl = make_fsl(20, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(123), - num_rounds: 5, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - let (tq_child, _norms) = unwrap_l2denorm(&encoded); - - let dtype = tq_child.dtype().clone(); - let len = tq_child.len(); - - // Serialize the TQ child. - let array_ctx = ArrayContext::empty(); - let serde_session = VortexSession::empty().with::(); - serde_session.arrays().register(TurboQuant); - let serialized = - tq_child.serialize(&array_ctx, &serde_session, &SerializeOptions::default())?; - - let mut concat = ByteBufferMut::empty(); - for buf in serialized { - concat.extend_from_slice(buf.as_ref()); - } - - // Deserialize. The session needs TurboQuant and BitPacked (for rotation signs) registered. - serde_session.arrays().register(BitPacked); - - let parts = SerializedArray::try_from(concat.freeze())?; - let decoded = parts.decode( - &dtype, - len, - &ReadContext::new(array_ctx.to_ids()), - &serde_session, - )?; - - assert!( - decoded.array_eq(&tq_child, Precision::Value), - "serde roundtrip did not preserve array equality" - ); - Ok(()) -} - -/// Verify that a degenerate (empty) TurboQuant array survives serialize/deserialize. -#[test] -fn serde_roundtrip_empty() -> VortexResult<()> { - use vortex_array::ArrayContext; - use vortex_array::ArrayEq; - use vortex_array::Precision; - use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; - use vortex_array::session::ArraySessionExt; - use vortex_buffer::ByteBufferMut; - use vortex_fastlanes::BitPacked; - use vortex_session::registry::ReadContext; - - let fsl = make_fsl(0, 128, 42); - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 2, - seed: Some(123), - num_rounds: 3, - }; - let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; - assert_eq!(encoded.len(), 0); - let (tq_child, _norms) = unwrap_l2denorm(&encoded); - - let dtype = tq_child.dtype().clone(); - let len = tq_child.len(); - - let serde_session = VortexSession::empty().with::(); - serde_session.arrays().register(TurboQuant); - serde_session.arrays().register(BitPacked); - - let array_ctx = ArrayContext::empty(); - let serialized = - tq_child.serialize(&array_ctx, &serde_session, &SerializeOptions::default())?; - - let mut concat = ByteBufferMut::empty(); - for buf in serialized { - concat.extend_from_slice(buf.as_ref()); - } - - let parts = SerializedArray::try_from(concat.freeze())?; - let decoded = parts.decode( - &dtype, - len, - &ReadContext::new(array_ctx.to_ids()), - &serde_session, - )?; - - assert!( - decoded.array_eq(&tq_child, Precision::Value), - "serde roundtrip did not preserve array equality" - ); - Ok(()) -} diff --git a/vortex-tensor/src/encodings/turboquant/tests/compute.rs b/vortex-tensor/src/encodings/turboquant/tests/compute.rs new file mode 100644 index 00000000000..ac0389048f4 --- /dev/null +++ b/vortex-tensor/src/encodings/turboquant/tests/compute.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_error::VortexResult; + +use super::*; +use crate::scalar_fns::cosine_similarity::CosineSimilarity; +use crate::scalar_fns::l2_norm::L2Norm; + +fn execute_l2_norm( + input: ArrayRef, + len: usize, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + L2Norm::try_new_array(input, len)?.into_array().execute(ctx) +} + +fn execute_cosine_similarity( + lhs: ArrayRef, + rhs: ArrayRef, + len: usize, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + CosineSimilarity::try_new_array(lhs, rhs, len)? + .into_array() + .execute(ctx) +} + +#[test] +fn slice_preserves_data() -> VortexResult<()> { + let fsl = make_fsl(20, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 4, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + // Full decompress then slice. + let mut ctx = SESSION.create_execution_ctx(); + let full_decoded = encoded.clone().execute::(&mut ctx)?; + let full_fsl = full_decoded + .storage_array() + .clone() + .execute::(&mut ctx)?; + let expected = full_fsl.slice(5..10)?; + let expected_fsl = expected.execute::(&mut ctx)?; + let expected_elements = expected_fsl + .elements() + .clone() + .execute::(&mut ctx)?; + + // Slice then decompress. + let sliced = encoded.slice(5..10)?; + let sliced_decoded = sliced.execute::(&mut ctx)?; + let sliced_fsl = sliced_decoded + .storage_array() + .clone() + .execute::(&mut ctx)?; + let actual_elements = sliced_fsl + .elements() + .clone() + .execute::(&mut ctx)?; + + assert_eq!( + expected_elements.as_slice::(), + actual_elements.as_slice::() + ); + Ok(()) +} + +#[test] +fn scalar_at_matches_decompress() -> VortexResult<()> { + let fsl = make_fsl(10, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 2, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let full_decoded = encoded.clone().execute::(&mut ctx)?; + + for i in [0, 1, 5, 9] { + let expected = full_decoded.scalar_at(i)?; + let actual = encoded.scalar_at(i)?; + assert_eq!(expected, actual, "scalar_at mismatch at index {i}"); + } + Ok(()) +} + +#[test] +fn l2_norm_readthrough() -> VortexResult<()> { + let fsl = make_fsl(10, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 5, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); + + // Stored norms should match the actual L2 norms of the input. + let norms_prim = norms_child.execute::(&mut ctx)?; + let stored_norms = norms_prim.as_slice::(); + + let input_prim = fsl.elements().clone().execute::(&mut ctx)?; + let input_f32 = input_prim.as_slice::(); + for row in 0..10 { + let vec = &input_f32[row * 128..(row + 1) * 128]; + let actual_norm: f32 = vec.iter().map(|&v| v * v).sum::().sqrt(); + assert!( + (stored_norms[row] - actual_norm).abs() < 1e-5, + "norm mismatch at row {row}: stored={}, actual={}", + stored_norms[row], + actual_norm + ); + } + + // Also verify L2Norm readthrough shortcut works. + let norms = execute_l2_norm(encoded, 10, &mut ctx)?; + assert_eq!(norms.as_slice::(), stored_norms); + assert_eq!(norms.len(), 10); + Ok(()) +} + +#[test] +fn l2_norm_readthrough_is_authoritative_for_lossy_storage() -> VortexResult<()> { + let num_rows = 12; + let fsl = make_fsl(num_rows, 128, 7); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 1, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); + + let stored_norms: PrimitiveArray = norms_child.execute(&mut ctx)?; + let encoded_norms = execute_l2_norm(encoded.clone(), num_rows, &mut ctx)?; + assert_eq!( + encoded_norms.as_slice::(), + stored_norms.as_slice::() + ); + + let decoded = encoded.execute::(&mut ctx)?.into_array(); + let decoded_norms = execute_l2_norm(decoded, num_rows, &mut ctx)?; + let max_gap = stored_norms + .as_slice::() + .iter() + .zip(decoded_norms.as_slice::().iter()) + .map(|(&stored, &decoded)| (stored - decoded).abs()) + .fold(0.0f32, f32::max); + + assert!( + max_gap > 1e-3, + "expected at least one decoded norm to drift from the authoritative stored norms, got max gap {max_gap:.6}", + ); + Ok(()) +} + +#[test] +fn cosine_similarity_readthrough_is_authoritative_for_lossy_storage() -> VortexResult<()> { + let num_rows = 12; + let fsl = make_fsl(num_rows, 128, 11); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 1, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let encoded_cos = + execute_cosine_similarity(encoded.clone(), encoded.clone(), num_rows, &mut ctx)?; + let decoded = encoded.execute::(&mut ctx)?.into_array(); + let decoded_cos = execute_cosine_similarity(decoded.clone(), decoded, num_rows, &mut ctx)?; + + let decoded_values = decoded_cos.as_slice::(); + assert!( + decoded_values + .iter() + .all(|&value| (value - 1.0).abs() < 1e-5), + "decoded cosine(x, x) should stay at 1.0", + ); + + let max_gap = encoded_cos + .as_slice::() + .iter() + .zip(decoded_values.iter()) + .map(|(&encoded, &decoded)| (encoded - decoded).abs()) + .fold(0.0f32, f32::max); + assert!( + max_gap > 1e-3, + "expected encoded cosine readthrough to differ from decoded recomputation, got max gap {max_gap:.6}", + ); + Ok(()) +} diff --git a/vortex-tensor/src/encodings/turboquant/tests/mod.rs b/vortex-tensor/src/encodings/turboquant/tests/mod.rs new file mode 100644 index 00000000000..4c01affa27d --- /dev/null +++ b/vortex-tensor/src/encodings/turboquant/tests/mod.rs @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Tests for TurboQuant encoding with decomposed SorfTransform + DictArray tree. + +mod compute; +mod nullable; +mod roundtrip; +mod structural; + +use std::sync::LazyLock; + +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand_distr::Distribution; +use rand_distr::Normal; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Dict; +use vortex_array::arrays::Extension; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFnVTable; +use vortex_array::arrays::dict::DictArraySlotsExt; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::extension::EmptyMetadata; +use vortex_array::session::ArraySession; +use vortex_array::validity::Validity; +use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +use crate::encodings::turboquant::TurboQuantConfig; +use crate::encodings::turboquant::turboquant_encode_unchecked; +use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use crate::vector::Vector; + +static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + +/// Create a FixedSizeListArray of random f32 vectors with the given validity. +fn make_fsl_with_validity( + num_rows: usize, + dim: usize, + seed: u64, + validity: Validity, +) -> FixedSizeListArray { + let mut rng = StdRng::seed_from_u64(seed); + let normal = Normal::new(0.0f32, 1.0).unwrap(); + + let mut buf = BufferMut::::with_capacity(num_rows * dim); + for _ in 0..(num_rows * dim) { + buf.push(normal.sample(&mut rng)); + } + + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into() + .expect("somehow got dimension greater than u32::MAX"), + validity, + num_rows, + ) + .unwrap() +} + +/// Create a non-nullable FixedSizeListArray of random f32 vectors. +fn make_fsl(num_rows: usize, dim: usize, seed: u64) -> FixedSizeListArray { + make_fsl_with_validity(num_rows, dim, seed, Validity::NonNullable) +} + +/// Wrap a `FixedSizeListArray` in a `Vector` extension array. +fn make_vector_ext(fsl: &FixedSizeListArray) -> ExtensionArray { + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) + .unwrap() + .erased(); + ExtensionArray::new(ext_dtype, fsl.clone().into_array()) +} + +/// Full encode pipeline: normalize → TQ-encode → wrap in L2Denorm. +fn normalize_and_encode( + ext: &ExtensionArray, + config: &TurboQuantConfig, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult { + let l2_denorm = normalize_as_l2_denorm(ext.as_ref().clone(), ctx)?; + let normalized = l2_denorm.child_at(0).clone(); + let norms = l2_denorm.child_at(1).clone(); + let num_rows = l2_denorm.len(); + + let normalized_ext = normalized + .as_opt::() + .vortex_expect("normalized child should be an Extension array"); + // SAFETY: We just normalized the input via `normalize_as_l2_denorm`. + let tq = unsafe { turboquant_encode_unchecked(normalized_ext, config, ctx)? }; + + Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) +} + +/// Unwrap an L2Denorm ScalarFnArray into (sorf_child, norms_child). +fn unwrap_l2denorm(encoded: &ArrayRef) -> (ArrayRef, ArrayRef) { + let sfn = encoded + .as_opt::() + .expect("expected ScalarFnArray (L2Denorm)"); + let sorf_child = sfn.child_at(0).clone(); + let norms_child = sfn.child_at(1).clone(); + (sorf_child, norms_child) +} + +/// Unwrap a SorfTransform ScalarFnArray to get the FSL(Dict) child. +fn unwrap_sorf(sorf: &ArrayRef) -> ArrayRef { + let sfn = sorf + .as_opt::() + .expect("expected ScalarFnArray (SorfTransform)"); + sfn.child_at(0).clone() +} + +/// Navigate the full tree to get (codes, centroids, norms) as flat arrays. +fn unwrap_codes_centroids_norms( + encoded: &ArrayRef, + ctx: &mut vortex_array::ExecutionCtx, +) -> VortexResult<(PrimitiveArray, PrimitiveArray, PrimitiveArray)> { + let (sorf_child, norms_child) = unwrap_l2denorm(encoded); + let padded_vector_child = unwrap_sorf(&sorf_child); + + // Vector wrapping FSL(Dict(codes, centroids)) + let padded_vector: ExtensionArray = padded_vector_child.execute(ctx)?; + let fsl: FixedSizeListArray = padded_vector.storage_array().clone().execute(ctx)?; + let dict = fsl + .elements() + .as_opt::() + .vortex_expect("FSL elements should be a DictArray"); + let codes: PrimitiveArray = dict.codes().clone().execute(ctx)?; + let centroids: PrimitiveArray = dict.values().clone().execute(ctx)?; + let norms: PrimitiveArray = norms_child.execute(ctx)?; + + Ok((codes, centroids, norms)) +} + +fn theoretical_mse_bound(bit_width: u8) -> f32 { + let sqrt3_pi_over_2 = (3.0f32).sqrt() * std::f32::consts::PI / 2.0; + sqrt3_pi_over_2 / (4.0f32).powi(bit_width as i32) +} + +fn per_vector_normalized_mse( + original: &[f32], + reconstructed: &[f32], + dim: usize, + num_rows: usize, +) -> f32 { + let mut total = 0.0f32; + for row in 0..num_rows { + let orig = &original[row * dim..(row + 1) * dim]; + let recon = &reconstructed[row * dim..(row + 1) * dim]; + let norm_sq: f32 = orig.iter().map(|&v| v * v).sum(); + if norm_sq < 1e-10 { + continue; + } + let err_sq: f32 = orig + .iter() + .zip(recon.iter()) + .map(|(&a, &b)| (a - b) * (a - b)) + .sum(); + total += err_sq / norm_sq; + } + total / num_rows as f32 +} + +/// Normalize, encode, and decode, returning (original, decoded) flat f32 slices. +fn encode_decode( + fsl: &FixedSizeListArray, + config: &TurboQuantConfig, +) -> VortexResult<(Vec, Vec)> { + let mut ctx = SESSION.create_execution_ctx(); + let original: Vec = { + let prim = fsl.elements().clone().execute::(&mut ctx)?; + prim.as_slice::().to_vec() + }; + let ext = make_vector_ext(fsl); + let encoded = normalize_and_encode(&ext, config, &mut ctx)?; + let decoded_ext = encoded.execute::(&mut ctx)?; + let decoded_fsl = decoded_ext + .storage_array() + .clone() + .execute::(&mut ctx)?; + let decoded_elements: Vec = { + let prim = decoded_fsl + .elements() + .clone() + .execute::(&mut ctx)?; + prim.as_slice::().to_vec() + }; + Ok((original, decoded_elements)) +} + +fn make_fsl_small(dim: usize) -> FixedSizeListArray { + let mut buf = BufferMut::::with_capacity(dim); + for i in 0..dim { + buf.push(i as f32 + 1.0); + } + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into() + .expect("somehow got dimension greater than u32::MAX"), + Validity::NonNullable, + 1, + ) + .unwrap() +} diff --git a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs new file mode 100644 index 00000000000..8d406239019 --- /dev/null +++ b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::validity::Validity; +use vortex_error::VortexResult; + +use super::*; + +/// Encode a nullable Vector array and verify roundtrip preserves validity and non-null values. +#[test] +fn nullable_vectors_roundtrip() -> VortexResult<()> { + let validity = Validity::from_iter([ + true, true, false, true, true, false, true, false, true, true, + ]); + let fsl = make_fsl_with_validity(10, 128, 42, validity); + let ext = make_vector_ext(&fsl); + + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 4, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + assert_eq!(encoded.len(), 10); + assert!(encoded.dtype().is_nullable()); + + let encoded_validity = encoded.validity()?; + for i in 0..10 { + let expected = ![2, 5, 7].contains(&i); + assert_eq!( + encoded_validity.is_valid(i)?, + expected, + "validity mismatch at row {i}" + ); + } + + let decoded_ext = encoded.execute::(&mut ctx)?; + assert_eq!(decoded_ext.len(), 10); + + let decoded_fsl = decoded_ext + .storage_array() + .clone() + .execute::(&mut ctx)?; + let decoded_prim = decoded_fsl + .elements() + .clone() + .execute::(&mut ctx)?; + let decoded_f32 = decoded_prim.as_slice::(); + + let orig_prim = fsl.elements().clone().execute::(&mut ctx)?; + let orig_f32 = orig_prim.as_slice::(); + + for row in [0, 1, 3, 4, 6, 8, 9] { + let orig_vec = &orig_f32[row * 128..(row + 1) * 128]; + let dec_vec = &decoded_f32[row * 128..(row + 1) * 128]; + let norm_sq: f32 = orig_vec.iter().map(|&v| v * v).sum(); + let err_sq: f32 = orig_vec + .iter() + .zip(dec_vec.iter()) + .map(|(&a, &b)| (a - b) * (a - b)) + .sum(); + assert!( + err_sq / norm_sq < 0.1, + "non-null row {row} has excessive reconstruction error" + ); + } + Ok(()) +} + +/// Verify that norms carry the validity: null vectors have null norms. +#[test] +fn nullable_norms_match_validity() -> VortexResult<()> { + let validity = Validity::from_iter([true, false, true, false, true]); + let fsl = make_fsl_with_validity(5, 128, 42, validity); + let ext = make_vector_ext(&fsl); + + let config = TurboQuantConfig { + bit_width: 2, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); + + let norms_validity = norms_child.validity()?; + for i in 0..5 { + let expected = i % 2 == 0; + assert_eq!( + norms_validity.is_valid(i)?, + expected, + "norms validity mismatch at row {i}" + ); + } + Ok(()) +} + +/// Verify that L2Norm readthrough works correctly on nullable TurboQuant arrays. +#[test] +fn nullable_l2_norm_readthrough() -> VortexResult<()> { + use crate::scalar_fns::l2_norm::L2Norm; + + let validity = Validity::from_iter([true, false, true, false, true]); + let fsl = make_fsl_with_validity(5, 128, 42, validity); + let ext = make_vector_ext(&fsl); + + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let norm_sfn = L2Norm::try_new_array(encoded, 5)?; + let norms: PrimitiveArray = norm_sfn.into_array().execute(&mut ctx)?; + + let orig_prim = fsl.elements().clone().execute::(&mut ctx)?; + let orig_f32 = orig_prim.as_slice::(); + for row in 0..5 { + if row % 2 == 0 { + assert!(norms.is_valid(row)?, "row {row} should be valid"); + let expected: f32 = orig_f32[row * 128..(row + 1) * 128] + .iter() + .map(|&v| v * v) + .sum::() + .sqrt(); + let actual = norms.as_slice::()[row]; + assert!( + (actual - expected).abs() < 1e-5, + "norm mismatch at valid row {row}: actual={actual}, expected={expected}" + ); + } else { + assert!(!norms.is_valid(row)?, "row {row} should be null"); + } + } + Ok(()) +} + +/// Verify that slicing a nullable TurboQuant array preserves validity. +#[test] +fn nullable_slice_preserves_validity() -> VortexResult<()> { + let validity = Validity::from_iter([ + true, true, false, true, true, false, true, false, true, true, + ]); + let fsl = make_fsl_with_validity(10, 128, 42, validity); + let ext = make_vector_ext(&fsl); + + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 2, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let sliced = encoded.slice(1..6)?; + assert_eq!(sliced.len(), 5); + + let sliced_validity = sliced.validity()?; + let expected = [true, false, true, true, false]; + for (i, &exp) in expected.iter().enumerate() { + assert_eq!( + sliced_validity.is_valid(i)?, + exp, + "sliced validity mismatch at index {i}" + ); + } + Ok(()) +} diff --git a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs new file mode 100644 index 00000000000..cd61d9193da --- /dev/null +++ b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use rstest::rstest; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::validity::Validity; +use vortex_buffer::BufferMut; +use vortex_error::VortexResult; + +use super::*; + +#[rstest] +#[case(128, 1)] +#[case(128, 2)] +#[case(128, 3)] +#[case(128, 4)] +#[case(128, 6)] +#[case(128, 8)] +#[case(256, 2)] +fn roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { + let fsl = make_fsl(10, dim, 42); + let config = TurboQuantConfig { + bit_width, + seed: Some(123), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + assert_eq!(decoded.len(), original.len()); + Ok(()) +} + +#[rstest] +#[case(128, 1)] +#[case(128, 2)] +#[case(128, 3)] +#[case(128, 4)] +#[case(256, 2)] +#[case(256, 4)] +fn mse_within_theoretical_bound(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { + let num_rows = 200; + let fsl = make_fsl(num_rows, dim, 42); + let config = TurboQuantConfig { + bit_width, + seed: Some(123), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + + let normalized_mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); + let bound = theoretical_mse_bound(bit_width); + + assert!( + normalized_mse < bound, + "Normalized MSE {normalized_mse:.6} exceeds bound {bound:.6} \ + for dim={dim}, bits={bit_width}", + ); + Ok(()) +} + +#[rstest] +#[case(128, 6)] +#[case(128, 8)] +#[case(256, 6)] +#[case(256, 8)] +fn high_bitwidth_mse_is_small(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { + let num_rows = 200; + let fsl = make_fsl(num_rows, dim, 42); + + let config_4bit = TurboQuantConfig { + bit_width: 4, + seed: Some(123), + num_rounds: 3, + }; + let (original_4, decoded_4) = encode_decode(&fsl, &config_4bit)?; + let mse_4bit = per_vector_normalized_mse(&original_4, &decoded_4, dim, num_rows); + + let config = TurboQuantConfig { + bit_width, + seed: Some(123), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + let mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); + + assert!( + mse < mse_4bit, + "{bit_width}-bit MSE ({mse:.6}) should be < 4-bit MSE ({mse_4bit:.6})" + ); + assert!(mse < 0.01, "{bit_width}-bit MSE ({mse:.6}) should be < 1%"); + Ok(()) +} + +#[test] +fn mse_decreases_with_bits() -> VortexResult<()> { + let dim = 128; + let num_rows = 50; + let fsl = make_fsl(num_rows, dim, 99); + + let mut prev_mse = f32::MAX; + for bit_width in 1..=8u8 { + let config = TurboQuantConfig { + bit_width, + seed: Some(123), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + let mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); + assert!( + mse <= prev_mse * 1.01, + "MSE should decrease: {bit_width}-bit={mse:.6} > prev={prev_mse:.6}" + ); + prev_mse = mse; + } + Ok(()) +} + +#[rstest] +#[case(0)] +#[case(1)] +fn roundtrip_edge_cases(#[case] num_rows: usize) -> VortexResult<()> { + let fsl = make_fsl(num_rows, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 2, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let decoded = encoded.execute::(&mut ctx)?; + assert_eq!(decoded.len(), num_rows); + Ok(()) +} + +#[rstest] +#[case(1)] +#[case(64)] +#[case(127)] +fn rejects_dimension_below_128(#[case] dim: usize) { + let fsl = make_fsl_small(dim); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 2, + seed: Some(0), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + assert!( + crate::encodings::turboquant::turboquant_encode(ext.as_view(), &config, &mut ctx).is_err() + ); +} + +#[rstest] +#[case(0)] +#[case(9)] +fn rejects_invalid_bit_width(#[case] bit_width: u8) { + let fsl = make_fsl(10, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width, + seed: Some(0), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let normalized = normalize_as_l2_denorm(ext.as_ref().clone(), &mut ctx) + .unwrap() + .child_at(0) + .clone(); + let normalized_ext = normalized + .as_opt::() + .expect("normalized child should be Extension"); + assert!(unsafe { turboquant_encode_unchecked(normalized_ext, &config, &mut ctx) }.is_err()); +} + +#[test] +fn all_zero_vectors_roundtrip() -> VortexResult<()> { + let num_rows = 10; + let dim = 128; + let buf = BufferMut::::full(0.0f32, num_rows * dim); + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into() + .expect("somehow got dimension greater than u32::MAX"), + Validity::NonNullable, + num_rows, + )?; + + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(42), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + for (i, (&o, &d)) in original.iter().zip(decoded.iter()).enumerate() { + assert_eq!(o, 0.0, "original[{i}] not zero"); + assert_eq!(d, 0.0, "decoded[{i}] not zero for all-zero input"); + } + Ok(()) +} + +/// Roundtrip at large embedding dimensions. +#[rstest] +#[case(768, 4)] +#[case(1024, 5)] +fn large_dimension_roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { + let num_rows = 10; + let fsl = make_fsl(num_rows, dim, 42); + let config = TurboQuantConfig { + bit_width, + seed: Some(123), + num_rounds: 3, + }; + let (original, decoded) = encode_decode(&fsl, &config)?; + assert_eq!(decoded.len(), original.len()); + + let normalized_mse = per_vector_normalized_mse(&original, &decoded, dim, num_rows); + // 2x slack for the SRHT-vs-Haar gap. + let bound = 2.0 * theoretical_mse_bound(bit_width); + assert!( + normalized_mse < bound, + "Normalized MSE {normalized_mse:.6} exceeds 2x bound {bound:.6} for dim={dim}, bits={bit_width}", + ); + Ok(()) +} + +/// Verify that f64 input is accepted and encoded. +#[test] +fn f64_input_encodes_successfully() -> VortexResult<()> { + let num_rows = 10; + let dim = 128; + let mut rng = StdRng::seed_from_u64(99); + let normal = Normal::new(0.0f64, 1.0).unwrap(); + + let mut buf = BufferMut::::with_capacity(num_rows * dim); + for _ in 0..(num_rows * dim) { + buf.push(normal.sample(&mut rng)); + } + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into().unwrap(), + Validity::NonNullable, + num_rows, + )?; + + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(42), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); + assert_eq!(norms_child.len(), num_rows); + Ok(()) +} + +/// Verify that f16 input is accepted and encoded. +#[test] +fn f16_input_encodes_successfully() -> VortexResult<()> { + let num_rows = 10; + let dim = 128; + let mut rng = StdRng::seed_from_u64(99); + let normal = Normal::new(0.0f32, 1.0).unwrap(); + + let mut buf = BufferMut::::with_capacity(num_rows * dim); + for _ in 0..(num_rows * dim) { + buf.push(half::f16::from_f32(normal.sample(&mut rng))); + } + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into().unwrap(), + Validity::NonNullable, + num_rows, + )?; + + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(42), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); + assert_eq!(norms_child.len(), num_rows); + + let decoded_ext = encoded.execute::(&mut ctx)?; + let decoded_fsl = decoded_ext + .storage_array() + .clone() + .execute::(&mut ctx)?; + assert_eq!(decoded_fsl.len(), num_rows); + Ok(()) +} + +/// Verify that the checked encode accepts normalized f16 input. +#[test] +fn checked_encode_accepts_normalized_f16_input() -> VortexResult<()> { + let num_rows = 10; + let dim = 128; + let mut rng = StdRng::seed_from_u64(99); + let normal = Normal::new(0.0f32, 1.0).unwrap(); + + let mut buf = BufferMut::::with_capacity(num_rows * dim); + for _ in 0..(num_rows * dim) { + buf.push(half::f16::from_f32(normal.sample(&mut rng))); + } + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into().unwrap(), + Validity::NonNullable, + num_rows, + )?; + + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(42), + num_rounds: 3, + }; + + let mut ctx = SESSION.create_execution_ctx(); + let normalized = normalize_as_l2_denorm(ext.as_ref().clone(), &mut ctx)? + .child_at(0) + .clone(); + let normalized_ext = normalized + .as_opt::() + .vortex_expect("normalized child should be an Extension array"); + + let encoded = + crate::encodings::turboquant::turboquant_encode(normalized_ext, &config, &mut ctx)?; + assert_eq!(encoded.len(), num_rows); + Ok(()) +} diff --git a/vortex-tensor/src/encodings/turboquant/tests/structural.rs b/vortex-tensor/src/encodings/turboquant/tests/structural.rs new file mode 100644 index 00000000000..87b59836b38 --- /dev/null +++ b/vortex-tensor/src/encodings/turboquant/tests/structural.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Tests that verify the internal structure of the encoded tree. + +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_error::VortexResult; + +use super::*; + +/// Verify that the centroids stored in the DictArray match what `get_centroids()` computes. +#[test] +fn stored_centroids_match_computed() -> VortexResult<()> { + let fsl = make_fsl(10, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let (_codes, centroids, _norms) = unwrap_codes_centroids_norms(&encoded, &mut ctx)?; + let stored = centroids.as_slice::(); + + // padded_dim for dim=128 is 128. + let computed = crate::encodings::turboquant::centroids::get_centroids(128, 3)?; + + assert_eq!(stored.len(), computed.len()); + for i in 0..stored.len() { + assert_eq!(stored[i], computed[i], "Centroid mismatch at {i}"); + } + Ok(()) +} + +/// Verify that the rotation is deterministic from seed by checking decode output. +#[test] +fn seed_deterministic_rotation_produces_correct_decode() -> VortexResult<()> { + let fsl = make_fsl(20, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 4, + }; + + // Encode twice with the same seed → should produce identical results. + let mut ctx = SESSION.create_execution_ctx(); + let encoded1 = normalize_and_encode(&ext, &config, &mut ctx)?; + let decoded1 = encoded1.execute::(&mut ctx)?; + let fsl1 = decoded1 + .storage_array() + .clone() + .execute::(&mut ctx)?; + let elems1 = fsl1 + .elements() + .clone() + .execute::(&mut ctx)?; + + let mut ctx = SESSION.create_execution_ctx(); + let encoded2 = normalize_and_encode(&ext, &config, &mut ctx)?; + let decoded2 = encoded2.execute::(&mut ctx)?; + let fsl2 = decoded2 + .storage_array() + .clone() + .execute::(&mut ctx)?; + let elems2 = fsl2 + .elements() + .clone() + .execute::(&mut ctx)?; + + assert_eq!( + elems1.as_slice::(), + elems2.as_slice::(), + "Two encodes with same seed should produce identical decode output" + ); + Ok(()) +} + +/// Verify that the encoded array's dtype is a Vector extension type. +#[test] +fn encoded_dtype_is_vector_extension() -> VortexResult<()> { + let fsl = make_fsl(10, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 3, + seed: Some(123), + num_rounds: 2, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + assert!( + encoded.dtype().is_extension(), + "TurboQuant dtype should be an extension type, got {}", + encoded.dtype() + ); + assert!( + encoded.dtype().as_extension().is::(), + "TurboQuant dtype should be a Vector extension type" + ); + Ok(()) +} + +/// Verify approximate cosine similarity in the quantized domain. +#[test] +fn cosine_similarity_quantized_accuracy() -> VortexResult<()> { + let fsl = make_fsl(20, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 4, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let input_prim = fsl.elements().clone().execute::(&mut ctx)?; + let input_f32 = input_prim.as_slice::(); + + // Navigate tree to get codes, centroids, norms. + let (codes_prim, centroids_prim, norms_prim) = + unwrap_codes_centroids_norms(&encoded, &mut ctx)?; + let all_codes = codes_prim.as_slice::(); + let centroid_vals = centroids_prim.as_slice::(); + let norms = norms_prim.as_slice::(); + + // padded_dim for dim=128. + let pd = 128usize; + + for (row_a, row_b) in [(0, 1), (5, 10), (0, 19)] { + let vec_a = &input_f32[row_a * 128..(row_a + 1) * 128]; + let vec_b = &input_f32[row_b * 128..(row_b + 1) * 128]; + + let dot: f32 = vec_a.iter().zip(vec_b.iter()).map(|(&x, &y)| x * y).sum(); + let norm_a: f32 = vec_a.iter().map(|&v| v * v).sum::().sqrt(); + let norm_b: f32 = vec_b.iter().map(|&v| v * v).sum::().sqrt(); + let exact_cos = dot / (norm_a * norm_b); + + let approx_cos = if norms[row_a] == 0.0 || norms[row_b] == 0.0 { + 0.0 + } else { + let codes_a = &all_codes[row_a * pd..(row_a + 1) * pd]; + let codes_b = &all_codes[row_b * pd..(row_b + 1) * pd]; + codes_a + .iter() + .zip(codes_b.iter()) + .map(|(&ca, &cb)| centroid_vals[ca as usize] * centroid_vals[cb as usize]) + .sum::() + }; + + let error = (exact_cos - approx_cos).abs(); + assert!( + error < 0.15, + "cosine similarity error too large for ({row_a}, {row_b}): \ + exact={exact_cos:.4}, approx={approx_cos:.4}, error={error:.4}" + ); + } + Ok(()) +} + +/// Verify approximate dot product in the quantized domain. +#[test] +fn dot_product_quantized_accuracy() -> VortexResult<()> { + let fsl = make_fsl(20, 128, 42); + let ext = make_vector_ext(&fsl); + let config = TurboQuantConfig { + bit_width: 8, + seed: Some(123), + num_rounds: 3, + }; + let mut ctx = SESSION.create_execution_ctx(); + let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + + let input_prim = fsl.elements().clone().execute::(&mut ctx)?; + let input_f32 = input_prim.as_slice::(); + + let (codes_prim, centroids_prim, norms_prim) = + unwrap_codes_centroids_norms(&encoded, &mut ctx)?; + let all_codes = codes_prim.as_slice::(); + let centroid_vals = centroids_prim.as_slice::(); + let norms = norms_prim.as_slice::(); + + let pd = 128usize; + + for (row_a, row_b) in [(0, 1), (5, 10), (0, 19)] { + let vec_a = &input_f32[row_a * 128..(row_a + 1) * 128]; + let vec_b = &input_f32[row_b * 128..(row_b + 1) * 128]; + + let exact_dot: f32 = vec_a.iter().zip(vec_b.iter()).map(|(&x, &y)| x * y).sum(); + + let codes_a = &all_codes[row_a * pd..(row_a + 1) * pd]; + let codes_b = &all_codes[row_b * pd..(row_b + 1) * pd]; + let unit_dot: f32 = codes_a + .iter() + .zip(codes_b.iter()) + .map(|(&ca, &cb)| centroid_vals[ca as usize] * centroid_vals[cb as usize]) + .sum(); + let approx_dot = norms[row_a] * norms[row_b] * unit_dot; + + let scale = exact_dot.abs().max(1.0); + let rel_error = (exact_dot - approx_dot).abs() / scale; + assert!( + rel_error < 0.15, + "dot product error too large for ({row_a}, {row_b}): \ + exact={exact_dot:.4}, approx={approx_dot:.4}, rel_error={rel_error:.4}" + ); + } + Ok(()) +} + +/// Verify SorfTransform in isolation: manually forward-rotate known data, wrap in +/// FSL(Dict), execute SorfTransform, and check inverse rotation recovers the original. +#[test] +#[expect( + clippy::cast_possible_truncation, + reason = "test uses known small dimensions" +)] +fn sorf_transform_roundtrip_isolation() -> VortexResult<()> { + use vortex_array::IntoArray; + use vortex_array::arrays::dict::DictArray; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::extension::EmptyMetadata; + use vortex_array::validity::Validity; + use vortex_buffer::BufferMut; + + use crate::encodings::turboquant::centroids::compute_centroid_boundaries; + use crate::encodings::turboquant::centroids::find_nearest_centroid; + use crate::encodings::turboquant::centroids::get_centroids; + use crate::scalar_fns::sorf_transform::SorfMatrix; + use crate::scalar_fns::sorf_transform::SorfOptions; + use crate::scalar_fns::sorf_transform::SorfTransform; + use crate::vector::Vector; + + let dim = 128usize; + let seed = 99u64; + let num_rounds = 3u8; + let num_rows = 5; + + // Build a known input: simple increasing values, then normalize each row to unit norm. + let mut input_f32 = vec![0.0f32; num_rows * dim]; + for row in 0..num_rows { + let mut norm_sq = 0.0f32; + for i in 0..dim { + let val = ((row * dim + i) as f32 + 1.0) * 0.01; + input_f32[row * dim + i] = val; + norm_sq += val * val; + } + let norm = norm_sq.sqrt(); + for i in 0..dim { + input_f32[row * dim + i] /= norm; + } + } + + // Forward transform + quantize (mimicking what turboquant_quantize_core does). + let rotation = SorfMatrix::try_new(seed, dim, num_rounds as usize)?; + let padded_dim = rotation.padded_dim(); + let centroids = get_centroids(padded_dim as u32, 8)?; + let boundaries = compute_centroid_boundaries(¢roids); + + let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); + let mut padded = vec![0.0f32; padded_dim]; + let mut rotated = vec![0.0f32; padded_dim]; + + for row in 0..num_rows { + padded[..dim].copy_from_slice(&input_f32[row * dim..(row + 1) * dim]); + padded[dim..].fill(0.0); + rotation.rotate(&padded, &mut rotated); + for j in 0..padded_dim { + all_indices.push(find_nearest_centroid(rotated[j], &boundaries)); + } + } + + // Build FSL(Dict(codes, centroids)). + let codes = PrimitiveArray::new::(all_indices.freeze(), Validity::NonNullable); + let mut centroids_buf = BufferMut::::with_capacity(centroids.len()); + centroids_buf.extend_from_slice(¢roids); + let centroids_arr = PrimitiveArray::new::(centroids_buf.freeze(), Validity::NonNullable); + let dict = DictArray::try_new(codes.into_array(), centroids_arr.into_array())?; + let fsl = FixedSizeListArray::try_new( + dict.into_array(), + padded_dim as u32, + Validity::NonNullable, + num_rows, + )?; + + // Wrap the padded FSL in a Vector extension so it can be the SorfTransform child. + let padded_vector_dtype = + ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + let padded_vector = ExtensionArray::new(padded_vector_dtype, fsl.into_array()); + + // Wrap in SorfTransform and execute. + let sorf_options = SorfOptions { + seed, + num_rounds, + dimension: dim as u32, + element_ptype: vortex_array::dtype::PType::F32, + }; + let sorf_array = + SorfTransform::try_new_array(&sorf_options, padded_vector.into_array(), num_rows)?; + + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf_array.into_array().execute(&mut ctx)?; + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + let result_prim: PrimitiveArray = result_fsl.elements().clone().execute(&mut ctx)?; + let result_f32 = result_prim.as_slice::(); + + assert_eq!(result_f32.len(), num_rows * dim); + + // At 8-bit quantization, reconstruction should be very close to input. + for row in 0..num_rows { + let orig = &input_f32[row * dim..(row + 1) * dim]; + let recon = &result_f32[row * dim..(row + 1) * dim]; + let err_sq: f32 = orig + .iter() + .zip(recon) + .map(|(&a, &b)| (a - b) * (a - b)) + .sum(); + let norm_sq: f32 = orig.iter().map(|&v| v * v).sum(); + assert!( + err_sq / norm_sq < 1e-3, + "SorfTransform isolation: row {row} MSE too high: {:.6}", + err_sq / norm_sq + ); + } + Ok(()) +} diff --git a/vortex-tensor/src/encodings/turboquant/vtable.rs b/vortex-tensor/src/encodings/turboquant/vtable.rs deleted file mode 100644 index b673dea6ba5..00000000000 --- a/vortex-tensor/src/encodings/turboquant/vtable.rs +++ /dev/null @@ -1,382 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! VTable implementation for TurboQuant encoding. - -use std::hash::Hash; -use std::hash::Hasher; -use std::sync::Arc; - -use prost::Message; -use vortex_array::Array; -use vortex_array::ArrayEq; -use vortex_array::ArrayHash; -use vortex_array::ArrayId; -use vortex_array::ArrayParts; -use vortex_array::ArrayRef; -use vortex_array::ArrayView; -use vortex_array::ExecutionCtx; -use vortex_array::ExecutionResult; -use vortex_array::Precision; -use vortex_array::buffer::BufferHandle; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; -use vortex_array::serde::ArrayChildren; -use vortex_array::validity::Validity; -use vortex_array::vtable::VTable; -use vortex_array::vtable::ValidityVTable; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_ensure_eq; -use vortex_error::vortex_err; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; - -use crate::encodings::turboquant::TurboQuantData; -use crate::encodings::turboquant::array::slots::Slot; -use crate::encodings::turboquant::compute::rules::PARENT_KERNELS; -use crate::encodings::turboquant::compute::rules::RULES; -use crate::encodings::turboquant::metadata::TurboQuantMetadata; -use crate::encodings::turboquant::scheme::decompress::execute_decompress; -use crate::vector::AnyVector; -use crate::vector::VectorMatcherMetadata; - -/// Encoding marker type for TurboQuant. -#[derive(Clone, Debug)] -pub struct TurboQuant; - -impl TurboQuant { - pub const ID: ArrayId = ArrayId::new_ref("vortex.turboquant"); - - /// Minimum vector dimension for TurboQuant encoding. - /// - /// Note that this is not a theoretical minimum, it is mostly a practical one to limit the total - /// amount of distortion. - pub const MIN_DIMENSION: u32 = 128; - - /// Maximum supported number of bits per quantized coordinate. - pub const MAX_BIT_WIDTH: u8 = 8; - - /// Maximum supported number of centroids in the scalar quantizer codebook. - pub const MAX_CENTROIDS: usize = 1usize << (Self::MAX_BIT_WIDTH as usize); - - /// Validates that `dtype` is a [`Vector`](crate::vector::Vector) extension type with - /// dimension >= [`MIN_DIMENSION`](Self::MIN_DIMENSION). - /// - /// Returns the validated vector metadata on success. - pub fn validate_dtype(dtype: &DType) -> VortexResult { - let vector_metadata = dtype - .as_extension_opt() - .and_then(|ext| ext.metadata_opt::()) - .ok_or_else(|| { - vortex_err!("TurboQuant dtype must be a Vector extension type, got {dtype}") - })?; - - let dimensions = vector_metadata.dimensions(); - vortex_ensure!( - dimensions >= Self::MIN_DIMENSION, - "TurboQuant requires dimension >= {}, got {dimensions}", - Self::MIN_DIMENSION - ); - - Ok(vector_metadata) - } - - /// Creates a new [`TurboQuantArray`]. - /// - /// The `dtype` must be a non-nullable [`Vector`](crate::vector::Vector) extension type. - /// Nullability is handled externally by the [`L2Denorm`](crate::scalar_fns::l2_denorm::L2Denorm) - /// ScalarFnArray wrapper. - /// - /// Internally calls [`TurboQuantData::validate`] and [`TurboQuantData::try_new`], then - /// delegates to [`new_array_unchecked`](Self::new_array_unchecked). - pub fn try_new_array( - dtype: DType, - codes: ArrayRef, - centroids: ArrayRef, - rotation_signs: ArrayRef, - ) -> VortexResult { - TurboQuantData::validate(&dtype, &codes, ¢roids, &rotation_signs)?; - - Ok(unsafe { Self::new_array_unchecked(dtype, codes, centroids, rotation_signs) }) - } - - /// Creates a new [`TurboQuantArray`] without validation. - /// - /// # Safety - /// - /// The caller must ensure all invariants required by [`TurboQuantData::validate`] hold: - /// - /// - `dtype` is a non-nullable [`Vector`](crate::vector::Vector) extension type with - /// dimension >= [`MIN_DIMENSION`](Self::MIN_DIMENSION). - /// - `codes` is a non-nullable `FixedSizeList` with `list_size == padded_dim`. - /// - `centroids` is a non-nullable `Primitive` with a power-of-2 length in - /// `[2, MAX_CENTROIDS]` (or empty for degenerate arrays). - /// - `rotation_signs` is a non-nullable `FixedSizeList` with `list_size == padded_dim`. - /// - /// Violating these invariants may produce incorrect results during decompression or panics - /// during array access. - pub unsafe fn new_array_unchecked( - dtype: DType, - codes: ArrayRef, - centroids: ArrayRef, - rotation_signs: ArrayRef, - ) -> TurboQuantArray { - #[cfg(debug_assertions)] - TurboQuantData::validate(&dtype, &codes, ¢roids, &rotation_signs) - .vortex_expect("[DEBUG ASSERTION]: TurboQuantData arrays are invalid"); - - let len = codes.len(); - - let dimension = dtype - .as_extension_opt() - .vortex_expect("we validated the dtype") - .metadata_opt::() - .vortex_expect("we validated that this is a vector") - .dimensions(); - - let bit_width = if centroids.is_empty() { - 0 - } else { - #[expect( - clippy::cast_possible_truncation, - reason = "bit_width is guaranteed <= 8" - )] - (centroids.len().trailing_zeros() as u8) - }; - - #[expect( - clippy::cast_possible_truncation, - reason = "num_rounds fits in u8 by the caller's invariants" - )] - let num_rounds = rotation_signs.len() as u8; - - // SAFETY: The caller guarantees that dimension, bit_width, and num_rounds satisfy the - // invariants documented on `TurboQuantData::new_unchecked`. - let data = unsafe { TurboQuantData::new_unchecked(dimension, bit_width, num_rounds) }; - let parts = ArrayParts::new(TurboQuant, dtype, len, data) - .with_slots(TurboQuantData::make_slots(codes, centroids, rotation_signs)); - - // SAFETY: The caller guarantees the parts are logically consistent. - unsafe { Array::from_parts_unchecked(parts) } - } -} - -/// A [`TurboQuant`]-encoded Vortex array. -pub type TurboQuantArray = Array; - -impl VTable for TurboQuant { - type ArrayData = TurboQuantData; - type OperationsVTable = TurboQuant; - type ValidityVTable = TurboQuant; - - fn id(&self) -> ArrayId { - Self::ID - } - - fn validate( - &self, - data: &Self::ArrayData, - dtype: &DType, - len: usize, - slots: &[Option], - ) -> VortexResult<()> { - vortex_ensure_eq!( - slots.len(), - Slot::COUNT, - "TurboQuantArray got incorrect amount of slots", - ); - - // Even if the array is degenerate (empty), the arrays still have to exist - // (they will be empty). - let codes = slots[Slot::Codes as usize] - .as_ref() - .ok_or_else(|| vortex_err!("TurboQuantArray missing codes slot"))?; - let centroids = slots[Slot::Centroids as usize] - .as_ref() - .ok_or_else(|| vortex_err!("TurboQuantArray missing centroids slot"))?; - let rotation_signs = slots[Slot::RotationSigns as usize] - .as_ref() - .ok_or_else(|| vortex_err!("TurboQuantArray missing rotation_signs slot"))?; - - vortex_ensure_eq!( - codes.len(), - len, - "TurboQuant codes length does not match outer length", - ); - - TurboQuantData::validate(dtype, codes, centroids, rotation_signs)?; - - vortex_ensure_eq!(data.dimension, Self::validate_dtype(dtype)?.dimensions()); - - let expected_bit_width = if centroids.is_empty() { - 0 - } else { - u8::try_from(centroids.len().trailing_zeros()) - .map_err(|_| vortex_err!("centroids bit_width does not fit in u8"))? - }; - vortex_ensure_eq!( - data.bit_width, - expected_bit_width, - "TurboQuant bit_width does not match centroids slot", - ); - - // Verify num_rounds matches the rotation_signs FSL length. - let expected_num_rounds = u8::try_from(rotation_signs.len()) - .map_err(|_| vortex_err!("rotation_signs num_rounds does not fit in u8"))?; - vortex_ensure_eq!( - data.num_rounds, - expected_num_rounds, - "TurboQuant num_rounds does not match rotation_signs slot", - ); - - Ok(()) - } - - fn nbuffers(_array: ArrayView) -> usize { - 0 - } - - fn buffer(_array: ArrayView, idx: usize) -> BufferHandle { - vortex_panic!("TurboQuantArray buffer index {idx} out of bounds") - } - - fn buffer_name(_array: ArrayView, _idx: usize) -> Option { - None - } - - fn serialize( - array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - Ok(Some( - TurboQuantMetadata::new(array.bit_width, array.num_rounds).encode_to_vec(), - )) - } - - fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - _buffers: &[BufferHandle], - children: &dyn ArrayChildren, - _session: &VortexSession, - ) -> VortexResult> { - let metadata = TurboQuantMetadata::decode(metadata)?; - let bit_width = metadata.bit_width()?; - let num_rounds = metadata.num_rounds()?; - - // bit_width == 0 and num_rounds == 0 are only valid for degenerate (empty) arrays. - vortex_ensure!( - bit_width > 0 || len == 0, - "bit_width == 0 is only valid for empty arrays, got len={len}" - ); - vortex_ensure!( - num_rounds > 0 || len == 0, - "num_rounds == 0 is only valid for empty arrays, got len={len}" - ); - - // Validate and derive dimension from the Vector extension dtype. - let vector_metadata = TurboQuant::validate_dtype(dtype)?; - let dimensions = vector_metadata.dimensions(); - - // TurboQuant arrays are always non-nullable. - vortex_ensure!( - !dtype.is_nullable(), - "TurboQuant dtype must be non-nullable during deserialization" - ); - - let padded_dim = dimensions.next_power_of_two(); - - // Get the codes array (indices into the codebook). Codes are always non-nullable. - let codes_ptype = DType::Primitive(PType::U8, Nullability::NonNullable); - let codes_dtype = - DType::FixedSizeList(Arc::new(codes_ptype), padded_dim, Nullability::NonNullable); - let codes_array = children.get(0, &codes_dtype, len)?; - - // Get the centroids array (codebook). - let num_centroids = if bit_width == 0 { - 0 // A degenerate TQ array. - } else { - 1usize << bit_width - }; - let centroids_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - let centroids = children.get(1, ¢roids_dtype, num_centroids)?; - - // Get the rotation signs array (FixedSizeList with list_size = padded_dim). - let signs_len = if len == 0 { 0 } else { num_rounds as usize }; - let signs_dtype = DType::FixedSizeList( - Arc::new(DType::Primitive(PType::U8, Nullability::NonNullable)), - padded_dim, - Nullability::NonNullable, - ); - let rotation_signs = children.get(2, &signs_dtype, signs_len)?; - - Ok(ArrayParts::new( - TurboQuant, - dtype.clone(), - len, - TurboQuantData { - dimension: dimensions, - bit_width, - num_rounds, - }, - ) - .with_slots(TurboQuantData::make_slots( - codes_array, - centroids, - rotation_signs, - ))) - } - - fn slot_name(_array: ArrayView, idx: usize) -> String { - Slot::from_index(idx).name().to_string() - } - fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - Ok(ExecutionResult::done(execute_decompress(array, ctx)?)) - } - - fn execute_parent( - array: ArrayView, - parent: &ArrayRef, - child_idx: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult> { - PARENT_KERNELS.execute(array, parent, child_idx, ctx) - } - - fn reduce_parent( - array: ArrayView, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - RULES.evaluate(array, parent, child_idx) - } -} - -impl ValidityVTable for TurboQuant { - fn validity(_array: ArrayView<'_, TurboQuant>) -> VortexResult { - // TurboQuant arrays are always non-nullable. This method is only called when the dtype is - // nullable, which should never happen for TQ arrays. - Ok(Validity::NonNullable) - } -} - -impl ArrayHash for TurboQuantData { - fn array_hash(&self, state: &mut H, _precision: Precision) { - self.dimension.hash(state); - self.bit_width.hash(state); - self.num_rounds.hash(state); - } -} - -impl ArrayEq for TurboQuantData { - fn array_eq(&self, other: &Self, _precision: Precision) -> bool { - self.dimension == other.dimension - && self.bit_width == other.bit_width - && self.num_rounds == other.num_rounds - } -} diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index a4577f6d262..3d3563aa8e4 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -7,15 +7,14 @@ use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; -use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; -use crate::encodings::turboquant::TurboQuant; use crate::fixed_shape::FixedShapeTensor; use crate::scalar_fns::cosine_similarity::CosineSimilarity; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_norm::L2Norm; +use crate::scalar_fns::sorf_transform::SorfTransform; use crate::vector::Vector; pub mod matcher; @@ -33,10 +32,9 @@ pub fn initialize(session: &VortexSession) { session.dtypes().register(Vector); session.dtypes().register(FixedShapeTensor); - session.arrays().register(TurboQuant); - session.scalar_fns().register(CosineSimilarity); session.scalar_fns().register(InnerProduct); session.scalar_fns().register(L2Denorm); session.scalar_fns().register(L2Norm); + session.scalar_fns().register(SorfTransform); } diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 39c8b135e43..f4f129c2b27 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -20,6 +20,7 @@ use vortex_array::expr::and; use vortex_array::match_each_float_ptype; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; @@ -29,7 +30,6 @@ use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure; -use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_norm::L2Norm; @@ -45,16 +45,20 @@ use crate::utils::validate_tensor_float_input; /// Both inputs must be tensor-like extension arrays ([`FixedShapeTensor`] or [`Vector`]) with the /// same dtype and a float element type. The output is a float column of the same float type. /// +/// When either input is wrapped in [`L2Denorm`], this operator treats the stored norms and +/// normalized children as authoritative. For lossy encodings such as TurboQuant, that means the +/// optimized readthrough path may intentionally differ slightly from decoding both sides to dense +/// coordinates and recomputing cosine from scratch. +/// /// [`FixedShapeTensor`]: crate::fixed_shape::FixedShapeTensor /// [`Vector`]: crate::vector::Vector #[derive(Clone)] pub struct CosineSimilarity; impl CosineSimilarity { - /// Creates a new [`ScalarFn`] wrapping the cosine similarity operation with the given - /// [`ApproxOptions`] controlling approximation behavior. - pub fn new(options: &ApproxOptions) -> ScalarFn { - ScalarFn::new(CosineSimilarity, options.clone()) + /// Creates a new [`ScalarFn`] wrapping the cosine similarity operation. + pub fn new() -> ScalarFn { + ScalarFn::new(CosineSimilarity, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the cosine similarity between `lhs` and @@ -64,18 +68,13 @@ impl CosineSimilarity { /// /// Returns an error if the [`ScalarFnArray`] cannot be constructed (e.g. due to dtype /// mismatches). - pub fn try_new_array( - options: &ApproxOptions, - lhs: ArrayRef, - rhs: ArrayRef, - len: usize, - ) -> VortexResult { - ScalarFnArray::try_new(CosineSimilarity::new(options).erased(), vec![lhs, rhs], len) + pub fn try_new_array(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult { + ScalarFnArray::try_new(CosineSimilarity::new().erased(), vec![lhs, rhs], len) } } impl ScalarFnVTable for CosineSimilarity { - type Options = ApproxOptions; + type Options = EmptyOptions; fn id(&self) -> ScalarFnId { ScalarFnId::from("vortex.tensor.cosine_similarity") @@ -126,7 +125,7 @@ impl ScalarFnVTable for CosineSimilarity { fn execute( &self, - options: &Self::Options, + _options: &Self::Options, args: &dyn ExecutionArgs, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -140,12 +139,12 @@ impl ScalarFnVTable for CosineSimilarity { let rhs_is_denorm = rhs_ref.is::>(); if lhs_is_denorm && rhs_is_denorm { - return self.execute_both_denorm(options, &lhs_ref, &rhs_ref, len, ctx); + return self.execute_both_denorm(&lhs_ref, &rhs_ref, len, ctx); } else if lhs_is_denorm || rhs_is_denorm { if rhs_is_denorm { (lhs_ref, rhs_ref) = (rhs_ref, lhs_ref); } - return self.execute_one_denorm(options, &lhs_ref, &rhs_ref, len, ctx); + return self.execute_one_denorm(&lhs_ref, &rhs_ref, len, ctx); } } @@ -153,9 +152,9 @@ impl ScalarFnVTable for CosineSimilarity { let validity = lhs_ref.validity()?.and(rhs_ref.validity()?)?; // Compute inner product and norms as columnar operations, and propagate the options. - let norm_lhs_arr = L2Norm::try_new_array(options, lhs_ref.clone(), len)?; - let norm_rhs_arr = L2Norm::try_new_array(options, rhs_ref.clone(), len)?; - let dot_arr = InnerProduct::try_new_array(options, lhs_ref, rhs_ref, len)?; + let norm_lhs_arr = L2Norm::try_new_array(lhs_ref.clone(), len)?; + let norm_rhs_arr = L2Norm::try_new_array(rhs_ref.clone(), len)?; + let dot_arr = InnerProduct::try_new_array(lhs_ref, rhs_ref, len)?; // Execute to get the inner product and norms of the arrays. We only fully decompress // because we need to perform special logic (guard against 0) during division. @@ -208,10 +207,10 @@ impl ScalarFnVTable for CosineSimilarity { } impl CosineSimilarity { - /// Both sides are `L2Denorm`: norms cancel, so `cosine_similarity = dot(n_l, n_r)`. + /// Both sides are `L2Denorm`: treat the normalized children as authoritative, so + /// `cosine_similarity = dot(n_l, n_r)`. fn execute_both_denorm( &self, - options: &ApproxOptions, lhs_ref: &ArrayRef, rhs_ref: &ArrayRef, len: usize, @@ -222,9 +221,9 @@ impl CosineSimilarity { let (normalized_l, _) = extract_l2_denorm_children(lhs_ref); let (normalized_r, _) = extract_l2_denorm_children(rhs_ref); - // Dot product of already-normalized children IS the cosine similarity. - let dot = - InnerProduct::try_new_array(options, normalized_l, normalized_r, len)?.into_array(); + // `L2Denorm` makes the normalized children authoritative, so their dot product is the + // cosine similarity even for lossy storage wrappers. + let dot = InnerProduct::try_new_array(normalized_l, normalized_r, len)?.into_array(); if !matches!(validity, Validity::NonNullable) { // Masking always changes the nullability to nullable. @@ -234,12 +233,12 @@ impl CosineSimilarity { } } - /// One side is `L2Denorm`: `cosine_similarity = dot(n, b) / ||b||`. + /// One side is `L2Denorm`: treat the normalized child as authoritative, so + /// `cosine_similarity = dot(n, b) / ||b||`. /// /// The caller must pass the denorm array as `denorm_ref` and the plain array as `plain_ref`. fn execute_one_denorm( &self, - options: &ApproxOptions, denorm_ref: &ArrayRef, plain_ref: &ArrayRef, len: usize, @@ -249,8 +248,8 @@ impl CosineSimilarity { let (normalized, _) = extract_l2_denorm_children(denorm_ref); - let dot_arr = InnerProduct::try_new_array(options, normalized, plain_ref.clone(), len)?; - let norm_arr = L2Norm::try_new_array(options, plain_ref.clone(), len)?; + let dot_arr = InnerProduct::try_new_array(normalized, plain_ref.clone(), len)?; + let norm_arr = L2Norm::try_new_array(plain_ref.clone(), len)?; let dot: PrimitiveArray = dot_arr.into_array().execute(ctx)?; let plain_norm: PrimitiveArray = norm_arr.into_array().execute(ctx)?; @@ -286,13 +285,11 @@ mod tests { use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::scalar_fn::ScalarFn; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::cosine_similarity::CosineSimilarity; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::utils::test_helpers::assert_close; @@ -306,7 +303,7 @@ mod tests { /// Evaluates cosine similarity between two tensor arrays and returns the result as `Vec`. fn eval_cosine_similarity(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { - let scalar_fn = ScalarFn::new(CosineSimilarity, ApproxOptions::Exact).erased(); + let scalar_fn = CosineSimilarity::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], len)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; @@ -477,7 +474,7 @@ mod tests { let rhs = tensor_array(&[2], &[3.0, 4.0, 0.0, 1.0])?; let rhs = MaskedArray::try_new(rhs, Validity::from_iter([true, false]))?.into_array(); - let scalar_fn = ScalarFn::new(CosineSimilarity, ApproxOptions::Exact).erased(); + let scalar_fn = CosineSimilarity::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 2)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; @@ -499,10 +496,7 @@ mod tests { let normalized = tensor_array(shape, normalized_elements)?; let norms = PrimitiveArray::from_iter(norms.iter().copied()).into_array(); let mut ctx = SESSION.create_execution_ctx(); - Ok( - L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, len, &mut ctx)? - .into_array(), - ) + Ok(L2Denorm::try_new_array(normalized, norms, len, &mut ctx)?.into_array()) } #[test] @@ -570,11 +564,9 @@ mod tests { let normalized_r = tensor_array(&[2], &[0.6, 0.8, 1.0, 0.0])?; let norms_r = PrimitiveArray::from_option_iter([Some(5.0f64), None]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let rhs = - L2Denorm::try_new_array(&ApproxOptions::Exact, normalized_r, norms_r, 2, &mut ctx)? - .into_array(); + let rhs = L2Denorm::try_new_array(normalized_r, norms_r, 2, &mut ctx)?.into_array(); - let scalar_fn = ScalarFn::new(CosineSimilarity, ApproxOptions::Exact).erased(); + let scalar_fn = CosineSimilarity::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 2)?; let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index da1b62e6ca1..71288c241ae 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -22,6 +22,7 @@ use vortex_array::expr::and; use vortex_array::match_each_float_ptype; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; @@ -33,7 +34,6 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use crate::matcher::AnyTensor; -use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::utils::extract_flat_elements; use crate::utils::extract_l2_denorm_children; @@ -53,10 +53,9 @@ use crate::utils::extract_l2_denorm_children; pub struct InnerProduct; impl InnerProduct { - /// Creates a new [`ScalarFn`] wrapping the inner product operation with the given - /// [`ApproxOptions`] controlling approximation behavior. - pub fn new(options: &ApproxOptions) -> ScalarFn { - ScalarFn::new(InnerProduct, options.clone()) + /// Creates a new [`ScalarFn`] wrapping the inner product operation. + pub fn new() -> ScalarFn { + ScalarFn::new(InnerProduct, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the inner product between `lhs` and @@ -66,18 +65,13 @@ impl InnerProduct { /// /// Returns an error if the [`ScalarFnArray`] cannot be constructed (e.g. due to dtype /// mismatches). - pub fn try_new_array( - options: &ApproxOptions, - lhs: ArrayRef, - rhs: ArrayRef, - len: usize, - ) -> VortexResult { - ScalarFnArray::try_new(InnerProduct::new(options).erased(), vec![lhs, rhs], len) + pub fn try_new_array(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult { + ScalarFnArray::try_new(InnerProduct::new().erased(), vec![lhs, rhs], len) } } impl ScalarFnVTable for InnerProduct { - type Options = ApproxOptions; + type Options = EmptyOptions; fn id(&self) -> ScalarFnId { ScalarFnId::from("vortex.tensor.inner_product") @@ -144,7 +138,7 @@ impl ScalarFnVTable for InnerProduct { fn execute( &self, - options: &Self::Options, + _options: &Self::Options, args: &dyn ExecutionArgs, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -158,12 +152,12 @@ impl ScalarFnVTable for InnerProduct { let rhs_is_denorm = rhs_ref.is::>(); if lhs_is_denorm && rhs_is_denorm { - return self.execute_both_denorm(options, &lhs_ref, &rhs_ref, len, ctx); + return self.execute_both_denorm(&lhs_ref, &rhs_ref, len, ctx); } else if lhs_is_denorm || rhs_is_denorm { if rhs_is_denorm { (lhs_ref, rhs_ref) = (rhs_ref, lhs_ref); } - return self.execute_one_denorm(options, &lhs_ref, &rhs_ref, len, ctx); + return self.execute_one_denorm(&lhs_ref, &rhs_ref, len, ctx); } } @@ -225,7 +219,6 @@ impl InnerProduct { /// Both sides are `L2Denorm`: `inner_product = s_l * s_r * dot(n_l, n_r)`. fn execute_both_denorm( &self, - options: &ApproxOptions, lhs_ref: &ArrayRef, rhs_ref: &ArrayRef, len: usize, @@ -239,10 +232,9 @@ impl InnerProduct { let norms_l: PrimitiveArray = norms_l.execute(ctx)?; let norms_r: PrimitiveArray = norms_r.execute(ctx)?; - let dot: PrimitiveArray = - InnerProduct::try_new_array(options, normalized_l, normalized_r, len)? - .into_array() - .execute(ctx)?; + let dot: PrimitiveArray = InnerProduct::try_new_array(normalized_l, normalized_r, len)? + .into_array() + .execute(ctx)?; match_each_float_ptype!(dot.ptype(), |T| { let dots = dot.as_slice::(); @@ -260,7 +252,6 @@ impl InnerProduct { /// The caller must pass the denorm array as `denorm_ref` and the plain array as `plain_ref`. fn execute_one_denorm( &self, - options: &ApproxOptions, denorm_ref: &ArrayRef, plain_ref: &ArrayRef, len: usize, @@ -271,10 +262,9 @@ impl InnerProduct { let (normalized, norms) = extract_l2_denorm_children(denorm_ref); let denorm_norms: PrimitiveArray = norms.execute(ctx)?; - let dot: PrimitiveArray = - InnerProduct::try_new_array(options, normalized, plain_ref.clone(), len)? - .into_array() - .execute(ctx)?; + let dot: PrimitiveArray = InnerProduct::try_new_array(normalized, plain_ref.clone(), len)? + .into_array() + .execute(ctx)?; match_each_float_ptype!(dot.ptype(), |T| { let dots = dot.as_slice::(); @@ -308,13 +298,11 @@ mod tests { use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::scalar_fn::ScalarFn; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::utils::test_helpers::assert_close; @@ -326,7 +314,7 @@ mod tests { /// Evaluates inner product between two tensor arrays and returns the result as `Vec`. fn eval_inner_product(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { - let scalar_fn = ScalarFn::new(InnerProduct, ApproxOptions::Exact).erased(); + let scalar_fn = InnerProduct::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], len)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; @@ -404,7 +392,7 @@ mod tests { let rhs = tensor_array(&[2], &[7.0, 8.0, 9.0, 10.0, 11.0, 12.0])?; let lhs = MaskedArray::try_new(lhs, Validity::from_iter([true, false, true]))?.into_array(); - let scalar_fn = ScalarFn::new(InnerProduct, ApproxOptions::Exact).erased(); + let scalar_fn = InnerProduct::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; @@ -422,7 +410,7 @@ mod tests { fn rejects_non_extension_dtype() { let lhs = PrimitiveArray::from_iter([1.0_f64, 2.0]).into_array(); let rhs = PrimitiveArray::from_iter([3.0_f64, 4.0]).into_array(); - let result = InnerProduct::try_new_array(&ApproxOptions::Exact, lhs, rhs, 2); + let result = InnerProduct::try_new_array(lhs, rhs, 2); assert!(result.is_err()); } @@ -430,7 +418,7 @@ mod tests { fn rejects_mismatched_dtypes() -> VortexResult<()> { let lhs = tensor_array(&[2], &[1.0_f64, 2.0])?; let rhs = vector_array(2, &[3.0_f64, 4.0])?; - let result = InnerProduct::try_new_array(&ApproxOptions::Exact, lhs, rhs, 1); + let result = InnerProduct::try_new_array(lhs, rhs, 1); assert!(result.is_err()); Ok(()) } @@ -447,10 +435,7 @@ mod tests { let normalized = tensor_array(shape, normalized_elements)?; let norms = PrimitiveArray::from_iter(norms.iter().copied()).into_array(); let mut ctx = SESSION.create_execution_ctx(); - Ok( - L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, len, &mut ctx)? - .into_array(), - ) + Ok(L2Denorm::try_new_array(normalized, norms, len, &mut ctx)?.into_array()) } #[test] @@ -508,12 +493,10 @@ mod tests { let norms_l = PrimitiveArray::from_option_iter([Some(5.0f64), None]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let lhs = - L2Denorm::try_new_array(&ApproxOptions::Exact, normalized_l, norms_l, 2, &mut ctx)? - .into_array(); + let lhs = L2Denorm::try_new_array(normalized_l, norms_l, 2, &mut ctx)?.into_array(); let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; - let scalar_fn = ScalarFn::new(InnerProduct, ApproxOptions::Exact).erased(); + let scalar_fn = InnerProduct::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 2)?; let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 04a348ff7d8..d72940c8da8 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -23,6 +23,7 @@ use vortex_array::expr::and; use vortex_array::match_each_float_ptype; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; @@ -37,12 +38,11 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; use crate::matcher::AnyTensor; -use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_norm::L2Norm; use crate::utils::extract_flat_elements; use crate::utils::validate_tensor_float_input; -/// Re-applies L2 norms to a normalized tensor column. +/// Re-applies authoritative L2 norms to a normalized tensor column. /// /// Computes `normalized * norm` on each row over the flat backing buffer of each tensor-like type. /// @@ -51,17 +51,28 @@ use crate::utils::validate_tensor_float_input; /// /// The norms input must be a primitive float column with the same element type as the normalized /// tensor elements. +/// +/// [`L2Denorm`] is the norm-splitting wrapper used throughout the tensor crate. Callers that build +/// it through [`try_new_array`](Self::try_new_array) get an exact unit-norm invariant on the +/// `normalized` child. +/// +/// Advanced callers can also use [`new_array_unchecked`](Self::new_array_unchecked) to attach +/// authoritative stored norms to a lossy approximation of that child, such as quantized normalized +/// vectors. +/// +/// Downstream readthrough rules intentionally treat the stored norms and normalized child as the +/// encoding contract, even when that differs slightly from recomputing over fully decoded +/// coordinates. #[derive(Clone)] pub struct L2Denorm; impl L2Denorm { - /// Creates a new [`ScalarFn`] wrapping the L2 denormalization operation with the given - /// [`ApproxOptions`] controlling approximation behavior. + /// Creates a new [`ScalarFn`] wrapping the L2 denormalization operation. /// /// This is a low-level scalar-function descriptor constructor. To build a semantically valid /// [`L2Denorm`] array, prefer [`try_new_array`](Self::try_new_array). - pub fn new(options: &ApproxOptions) -> ScalarFn { - ScalarFn::new(L2Denorm, options.clone()) + pub fn new() -> ScalarFn { + ScalarFn::new(L2Denorm, EmptyOptions) } /// Constructs a validated [`ScalarFnArray`] that lazily re-applies `norms` to `normalized`. @@ -77,21 +88,15 @@ impl L2Denorm { /// Returns an error if the [`ScalarFnArray`] cannot be constructed (e.g. due to dtype /// mismatches) or if the `normalized` child is not row-wise L2-normalized. pub fn try_new_array( - options: &ApproxOptions, normalized: ArrayRef, norms: ArrayRef, len: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - let result = ScalarFnArray::try_new( - L2Denorm::new(options).erased(), - vec![normalized.clone(), norms.clone()], - len, - )?; - - validate_l2_denorm_children(normalized, norms, ctx)?; + validate_l2_denorm_children(&normalized, &norms, ctx)?; - Ok(result) + // SAFETY: We just validated that it is normalized. + unsafe { Self::new_array_unchecked(normalized, norms, len) } } /// Constructs an [`L2Denorm`] array without validating that the `normalized` child is actually @@ -104,24 +109,24 @@ impl L2Denorm { /// # Safety /// /// The caller must ensure the `normalized` child is semantically suitable for L2 - /// denormalization, which typically means every valid row is unit-norm or zero. Violating this - /// invariant will not cause memory unsafety, but may produce incorrect results. + /// denormalization. For exact wrappers, that means every valid row is unit-norm or zero. + /// + /// Lossy encodings may deliberately relax that invariant while still treating the stored norms + /// as authoritative. + /// + /// Violating the intended contract will not cause memory unsafety, but may produce incorrect + /// results. pub unsafe fn new_array_unchecked( - options: &ApproxOptions, normalized: ArrayRef, norms: ArrayRef, len: usize, ) -> VortexResult { - ScalarFnArray::try_new( - L2Denorm::new(options).erased(), - vec![normalized, norms], - len, - ) + ScalarFnArray::try_new(L2Denorm::new().erased(), vec![normalized, norms], len) } } impl ScalarFnVTable for L2Denorm { - type Options = ApproxOptions; + type Options = EmptyOptions; fn id(&self) -> ScalarFnId { ScalarFnId::new_ref("vortex.tensor.l2_denorm") @@ -258,8 +263,10 @@ impl ScalarFnVTable for L2Denorm { /// [`L2Norm`]'s validity propagation. When the [`L2Denorm`] wrapper is executed, its validity is /// `and(normalized_validity, norms_validity)`, which correctly identifies originally-null rows /// since the normalized child is all-valid and the norms child carries the original nulls. +/// +/// Because this helper computes exact norms first and then divides by those norms, the returned +/// `normalized` child satisfies the strict unit-norm invariant required by [`L2Denorm`]. pub fn normalize_as_l2_denorm( - options: &ApproxOptions, input: ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -267,7 +274,7 @@ pub fn normalize_as_l2_denorm( let tensor_match = validate_tensor_float_input(input.dtype())?; let tensor_flat_size = tensor_match.list_size(); - let norms_sfn = L2Norm::try_new_array(options, input.clone(), row_count)?; + let norms_sfn = L2Norm::try_new_array(input.clone(), row_count)?; let norms_array: ArrayRef = norms_sfn.into_array().execute(ctx)?; let norms: PrimitiveArray = norms_array.clone().execute(ctx)?; let norms_validity = norms.validity()?; @@ -307,8 +314,15 @@ pub fn normalize_as_l2_denorm( ) })?; - // TODO(connor): Need to figure out a way to not run validation. - L2Denorm::try_new_array(options, normalized, norms_array, row_count, ctx) + // SAFETY: + // - `norms_array` was produced by `L2Norm(input)`, so every stored norm is non-negative and + // null rows already carry null validity through that child. + // - For every valid row, we either emit all zeros when the norm is zero or divide every + // element by the exact stored norm, so the normalized child is unit-norm (or zero) by + // construction. + // - Null rows are zeroed out above to avoid propagating arbitrary physical storage values into + // downstream lossy encodings. + unsafe { L2Denorm::new_array_unchecked(normalized, norms_array, row_count) } } /// Rebuilds a tensor-like extension array from flat primitive elements. @@ -347,7 +361,7 @@ fn unit_norm_tolerance(element_ptype: PType) -> f64 { } /// Validates that every valid row of `input` is already L2-normalized (either length 1 or 0). -pub fn validate_l2_normalized_rows(input: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { +pub fn validate_l2_normalized_rows(input: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { validate_l2_normalized_rows_impl(input, None, ctx) } @@ -357,16 +371,16 @@ pub fn validate_l2_normalized_rows(input: ArrayRef, ctx: &mut ExecutionCtx) -> V /// - All vectors in the normalized array have length 1 or 0. /// - If the vector has a norm of 0, then the vector must be all 0s. fn validate_l2_denorm_children( - normalized: ArrayRef, - norms: ArrayRef, + normalized: &ArrayRef, + norms: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { validate_l2_normalized_rows_impl(normalized, Some(norms), ctx) } fn validate_l2_normalized_rows_impl( - normalized: ArrayRef, - norms: Option, + normalized: &ArrayRef, + norms: Option<&ArrayRef>, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { let row_count = normalized.len(); @@ -379,11 +393,19 @@ fn validate_l2_normalized_rows_impl( let tolerance = unit_norm_tolerance(element_ptype); let tensor_flat_size = tensor_match.list_size(); - let normalized: ExtensionArray = normalized.execute(ctx)?; + if let Some(norms) = norms { + vortex_ensure_eq!( + norms.dtype().as_ptype(), + element_ptype, + "L2Denorm norms ptype must match normalized element ptype" + ); + } + + let normalized: ExtensionArray = normalized.clone().execute(ctx)?; let normalized_validity = normalized.as_ref().validity()?; let flat = extract_flat_elements(normalized.storage_array(), tensor_flat_size, ctx)?; let norms = norms - .map(|norms| norms.execute::(ctx)) + .map(|norms| norms.clone().execute::(ctx)) .transpose()?; let combined_validity = match &norms { @@ -461,7 +483,6 @@ mod tests { use crate::fixed_shape::FixedShapeTensor; use crate::fixed_shape::FixedShapeTensorMetadata; - use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows; @@ -478,8 +499,7 @@ mod tests { /// Evaluates L2 denorm on a tensor/vector array and returns the executed array. fn eval_l2_denorm(normalized: ArrayRef, norms: ArrayRef, len: usize) -> VortexResult { let mut ctx = SESSION.create_execution_ctx(); - let result = - L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, len, &mut ctx)?; + let result = L2Denorm::try_new_array(normalized, norms, len, &mut ctx)?; result.into_array().execute(&mut ctx) } @@ -585,7 +605,7 @@ mod tests { let rhs = PrimitiveArray::from_iter([1.0f64, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, lhs, rhs, 2, &mut ctx); + let result = L2Denorm::try_new_array(lhs, rhs, 2, &mut ctx); assert!(result.is_err()); } @@ -595,7 +615,7 @@ mod tests { let rhs = PrimitiveArray::from_iter([1.0f64, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, lhs, rhs, 2, &mut ctx); + let result = L2Denorm::try_new_array(lhs, rhs, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -606,7 +626,7 @@ mod tests { let rhs = PrimitiveArray::from_iter([1.0f64, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, lhs, rhs, 2, &mut ctx); + let result = L2Denorm::try_new_array(lhs, rhs, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -617,7 +637,7 @@ mod tests { let rhs = PrimitiveArray::from_iter([1.0f32, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, lhs, rhs, 2, &mut ctx); + let result = L2Denorm::try_new_array(lhs, rhs, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -626,8 +646,8 @@ mod tests { fn validate_l2_normalized_rows_accepts_normalized_f16_input() -> VortexResult<()> { let input = f16_vector_array(2, &[3.0, 4.0, 0.0, 0.0])?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input, &mut ctx)?; - validate_l2_normalized_rows(roundtrip.child_at(0).clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input, &mut ctx)?; + validate_l2_normalized_rows(&roundtrip.child_at(0).clone(), &mut ctx)?; Ok(()) } @@ -635,7 +655,7 @@ mod tests { fn validate_l2_normalized_rows_rejects_unnormalized_input() -> VortexResult<()> { let input = vector_array(2, &[3.0, 4.0, 1.0, 0.0])?; let mut ctx = SESSION.create_execution_ctx(); - let result = validate_l2_normalized_rows(input, &mut ctx); + let result = validate_l2_normalized_rows(&input, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -646,7 +666,7 @@ mod tests { let norms = PrimitiveArray::from_iter([5.0f64, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, 2, &mut ctx); + let result = L2Denorm::try_new_array(normalized, norms, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -657,7 +677,7 @@ mod tests { let norms = PrimitiveArray::from_iter([0.0f64, 0.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, 2, &mut ctx); + let result = L2Denorm::try_new_array(normalized, norms, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -668,7 +688,7 @@ mod tests { let norms = PrimitiveArray::from_iter([1.0f64, -1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); - let result = L2Denorm::try_new_array(&ApproxOptions::Exact, normalized, norms, 2, &mut ctx); + let result = L2Denorm::try_new_array(normalized, norms, 2, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -678,8 +698,7 @@ mod tests { let normalized = vector_array(2, &[3.0, 4.0, 1.0, 0.0])?; let norms = PrimitiveArray::from_iter([5.0f64, 1.0]).into_array(); - let result = - unsafe { L2Denorm::new_array_unchecked(&ApproxOptions::Exact, normalized, norms, 2) }; + let result = unsafe { L2Denorm::new_array_unchecked(normalized, norms, 2) }; assert!(result.is_ok()); Ok(()) } @@ -688,7 +707,7 @@ mod tests { fn normalize_as_l2_denorm_roundtrips_vectors() -> VortexResult<()> { let input = vector_array(3, &[3.0, 4.0, 0.0, 0.0, 0.0, 0.0])?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input.clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let actual = roundtrip.into_array().execute(&mut ctx)?; assert_tensor_arrays_eq(actual, input)?; @@ -699,7 +718,7 @@ mod tests { fn normalize_as_l2_denorm_roundtrips_fixed_shape_tensors() -> VortexResult<()> { let input = tensor_array(&[2, 2], &[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0])?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input.clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let actual = roundtrip.into_array().execute(&mut ctx)?; assert_tensor_arrays_eq(actual, input)?; @@ -710,7 +729,7 @@ mod tests { fn normalize_as_l2_denorm_supports_constant_tensors() -> VortexResult<()> { let input = constant_tensor_array(&[2], &[3.0, 4.0], 3)?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input.clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let actual = roundtrip.into_array().execute(&mut ctx)?; assert_tensor_arrays_eq(actual, input)?; @@ -721,7 +740,7 @@ mod tests { fn normalize_as_l2_denorm_supports_constant_vectors() -> VortexResult<()> { let input = constant_vector_array(&[3.0, 4.0], 2)?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input.clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let actual = roundtrip.into_array().execute(&mut ctx)?; assert_tensor_arrays_eq(actual, input)?; @@ -732,7 +751,7 @@ mod tests { fn normalize_as_l2_denorm_uses_zero_rows_for_zero_norms() -> VortexResult<()> { let input = vector_array(2, &[0.0, 0.0, 3.0, 4.0])?; let mut ctx = SESSION.create_execution_ctx(); - let roundtrip = normalize_as_l2_denorm(&ApproxOptions::Exact, input.clone(), &mut ctx)?; + let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let normalized: ExtensionArray = roundtrip.child_at(0).clone().execute(&mut ctx)?; let storage: FixedSizeListArray = normalized.storage_array().clone().execute(&mut ctx)?; let elements: PrimitiveArray = storage.elements().clone().execute(&mut ctx)?; diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 2632c07cffd..170ac800d10 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -22,6 +22,7 @@ use vortex_array::expr::Expression; use vortex_array::match_each_float_ptype; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; @@ -32,7 +33,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use crate::matcher::AnyTensor; -use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::utils::extract_flat_elements; use crate::utils::validate_tensor_float_input; @@ -43,14 +43,18 @@ use crate::utils::validate_tensor_float_input; /// /// The input must be a tensor-like extension array with a float element type. The output is a float /// column of the same float type. +/// +/// When the input is wrapped in [`L2Denorm`], this operator treats the stored norms as +/// authoritative. For lossy encodings such as TurboQuant, that means `L2Norm` may intentionally +/// read the stored norms instead of re-deriving them from fully decoded coordinates. That behavior +/// is part of the lossy storage contract, not a separate lossy-compute mode. #[derive(Clone)] pub struct L2Norm; impl L2Norm { - /// Creates a new [`ScalarFn`] wrapping the L2 norm operation with the given [`ApproxOptions`] - /// controlling approximation behavior. - pub fn new(options: &ApproxOptions) -> ScalarFn { - ScalarFn::new(L2Norm, options.clone()) + /// Creates a new [`ScalarFn`] wrapping the L2 norm operation. + pub fn new() -> ScalarFn { + ScalarFn::new(L2Norm, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the L2 norm over `child`. @@ -59,17 +63,13 @@ impl L2Norm { /// /// Returns an error if the [`ScalarFnArray`] cannot be constructed (e.g. due to dtype /// mismatches). - pub fn try_new_array( - options: &ApproxOptions, - child: ArrayRef, - len: usize, - ) -> VortexResult { - ScalarFnArray::try_new(L2Norm::new(options).erased(), vec![child], len) + pub fn try_new_array(child: ArrayRef, len: usize) -> VortexResult { + ScalarFnArray::try_new(L2Norm::new().erased(), vec![child], len) } } impl ScalarFnVTable for L2Norm { - type Options = ApproxOptions; + type Options = EmptyOptions; fn id(&self) -> ScalarFnId { ScalarFnId::from("vortex.tensor.l2_norm") @@ -122,9 +122,9 @@ impl ScalarFnVTable for L2Norm { let tensor_flat_size = tensor_match.list_size(); let element_ptype = tensor_match.element_ptype(); - // L2Norm(L2Denorm(normalized, norms)) == norms, since normalized vectors have unit norm - // and L2 norms are non-negative. This avoids decompressing the TQ child just to recompute - // norms that are already stored. + // L2Norm(L2Denorm(normalized, norms)) is defined to read back the authoritative stored + // norms. Exact callers of lossy encodings like TurboQuant opt into that storage semantics + // instead of forcing a decode-and-recompute path here. if let Some(sfn) = input_ref.as_opt::() && sfn.scalar_fn().as_opt::().is_some() { @@ -195,13 +195,11 @@ mod tests { use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::scalar_fn::ScalarFn; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::scalar_fns::ApproxOptions; use crate::scalar_fns::l2_norm::L2Norm; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::tensor_array; @@ -212,7 +210,7 @@ mod tests { /// Evaluates L2 norm on a tensor/vector array and returns the result as `Vec`. fn eval_l2_norm(input: ArrayRef, len: usize) -> VortexResult> { - let scalar_fn = ScalarFn::new(L2Norm, ApproxOptions::Exact).erased(); + let scalar_fn = L2Norm::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![input], len)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; @@ -267,7 +265,7 @@ mod tests { let arr = tensor_array(&[2], &[3.0, 4.0, 0.0, 0.0])?; let arr = MaskedArray::try_new(arr, Validity::from_iter([true, false]))?.into_array(); - let scalar_fn = ScalarFn::new(L2Norm, ApproxOptions::Exact).erased(); + let scalar_fn = L2Norm::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![arr], 2)?; let mut ctx = SESSION.create_execution_ctx(); let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; diff --git a/vortex-tensor/src/scalar_fns/mod.rs b/vortex-tensor/src/scalar_fns/mod.rs index 32db845a6e2..1d1a362c8af 100644 --- a/vortex-tensor/src/scalar_fns/mod.rs +++ b/vortex-tensor/src/scalar_fns/mod.rs @@ -3,40 +3,8 @@ //! Scalar function expressions defined on tensor and tensor-like extension types. -use std::fmt; - pub mod cosine_similarity; pub mod inner_product; pub mod l2_denorm; pub mod l2_norm; - -/// Options for tensor-related expressions that might have error. -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] -pub enum ApproxOptions { - /// Computes the exact result. - #[default] - Exact, - /// Allows approximate results. - Approximate, -} - -impl ApproxOptions { - /// Returns `true` if the option is [`Exact`](Self::Exact). - pub fn is_exact(&self) -> bool { - matches!(self, Self::Exact) - } - - /// Returns `true` if the option is [`Approximate`](Self::Approximate). - pub fn is_approx(&self) -> bool { - matches!(self, Self::Approximate) - } -} - -impl fmt::Display for ApproxOptions { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Exact => write!(f, "Exact"), - Self::Approximate => write!(f, "Approximate"), - } - } -} +pub mod sorf_transform; diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs new file mode 100644 index 00000000000..96e6af414ca --- /dev/null +++ b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! SORF inverse transform scalar function. +//! +//! SORF (Structured Orthogonal Random Features, [Yu et al. 2016][sorf-paper]) is a fast structured +//! approximation to a random orthogonal matrix. It composes random sign diagonals with the +//! Walsh-Hadamard transform to achieve O(d log d) matrix-vector products instead of the O(d^2) cost +//! of a dense orthogonal matrix. +//! +//! This module wraps a [`Vector`] extension array whose dimension is the padded SORF dimension +//! (e.g. a `Vector` wrapping `FSL(Dict(codes, centroids))`) and applies the inverse SORF transform +//! at execution time, producing a [`Vector`] extension array with the original (pre-padding) +//! dimensionality. +//! +//! The transform parameters are stored as a deterministic seed in [`SorfOptions`], so the +//! [`SorfMatrix`] is reconstructed cheaply at decode time. Sign diagonals are defined by Vortex's +//! frozen local SplitMix64 stream contract rather than by an external RNG crate. +//! +//! # Input element type: `f32` only (TODO(connor): for now...) +//! +//! The child [`Vector`] **must** have `f32` storage elements. This is a hard constraint that is +//! enforced by `SorfTransform`'s `return_dtype` check. Callers with `f16` or `f64` source data need +//! to cast to `f32` before wrapping in a [`Vector`] and handing it to SorfTransform. +//! +//! The reason for this constraint is that TurboQuant (the only production caller today) stores its +//! dictionary centroids as `f32`, and the SORF transform itself operates internally in `f32`. +//! +//! Supporting other float storage types would require an implicit up-/down-cast that we do not yet +//! want to bake into SorfTransform. This restriction is intentional and may be relaxed in the +//! future, but today it is load-bearing. +//! +//! # Output element type +//! +//! The output [`Vector`]'s element type is whatever [`SorfOptions::element_ptype`] is set to. It +//! does **not** have to match the child's `f32` storage: we apply an explicit `f32 -> T` cast +//! while materializing the output. This lets SorfTransform hand its result directly to a +//! downstream consumer (e.g. [`L2Denorm`](crate::scalar_fns::l2_denorm::L2Denorm)) whose +//! element-type expectation may differ from the `f32` the transform operated on internally. +//! +//! [sorf-paper]: https://proceedings.neurips.cc/paper_files/paper/2016/file/53adaf494dc89ef7196d73636eb2451b-Paper.pdf +//! [`Vector`]: crate::vector::Vector + +use std::fmt; +use std::fmt::Formatter; + +use vortex_array::ArrayRef; +use vortex_array::arrays::ScalarFnArray; +use vortex_array::dtype::PType; +use vortex_array::scalar_fn::ScalarFn; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; + +mod rotation; +mod splitmix64; +pub use rotation::SorfMatrix; + +mod vtable; + +/// Inverse SORF orthogonal transform scalar function. +/// +/// Takes a [`Vector`](crate::vector::Vector) extension child at the padded dimension with `f32` +/// storage, applies the inverse structured Walsh-Hadamard orthogonal transform, truncates to the +/// original (pre-padding) dimension, casts element-wise to [`SorfOptions::element_ptype`], and +/// wraps the result in a new [`Vector`](crate::vector::Vector) extension array. +/// +/// See the [module-level docs](crate::scalar_fns::sorf_transform) for the rationale behind the +/// `f32`-only input constraint. +#[derive(Clone)] +pub struct SorfTransform; + +/// Options for the SORF inverse transform scalar function. +/// +/// Stored in the [`ScalarFnArray`] and used to deterministically reconstruct the +/// [`SorfMatrix`] at decode time. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct SorfOptions { + /// Seed used to generate the structured sign diagonals via Vortex's frozen SplitMix64 stream. + pub seed: u64, + /// Number of sign-diagonal + WHT rounds in the structured orthogonal transform. + pub num_rounds: u8, + /// Original vector dimension (before power-of-2 padding). The output + /// [`Vector`](crate::vector::Vector) has this dimension. + pub dimension: u32, + /// Element type of the output [`Vector`](crate::vector::Vector). The child input must always + /// be `f32`, but the output can be any float type (`F16`, `F32`, `F64`); the final + /// `f32 -> element_ptype` cast happens while building the output. + pub element_ptype: PType, +} + +impl SorfTransform { + /// Creates a new [`ScalarFn`] wrapping the SORF inverse transform with the given options. + pub fn new(options: &SorfOptions) -> ScalarFn { + ScalarFn::new(SorfTransform, options.clone()) + } + + /// Constructs a validated [`ScalarFnArray`] that lazily applies the inverse SORF transform. + /// + /// The `child` must be a [`Vector`] extension array (or an array that executes to one) with: + /// + /// - dimension equal to `padded_dim` (i.e. `options.dimension.next_power_of_two()`), and + /// - `f32` storage elements. This is a hard requirement today; see the + /// [module-level docs](crate::scalar_fns::sorf_transform) for the rationale. + /// + /// The output [`Vector`] has dimension `options.dimension` and element type + /// `options.element_ptype`. + /// + /// [`Vector`]: crate::vector::Vector + pub fn try_new_array( + options: &SorfOptions, + child: ArrayRef, + len: usize, + ) -> VortexResult { + validate_sorf_options(options)?; + + ScalarFnArray::try_new(SorfTransform::new(options).erased(), vec![child], len) + } +} + +/// Checks that the SORF configuration is valid. +pub(crate) fn validate_sorf_options(options: &SorfOptions) -> VortexResult<()> { + vortex_ensure!( + options.num_rounds >= 1, + "SorfTransform num_rounds must be >= 1, got {}", + options.num_rounds + ); + vortex_ensure!( + options.element_ptype.is_float(), + "SorfTransform element_ptype must be a float type, got {}", + options.element_ptype + ); + Ok(()) +} + +impl fmt::Display for SorfOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "SorfOptions(seed={}, rounds={}, dim={}, ptype={})", + self.seed, self.num_rounds, self.dimension, self.element_ptype + ) + } +} + +#[cfg(test)] +mod tests; diff --git a/vortex-tensor/src/encodings/turboquant/array/rotation.rs b/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs similarity index 65% rename from vortex-tensor/src/encodings/turboquant/array/rotation.rs rename to vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs index 57c236c821a..9279a35259d 100644 --- a/vortex-tensor/src/encodings/turboquant/array/rotation.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs @@ -1,15 +1,31 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Deterministic random rotation for TurboQuant. +//! SORF (Structured Orthogonal Random Features) orthogonal transform. //! -//! The TurboQuant paper analyzes a full random orthogonal rotation. The current implementation -//! uses a cheaper structured Walsh-Hadamard-based surrogate instead of a dense d x d matrix. +//! Implements the SORF construction from [Yu et al. 2016][sorf-paper]: a fast structured +//! approximation to a random orthogonal matrix using random sign diagonals interleaved with the +//! Fast Walsh-Hadamard Transform (FWHT). //! -//! Concretely, this applies three rounds of random sign diagonals interleaved with the Fast -//! Walsh-Hadamard Transform (FWHT): `D3 * H * D2 * H * D1 * H`, followed by normalization. This is -//! a SORF-style structured approximation to a random orthogonal matrix, chosen for O(d log d) -//! encode/decode cost and compact serialized parameters. +//! For `k` rounds, the transform is `norm * H * D_k * ... * H * D_1 * x`, where `D_1` is the +//! first sign diagonal applied. The number of rounds is configurable (typically 3). Each round +//! applies a random sign diagonal `D_i` and then the Hadamard matrix `H`, giving O(d log d) cost +//! per matrix-vector product instead of the O(d^2) cost of a dense orthogonal matrix. +//! +//! Vortex defines those sign diagonals using a frozen local SplitMix64 stream rather than an +//! external RNG crate. The contract is: +//! +//! - state is a single `u64` seed, +//! - each `next_u64()` call uses the SplitMix64 reference algorithm with wrapping `u64` +//! arithmetic, +//! - signs are generated in round-major, block-major order, +//! - each generated `u64` contributes 64 signs in least-significant-bit-first order, +//! - bit `1` means `+1` and bit `0` means `-1`. +//! +//! This makes SORF sign generation stable as a Vortex format contract even if external RNG +//! implementations change. +//! +//! [sorf-paper]: https://proceedings.neurips.cc/paper_files/paper/2016/file/53adaf494dc89ef7196d73636eb2451b-Paper.pdf //! //! The FWHT exploits the Kronecker product structure of the Hadamard matrix (`H_n = H_2 (x) H_2 //! (x) ... (x) H_2`, with `log2(n)` factors) to compute the matrix-vector product in O(n log n) @@ -26,18 +42,22 @@ //! floating-point multiply, which avoids FP dependency chains and auto-vectorizes into //! `vpxor`/`veor`. -use rand::RngExt; -use rand::SeedableRng; -use rand::rngs::StdRng; use vortex_error::VortexResult; use vortex_error::vortex_ensure; +use super::splitmix64::SplitMix64; + /// IEEE 754 sign bit mask for f32. const F32_SIGN_BIT: u32 = 0x8000_0000; -/// A Walsh-Hadamard-based structured surrogate for a random orthogonal rotation. -pub struct RotationMatrix { - /// Flat XOR masks for all `num_rounds` diagonal matrices, total length `num_rounds * padded_dim`. +/// A Walsh-Hadamard-based structured orthogonal transform matrix. +/// +/// All computation is done in f32. The sign diagonals are stored as IEEE 754 XOR masks on +/// f32 bit patterns, and the Walsh-Hadamard butterfly operates on `&mut [f32]` slices. +pub struct SorfMatrix { + /// Flat XOR masks for all `num_rounds` diagonal matrices, total length + /// `num_rounds * padded_dim`. + /// /// Indexed as `round * padded_dim + i`. `0x00000000` = multiply by +1 (no-op), `0x80000000` = /// multiply by -1 (flip sign bit). sign_masks: Vec, @@ -49,22 +69,25 @@ pub struct RotationMatrix { norm_factor: f32, } -impl RotationMatrix { - /// Create a new structured Walsh-Hadamard-based rotation from a deterministic seed. +impl SorfMatrix { + /// Create a new structured Walsh-Hadamard-based orthogonal transform from a deterministic + /// seed. + /// + /// The seed is expanded using Vortex's frozen local SplitMix64 stream. Signs are generated in + /// round-major, block-major order, with each `u64` contributing 64 sign bits in + /// least-significant-bit-first order. pub fn try_new(seed: u64, dimension: usize, num_rounds: usize) -> VortexResult { vortex_ensure!(num_rounds >= 1, "num_rounds must be >= 1, got {num_rounds}"); let padded_dim = dimension.next_power_of_two(); - let mut rng = StdRng::seed_from_u64(seed); - - let mut sign_masks = Vec::with_capacity(num_rounds * padded_dim); - for _ in 0..num_rounds { - sign_masks.extend(gen_random_sign_masks(&mut rng, padded_dim)); - } + let sign_masks = gen_sign_masks_from_seed(seed, padded_dim, num_rounds); + // Compute in f64 for precision, then store as f32 since the WHT operates on f32 buffers. + // The result is always in (0, 1] for any valid padded_dim >= 2 and num_rounds >= 1, so + // the f64 -> f32 cast is a precision loss only -- it cannot overflow to infinity. #[expect( clippy::cast_possible_truncation, - reason = "Intentional f64 -> f32 truncation for normalization factor." + reason = "the norm factor is in (0, 1] so the f64 -> f32 cast cannot overflow" )] let norm_factor = (padded_dim as f64).powf(-(num_rounds as f64) / 2.0) as f32; @@ -76,7 +99,14 @@ impl RotationMatrix { }) } - /// Apply forward rotation: `output = R(input)`. + /// Returns the padded dimension (next power of 2 >= dim). + /// + /// All `rotate`/`inverse_rotate` buffers must be this length. + pub fn padded_dim(&self) -> usize { + self.padded_dim + } + + /// Apply the forward orthogonal transform: `output = R(input)`. /// /// Both `input` and `output` must have length [`padded_dim()`](Self::padded_dim). The caller is /// responsible for zero-padding input beyond `dim` positions. @@ -88,7 +118,7 @@ impl RotationMatrix { self.apply_srht(output); } - /// Apply inverse rotation: `output = R⁻¹(input)`. + /// Apply the inverse orthogonal transform: `output = R⁻¹(input)`. /// /// Both `input` and `output` must have length `padded_dim()`. pub fn inverse_rotate(&self, input: &[f32], output: &mut [f32]) { @@ -99,19 +129,7 @@ impl RotationMatrix { self.apply_inverse_srht(output); } - /// Returns the number of sign-diagonal + WHT rounds. - pub fn num_rounds(&self) -> usize { - self.num_rounds - } - - /// Returns the padded dimension (next power of 2 >= dim). - /// - /// All rotate/inverse_rotate buffers must be this length. - pub fn padded_dim(&self) -> usize { - self.padded_dim - } - - /// Apply the structured rotation: `D_k · H · ... · D₁ · H · x`, with normalization. + /// Apply the forward structured transform: `norm · H · D_k · ... · H · D₁ · x`. fn apply_srht(&self, buf: &mut [f32]) { for round in 0..self.num_rounds { let offset = round * self.padded_dim; @@ -123,9 +141,9 @@ impl RotationMatrix { buf.iter_mut().for_each(|val| *val *= norm); } - /// Apply the inverse structured rotation. + /// Apply the inverse structured transform. /// - /// Forward is: `norm · H · D_k · H · ... · D₁ · H`. + /// Forward is: `norm · H · D_k · ... · H · D₁`. /// Inverse is: `norm · D₁ · H · ... · D_k · H`. fn apply_inverse_srht(&self, buf: &mut [f32]) { for round in (0..self.num_rounds).rev() { @@ -144,6 +162,7 @@ impl RotationMatrix { /// Convention: `1` = positive (+1), `0` = negative (-1). The output has length /// `num_rounds * padded_dim` and is suitable for bitpacking via FastLanes /// `bitpack_encode(..., 1, None)`. + #[cfg(test)] pub fn export_inverse_signs_u8(&self) -> Vec { let total = self.num_rounds * self.padded_dim; let mut out = Vec::with_capacity(total); @@ -158,7 +177,7 @@ impl RotationMatrix { out } - /// Reconstruct a [`RotationMatrix`] from unpacked `u8` 0/1 values. + /// Reconstruct a [`SorfMatrix`] from unpacked `u8` 0/1 values. /// /// The input must have length `num_rounds * padded_dim` with signs in inverse application /// order `[D_k | ... | D₁]` (as produced by [`export_inverse_signs_u8`]). Convention: @@ -166,6 +185,7 @@ impl RotationMatrix { /// /// This is the decode-time reconstruction path: FastLanes SIMD-unpacks the stored /// [`BitPackedArray`] into `&[u8]`, which is passed here. + #[cfg(test)] pub fn from_u8_slice( signs_u8: &[u8], dimension: usize, @@ -196,9 +216,11 @@ impl RotationMatrix { } } + // Same norm factor computation as `try_new`. See the comment there for why this cast + // cannot overflow. #[expect( clippy::cast_possible_truncation, - reason = "Intentional f64 -> f32 truncation for normalization factor." + reason = "the norm factor is in (0, 1] so the f64 -> f32 cast cannot overflow" )] let norm_factor = (padded_dim as f64).powf(-(num_rounds as f64) / 2.0) as f32; @@ -211,24 +233,40 @@ impl RotationMatrix { } } -/// Generate a vector of random XOR sign masks. -fn gen_random_sign_masks(rng: &mut StdRng, len: usize) -> Vec { - (0..len) - .map(|_| { - if rng.random_bool(0.5) { - 0u32 // +1: no-op - } else { - F32_SIGN_BIT // -1: flip sign bit - } - }) - .collect() +/// Generate XOR sign masks from the frozen local SplitMix64 stream. +/// +/// Signs are produced in round-major, block-major order. For each block we call +/// [`SplitMix64::next_u64`] exactly once and unpack its bits from least significant to most +/// significant. Bit `1` means positive sign / `0x00000000`; bit `0` means negative sign / +/// [`F32_SIGN_BIT`]. +fn gen_sign_masks_from_seed(seed: u64, padded_dim: usize, num_rounds: usize) -> Vec { + let mut rng = SplitMix64::new(seed); + let mut sign_masks = Vec::with_capacity(num_rounds * padded_dim); + + for _round in 0..num_rounds { + for base_idx in (0..padded_dim).step_by(64) { + let word = rng.next_u64(); + let bits_in_block = (padded_dim - base_idx).min(64); + sign_masks.extend((0..bits_in_block).map(|bit_idx| sign_mask_from_word(word, bit_idx))); + } + } + + sign_masks +} + +/// Convert one bit from a SplitMix64 output word into an XOR sign mask. +fn sign_mask_from_word(word: u64, bit_idx: usize) -> u32 { + if ((word >> bit_idx) & 1) != 0 { + 0u32 + } else { + F32_SIGN_BIT + } } /// Apply sign masks via XOR on the IEEE 754 sign bit. /// /// This is branchless and auto-vectorizes into `vpxor` (x86) / `veor` (ARM). Equivalent to /// multiplying each element by +/-1.0, but avoids FP dependency chains. -#[inline] fn apply_signs_xor(buf: &mut [f32], masks: &[u32]) { for (val, &mask) in buf.iter_mut().zip(masks.iter()) { *val = f32::from_bits(val.to_bits() ^ mask); @@ -265,7 +303,6 @@ fn walsh_hadamard_transform(buf: &mut [f32]) { /// This is multiplication by the 2x2 Hadamard kernel `H_2 = [[1, 1], [1, -1]]` on each element /// pair. Factored into a separate function so LLVM can see the slice lengths match and /// auto-vectorize. -#[inline(always)] fn butterfly(lo: &mut [f32], hi: &mut [f32]) { debug_assert_eq!(lo.len(), hi.len()); for (a, b) in lo.iter_mut().zip(hi.iter_mut()) { @@ -282,11 +319,18 @@ mod tests { use vortex_error::VortexResult; use super::*; + use crate::scalar_fns::sorf_transform::splitmix64::SplitMix64; + + fn unpack_sign_bits(word: u64, count: usize) -> Vec { + (0..count) + .map(|bit_idx| u8::from(((word >> bit_idx) & 1) != 0)) + .collect() + } #[test] fn deterministic_from_seed() -> VortexResult<()> { - let r1 = RotationMatrix::try_new(42, 64, 3)?; - let r2 = RotationMatrix::try_new(42, 64, 3)?; + let r1 = SorfMatrix::try_new(42, 64, 3)?; + let r2 = SorfMatrix::try_new(42, 64, 3)?; let pd = r1.padded_dim(); let mut input = vec![0.0f32; pd]; @@ -303,6 +347,48 @@ mod tests { Ok(()) } + #[test] + fn export_inverse_signs_matches_golden_words() -> VortexResult<()> { + let rot = SorfMatrix::try_new(42, 64, 2)?; + let actual = rot.export_inverse_signs_u8(); + let mut rng = SplitMix64::new(42); + let round0_word = rng.next_u64(); + let round1_word = rng.next_u64(); + + let mut expected = Vec::with_capacity(128); + expected.extend(unpack_sign_bits(round1_word, 64)); + expected.extend(unpack_sign_bits(round0_word, 64)); + + assert_eq!(actual, expected); + Ok(()) + } + + #[test] + fn one_word_generates_64_signs_lsb_first() { + let masks = gen_sign_masks_from_seed(42, 64, 1); + assert_eq!(masks.len(), 64); + + let mut rng = SplitMix64::new(42); + let word = rng.next_u64(); + let expected: Vec<_> = (0..64) + .map(|bit_idx| sign_mask_from_word(word, bit_idx)) + .collect(); + assert_eq!(masks, expected); + } + + #[test] + fn tail_block_uses_only_required_bits() { + let masks = gen_sign_masks_from_seed(42, 32, 1); + assert_eq!(masks.len(), 32); + + let mut rng = SplitMix64::new(42); + let word = rng.next_u64(); + let expected: Vec<_> = (0..32) + .map(|bit_idx| sign_mask_from_word(word, bit_idx)) + .collect(); + assert_eq!(masks, expected); + } + /// Verify roundtrip is exact to f32 precision across many dimensions and round counts, /// including non-power-of-two dimensions that require padding. #[rstest] @@ -318,7 +404,7 @@ mod tests { #[case(768, 3)] #[case(1024, 3)] fn roundtrip_exact(#[case] dim: usize, #[case] num_rounds: usize) -> VortexResult<()> { - let rot = RotationMatrix::try_new(42, dim, num_rounds)?; + let rot = SorfMatrix::try_new(42, dim, num_rounds)?; let padded_dim = rot.padded_dim(); let mut input = vec![0.0f32; padded_dim]; @@ -354,7 +440,7 @@ mod tests { #[case(128, 5)] #[case(768, 3)] fn preserves_norm(#[case] dim: usize, #[case] num_rounds: usize) -> VortexResult<()> { - let rot = RotationMatrix::try_new(7, dim, num_rounds)?; + let rot = SorfMatrix::try_new(7, dim, num_rounds)?; let padded_dim = rot.padded_dim(); let mut input = vec![0.0f32; padded_dim]; @@ -377,7 +463,7 @@ mod tests { Ok(()) } - /// Verify that export -> [`from_u8_slice`] produces identical rotation output. + /// Verify that export -> [`from_u8_slice`] produces identical transform output. #[rstest] #[case(64, 3)] #[case(128, 1)] @@ -388,11 +474,11 @@ mod tests { #[case] dim: usize, #[case] num_rounds: usize, ) -> VortexResult<()> { - let rot = RotationMatrix::try_new(42, dim, num_rounds)?; + let rot = SorfMatrix::try_new(42, dim, num_rounds)?; let padded_dim = rot.padded_dim(); let signs_u8 = rot.export_inverse_signs_u8(); - let rot2 = RotationMatrix::from_u8_slice(&signs_u8, dim, num_rounds)?; + let rot2 = SorfMatrix::from_u8_slice(&signs_u8, dim, num_rounds)?; let mut input = vec![0.0f32; padded_dim]; for i in 0..dim { @@ -403,12 +489,12 @@ mod tests { let mut out2 = vec![0.0f32; padded_dim]; rot.rotate(&input, &mut out1); rot2.rotate(&input, &mut out2); - assert_eq!(out1, out2, "Forward rotation mismatch after export/import"); + assert_eq!(out1, out2, "Forward transform mismatch after export/import"); rot.inverse_rotate(&out1, &mut out2); let mut out3 = vec![0.0f32; padded_dim]; rot2.inverse_rotate(&out1, &mut out3); - assert_eq!(out2, out3, "Inverse rotation mismatch after export/import"); + assert_eq!(out2, out3, "Inverse transform mismatch after export/import"); Ok(()) } diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/splitmix64.rs b/vortex-tensor/src/scalar_fns/sorf_transform/splitmix64.rs new file mode 100644 index 00000000000..23345cbcb7d --- /dev/null +++ b/vortex-tensor/src/scalar_fns/sorf_transform/splitmix64.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Frozen local SplitMix64 stream used to define SORF sign diagonals. +//! +//! This is a direct translation of the `splitmix64.c` reference implementation. The state is a +//! single `u64`, and `next_u64()` first adds [`SPLITMIX64_INCREMENT`] with wrapping arithmetic, +//! then applies the two reference mixing steps and final xor-shift. + +/// SplitMix64 additive constant from the reference implementation. +const SPLITMIX64_INCREMENT: u64 = 0x9E37_79B9_7F4A_7C15; + +/// First SplitMix64 mixing multiplier from the reference implementation. +const SPLITMIX64_MUL1: u64 = 0xBF58_476D_1CE4_E5B9; + +/// Second SplitMix64 mixing multiplier from the reference implementation. +const SPLITMIX64_MUL2: u64 = 0x94D0_49BB_1331_11EB; + +/// Frozen local SplitMix64 stream used to define SORF sign diagonals. +pub(crate) struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + pub(crate) fn new(seed: u64) -> Self { + Self { state: seed } + } + + pub(crate) fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(SPLITMIX64_INCREMENT); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(SPLITMIX64_MUL1); + z = (z ^ (z >> 27)).wrapping_mul(SPLITMIX64_MUL2); + z ^ (z >> 31) + } +} + +#[cfg(test)] +mod tests { + use super::SplitMix64; + + const SPLITMIX64_SEED0_GOLDEN: [u64; 4] = [ + 0xE220_A839_7B1D_CDAF, + 0x6E78_9E6A_A1B9_65F4, + 0x06C4_5D18_8009_454F, + 0xF88B_B8A8_724C_81EC, + ]; + + const SPLITMIX64_SEED42_GOLDEN: [u64; 4] = [ + 0xBDD7_3226_2FEB_6E95, + 0x28EF_E333_B266_F103, + 0x4752_6757_130F_9F52, + 0x581C_E1FF_0E4A_E394, + ]; + + #[test] + fn splitmix64_seed0_matches_golden_outputs() { + let mut rng = SplitMix64::new(0); + let actual: Vec<_> = (0..SPLITMIX64_SEED0_GOLDEN.len()) + .map(|_| rng.next_u64()) + .collect(); + assert_eq!(actual, SPLITMIX64_SEED0_GOLDEN); + } + + #[test] + fn splitmix64_seed42_matches_golden_outputs() { + let mut rng = SplitMix64::new(42); + let actual: Vec<_> = (0..SPLITMIX64_SEED42_GOLDEN.len()) + .map(|_| rng.next_u64()) + .collect(); + assert_eq!(actual, SPLITMIX64_SEED42_GOLDEN); + } +} diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs new file mode 100644 index 00000000000..99d97fbe87f --- /dev/null +++ b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Unit tests for the [`SorfTransform`] scalar function. + +#![allow(clippy::cast_possible_truncation)] + +use std::sync::Arc; +use std::sync::LazyLock; + +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::dict::DictArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::extension::EmptyMetadata; +use vortex_array::session::ArraySession; +use vortex_array::validity::Validity; +use vortex_buffer::BufferMut; +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +use super::SorfOptions; +use super::SorfTransform; +use super::rotation::SorfMatrix; +use crate::encodings::turboquant::centroids::compute_centroid_boundaries; +use crate::encodings::turboquant::centroids::find_nearest_centroid; +use crate::encodings::turboquant::centroids::get_centroids; +use crate::vector::Vector; + +static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + +/// Build a unit-normalized input vector array and forward-transform + quantize it, returning +/// `(input_f32, Vector(FSL(Dict(codes, centroids))), padded_dim)`. +/// +/// This mimics what the TurboQuant compression pipeline does, but directly, so we can test +/// `SorfTransform` in isolation. +fn forward_rotate_and_quantize( + dim: usize, + num_rows: usize, + seed: u64, + num_rounds: usize, + bit_width: u8, +) -> VortexResult<(Vec, ExtensionArray, usize)> { + // Build simple unit-normalized input vectors. + let mut input_f32 = vec![0.0f32; num_rows * dim]; + for row in 0..num_rows { + let mut norm_sq = 0.0f32; + for i in 0..dim { + let val = ((row * dim + i) as f32 + 1.0) * 0.01; + input_f32[row * dim + i] = val; + norm_sq += val * val; + } + let norm = norm_sq.sqrt(); + for i in 0..dim { + input_f32[row * dim + i] /= norm; + } + } + + let rotation = SorfMatrix::try_new(seed, dim, num_rounds)?; + let padded_dim = rotation.padded_dim(); + let centroids = get_centroids(padded_dim as u32, bit_width)?; + let boundaries = compute_centroid_boundaries(¢roids); + + let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); + let mut padded = vec![0.0f32; padded_dim]; + let mut rotated = vec![0.0f32; padded_dim]; + + for row in 0..num_rows { + padded[..dim].copy_from_slice(&input_f32[row * dim..(row + 1) * dim]); + padded[dim..].fill(0.0); + rotation.rotate(&padded, &mut rotated); + for j in 0..padded_dim { + all_indices.push(find_nearest_centroid(rotated[j], &boundaries)); + } + } + + let codes = PrimitiveArray::new::(all_indices.freeze(), Validity::NonNullable); + let mut centroids_buf = BufferMut::::with_capacity(centroids.len()); + centroids_buf.extend_from_slice(¢roids); + let centroids_arr = PrimitiveArray::new::(centroids_buf.freeze(), Validity::NonNullable); + let dict = DictArray::try_new(codes.into_array(), centroids_arr.into_array())?; + let fsl = FixedSizeListArray::try_new( + dict.into_array(), + padded_dim as u32, + Validity::NonNullable, + num_rows, + )?; + let padded_vector = wrap_as_vector(fsl, Validity::NonNullable)?; + + Ok((input_f32, padded_vector, padded_dim)) +} + +/// Wrap an FSL in a Vector extension, optionally re-tagging its validity. This is used by tests +/// that need to adjust top-level nullability of a padded vector child. +fn wrap_as_vector(fsl: FixedSizeListArray, validity: Validity) -> VortexResult { + let list_size = fsl.list_size(); + let num_rows = fsl.len(); + let elements = fsl.elements().clone(); + let fsl = FixedSizeListArray::try_new(elements, list_size, validity, num_rows)?; + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array())) +} + +/// Helper to build `SorfOptions` with common defaults. +fn default_options(dim: u32, seed: u64) -> SorfOptions { + SorfOptions { + seed, + num_rounds: 3, + dimension: dim, + element_ptype: PType::F32, + } +} + +/// Execute a `SorfTransform` array and return the decoded flat f32 elements. +fn execute_sorf( + options: &SorfOptions, + child: ExtensionArray, + num_rows: usize, +) -> VortexResult> { + let sorf = SorfTransform::try_new_array(options, child.into_array(), num_rows)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + let result_prim: PrimitiveArray = result_fsl.elements().clone().execute(&mut ctx)?; + Ok(result_prim.as_slice::().to_vec()) +} + +/// Build an empty `Vector` extension array wrapping an empty FSL. +fn empty_padded_vector(padded_dim: u32, validity: Validity) -> VortexResult { + let elements = PrimitiveArray::empty::(Nullability::NonNullable); + let fsl = FixedSizeListArray::try_new(elements.into_array(), padded_dim, validity, 0)?; + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array())) +} + +#[test] +fn roundtrip_recovery() -> VortexResult<()> { + let dim = 128; + let num_rows = 10; + let seed = 42u64; + let (input_f32, padded_vector, _) = forward_rotate_and_quantize(dim, num_rows, seed, 3, 8)?; + let options = default_options(dim as u32, seed); + let result = execute_sorf(&options, padded_vector, num_rows)?; + + assert_eq!(result.len(), num_rows * dim); + + // At 8-bit quantization, the reconstruction should be very close to the input. + for row in 0..num_rows { + let orig = &input_f32[row * dim..(row + 1) * dim]; + let recon = &result[row * dim..(row + 1) * dim]; + let err_sq: f32 = orig + .iter() + .zip(recon) + .map(|(&a, &b)| (a - b) * (a - b)) + .sum(); + let norm_sq: f32 = orig.iter().map(|&v| v * v).sum(); + assert!( + err_sq / norm_sq < 1e-3, + "row {row} MSE too high: {:.6}", + err_sq / norm_sq + ); + } + Ok(()) +} + +#[test] +fn empty_array_non_nullable() -> VortexResult<()> { + let dim = 128u32; + let padded_dim = dim.next_power_of_two(); + let options = default_options(dim, 42); + + // Build an empty Vector child. + let child = empty_padded_vector(padded_dim, Validity::NonNullable)?; + + let sorf = SorfTransform::try_new_array(&options, child.into_array(), 0)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + + assert_eq!(result.len(), 0); + + // Output should be non-nullable. + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + assert!(!result_fsl.dtype().is_nullable()); + + Ok(()) +} + +#[test] +fn empty_array_nullable() -> VortexResult<()> { + let dim = 128u32; + let padded_dim = dim.next_power_of_two(); + let options = default_options(dim, 42); + + // Build an empty but nullable Vector child. + let child = empty_padded_vector(padded_dim, Validity::from(Nullability::Nullable))?; + + let sorf = SorfTransform::try_new_array(&options, child.into_array(), 0)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + + assert_eq!(result.len(), 0); + + // Output should be nullable (matching the child). + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + assert!(result_fsl.dtype().is_nullable()); + + Ok(()) +} + +#[test] +fn nullable_validity_propagation() -> VortexResult<()> { + let dim = 128; + let num_rows = 4; + let seed = 42u64; + let (_, non_nullable_vector, padded_dim) = + forward_rotate_and_quantize(dim, num_rows, seed, 3, 8)?; + + // Re-wrap the underlying FSL with a validity mask: rows 0 and 2 are valid, rows 1 and 3 + // are null. + let validity = Validity::from_iter([true, false, true, false]); + let fsl_non_nullable: FixedSizeListArray = non_nullable_vector + .storage_array() + .clone() + .execute(&mut SESSION.create_execution_ctx())?; + let fsl_nullable = FixedSizeListArray::try_new( + fsl_non_nullable.elements().clone(), + padded_dim as u32, + validity.clone(), + num_rows, + )?; + let nullable_vector = wrap_as_vector(fsl_nullable, validity.clone())?; + + let options = default_options(dim as u32, seed); + let sorf = SorfTransform::try_new_array(&options, nullable_vector.into_array(), num_rows)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + + // The output FSL validity should match the input. + let output_validity = result_fsl.validity()?; + for row in 0..num_rows { + assert_eq!( + output_validity.is_valid(row)?, + validity.is_valid(row)?, + "validity mismatch at row {row}" + ); + } + + Ok(()) +} + +#[test] +fn dimension_truncation() -> VortexResult<()> { + // Use a non-power-of-2 dimension (padded 200 -> 256). + let dim = 200; + let num_rows = 3; + let seed = 42u64; + let (_, padded_vector, padded_dim) = forward_rotate_and_quantize(dim, num_rows, seed, 3, 8)?; + + assert_eq!(padded_dim, 256, "200 should pad to 256"); + + let options = default_options(dim as u32, seed); + let result = execute_sorf(&options, padded_vector, num_rows)?; + + // Output should have original dimension, not padded. + assert_eq!(result.len(), num_rows * dim); + + Ok(()) +} + +#[test] +fn return_dtype_is_vector_extension() -> VortexResult<()> { + let dim = 128u32; + let padded_dim = dim.next_power_of_two(); + let options = default_options(dim, 42); + + // Input must be a Vector extension dtype. + let child_elem_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let child_storage_dtype = DType::FixedSizeList( + Arc::new(child_elem_dtype), + padded_dim, + Nullability::NonNullable, + ); + let child_ext_dtype = ExtDType::::try_new(EmptyMetadata, child_storage_dtype)?.erased(); + let child_dtype = DType::Extension(child_ext_dtype); + + use vortex_array::scalar_fn::ScalarFnVTable; + let return_dtype = SorfTransform.return_dtype(&options, &[child_dtype])?; + + // Should be a Vector extension type. + let ext = return_dtype + .as_extension_opt() + .expect("return dtype should be an extension type"); + assert!(ext.metadata_opt::().is_some()); + + // Inner FSL should have the original (unpadded) dimension. + let DType::FixedSizeList(_, inner_dim, _) = ext.storage_dtype() else { + panic!("expected storage dtype to be FSL"); + }; + assert_eq!(*inner_dim, dim); + + Ok(()) +} + +#[test] +fn rejects_zero_rounds_at_construction() { + let options = SorfOptions { + seed: 42, + num_rounds: 0, + dimension: 128, + element_ptype: PType::F32, + }; + let elements = PrimitiveArray::from_iter([0.0f32; 128]).into_array(); + let child = FixedSizeListArray::try_new(elements, 128, Validity::NonNullable, 1) + .expect("test child should be valid"); + + let err = SorfTransform::try_new_array(&options, child.into_array(), 1) + .expect_err("zero rounds should be rejected at construction time"); + assert!(err.to_string().contains("num_rounds")); +} + +#[test] +fn rejects_non_float_output_ptype_at_construction() { + let options = SorfOptions { + seed: 42, + num_rounds: 3, + dimension: 128, + element_ptype: PType::U8, + }; + let elements = PrimitiveArray::from_iter([0.0f32; 128]).into_array(); + let child = FixedSizeListArray::try_new(elements, 128, Validity::NonNullable, 1) + .expect("test child should be valid"); + + let err = SorfTransform::try_new_array(&options, child.into_array(), 1) + .expect_err("non-float output ptypes should be rejected at construction time"); + assert!(err.to_string().contains("element_ptype")); +} + +#[test] +fn rejects_non_vector_extension_child_at_construction() { + let options = default_options(128, 42); + // A bare FSL child (not wrapped in a Vector extension) should be rejected. + let elements = PrimitiveArray::from_iter([0.0f32; 128]).into_array(); + let child = FixedSizeListArray::try_new(elements, 128, Validity::NonNullable, 1) + .expect("test child should be valid"); + + let err = SorfTransform::try_new_array(&options, child.into_array(), 1) + .expect_err("non-Vector-extension children should be rejected at construction time"); + assert!(err.to_string().contains("Vector extension")); +} + +#[test] +fn rejects_wrong_padded_dimension_at_construction() { + // Options say dimension=128 so padded_dim should be 128. Pass a Vector<256> instead. + let options = default_options(128, 42); + let elements = PrimitiveArray::from_iter([0.0f32; 256]).into_array(); + let fsl = FixedSizeListArray::try_new(elements, 256, Validity::NonNullable, 1) + .expect("test child should be valid"); + let child = wrap_as_vector(fsl, Validity::NonNullable).expect("wrap should succeed"); + + let err = SorfTransform::try_new_array(&options, child.into_array(), 1) + .expect_err("mismatched padded dimension should be rejected at construction time"); + assert!(err.to_string().contains("dimension")); +} + +#[test] +fn rejects_non_f32_child_storage_at_construction() { + // Options are valid and target f32 output. Pass a Vector<128> whose storage is f16 instead + // of f32 -- SorfTransform's f32-only input constraint should reject this. + let options = default_options(128, 42); + let elements = PrimitiveArray::from_iter([half::f16::from_f32(0.0); 128]).into_array(); + let fsl = FixedSizeListArray::try_new(elements, 128, Validity::NonNullable, 1) + .expect("test child should be valid"); + let child = wrap_as_vector(fsl, Validity::NonNullable).expect("wrap should succeed"); + + let err = SorfTransform::try_new_array(&options, child.into_array(), 1) + .expect_err("non-f32 Vector storage should be rejected at construction time"); + assert!(err.to_string().contains("f32")); +} + +#[test] +fn f16_output_type() -> VortexResult<()> { + let dim = 128; + let num_rows = 3; + let seed = 42u64; + let (_, padded_vector, _) = forward_rotate_and_quantize(dim, num_rows, seed, 3, 8)?; + + let options = SorfOptions { + seed, + num_rounds: 3, + dimension: dim as u32, + element_ptype: PType::F16, + }; + let sorf = SorfTransform::try_new_array(&options, padded_vector.into_array(), num_rows)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + let result_prim: PrimitiveArray = result_fsl.elements().clone().execute(&mut ctx)?; + + assert_eq!(result_prim.ptype(), PType::F16); + assert_eq!(result_prim.as_slice::().len(), num_rows * dim); + + Ok(()) +} + +#[test] +fn f64_output_type() -> VortexResult<()> { + let dim = 128; + let num_rows = 3; + let seed = 42u64; + let (_, padded_vector, _) = forward_rotate_and_quantize(dim, num_rows, seed, 3, 8)?; + + let options = SorfOptions { + seed, + num_rounds: 3, + dimension: dim as u32, + element_ptype: PType::F64, + }; + let sorf = SorfTransform::try_new_array(&options, padded_vector.into_array(), num_rows)?; + let mut ctx = SESSION.create_execution_ctx(); + let result: ExtensionArray = sorf.into_array().execute(&mut ctx)?; + let result_fsl: FixedSizeListArray = result.storage_array().clone().execute(&mut ctx)?; + let result_prim: PrimitiveArray = result_fsl.elements().clone().execute(&mut ctx)?; + + assert_eq!(result_prim.ptype(), PType::F64); + assert_eq!(result_prim.as_slice::().len(), num_rows * dim); + + Ok(()) +} diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs new file mode 100644 index 00000000000..69ae671bc85 --- /dev/null +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! [`ScalarFnVTable`] implementation for [`SorfTransform`]. + +use std::fmt; +use std::fmt::Formatter; +use std::sync::Arc; + +use num_traits::Float; +use num_traits::FromPrimitive; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::expr::Expression; +use vortex_array::extension::EmptyMetadata; +use vortex_array::match_each_float_ptype; +use vortex_array::scalar_fn::Arity; +use vortex_array::scalar_fn::ChildName; +use vortex_array::scalar_fn::ExecutionArgs; +use vortex_array::scalar_fn::ScalarFnId; +use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::validity::Validity; +use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; + +use super::SorfOptions; +use super::SorfTransform; +use super::rotation::SorfMatrix; +use super::validate_sorf_options; +use crate::vector::AnyVector; +use crate::vector::Vector; + +impl ScalarFnVTable for SorfTransform { + type Options = SorfOptions; + + fn id(&self) -> ScalarFnId { + ScalarFnId::new_ref("vortex.tensor.sorf_transform") + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(1) + } + + fn child_name(&self, _options: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("rotated"), + _ => unreachable!("SorfTransform must have exactly one child"), + } + } + + fn fmt_sql( + &self, + _options: &Self::Options, + expr: &Expression, + f: &mut Formatter<'_>, + ) -> fmt::Result { + write!(f, "sorf_transform(")?; + expr.child(0).fmt_sql(f)?; + write!(f, ")") + } + + fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult { + validate_sorf_options(options)?; + + let child_dtype = &arg_dtypes[0]; + let vector_metadata = child_dtype + .as_extension_opt() + .and_then(|ext| ext.metadata_opt::()) + .ok_or_else(|| { + vortex_err!("SorfTransform child must be a Vector extension, got {child_dtype}") + })?; + + let expected_padded = options.dimension.next_power_of_two(); + vortex_ensure_eq!( + vector_metadata.dimensions(), + expected_padded, + "SorfTransform child Vector must have dimension {expected_padded} (next power of two \ + for dimension {})", + options.dimension, + ); + + // For now, the child Vector storage must be f32. TurboQuant stores its centroids as f32, + // and the SORF transform itself operates in f32, so any other input type would require an + // implicit cast that we do not yet support. The output element type is independently + // specified via `options.element_ptype` and is built below. + vortex_ensure_eq!( + vector_metadata.element_ptype(), + PType::F32, + "SorfTransform child Vector storage must be f32 (for now), got {}", + vector_metadata.element_ptype(), + ); + + let output_elem_dtype = DType::Primitive(options.element_ptype, Nullability::NonNullable); + let storage_dtype = DType::FixedSizeList( + Arc::new(output_elem_dtype), + options.dimension, + child_dtype.nullability(), + ); + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage_dtype)?.erased(); + Ok(DType::Extension(ext_dtype)) + } + + fn execute( + &self, + options: &Self::Options, + args: &dyn ExecutionArgs, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + validate_sorf_options(options)?; + let dim = options.dimension as usize; + let num_rows = args.row_count(); + + if num_rows == 0 { + let child_nullability = args.get(0)?.dtype().nullability(); + let validity = Validity::from(child_nullability); + + return match_each_float_ptype!(options.element_ptype, |T| { + let elements = PrimitiveArray::empty::(Nullability::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + options.dimension, + validity, + 0, + )?; + let ext_dtype = + ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + }); + } + + // Execute the child to get the Vector extension wrapping an FSL of f32 coordinates. The + // `return_dtype` check guarantees the child is a `Vector`, so the + // materialized FSL elements are always f32. + let child_ext: ExtensionArray = args.get(0)?.execute(ctx)?; + let child_validity = child_ext.as_ref().validity()?; + let child_fsl: FixedSizeListArray = child_ext.storage_array().clone().execute(ctx)?; + let padded_dim = + usize::try_from(child_fsl.list_size()).vortex_expect("list_size fits usize"); + + let elements_prim: PrimitiveArray = child_fsl.elements().clone().execute(ctx)?; + let f32_elements = elements_prim.into_buffer::(); + + // Reconstruct the orthogonal transform matrix from the seed. + let rotation = SorfMatrix::try_new(options.seed, dim, options.num_rounds as usize)?; + + // Inverse transform each row, truncate to original dimension, cast to target type. + match_each_float_ptype!(options.element_ptype, |T| { + inverse_rotate_typed::( + &f32_elements, + &rotation, + dim, + padded_dim, + num_rows, + child_validity, + ) + }) + } + + fn validity( + &self, + _options: &Self::Options, + expression: &Expression, + ) -> VortexResult> { + Ok(Some(expression.child(0).validity()?)) + } + + fn is_null_sensitive(&self, _options: &Self::Options) -> bool { + false + } + + fn is_fallible(&self, _options: &Self::Options) -> bool { + false + } +} + +/// Convert an f32 value to a float type `T`. +/// +/// `FromPrimitive::from_f32` is infallible for all Vortex float types: f16 saturates via the +/// inherent `f16::from_f32()`, f32 is identity, f64 is lossless widening. +fn float_from_f32(v: f32) -> T { + FromPrimitive::from_f32(v).vortex_expect("f32-to-float conversion is infallible") +} + +/// Apply the inverse SORF transform on f32 data, truncate to the original dimension, cast each +/// element to `T`, and build the output [`Vector`] extension array. +fn inverse_rotate_typed( + f32_elements: &[f32], + rotation: &SorfMatrix, + dim: usize, + padded_dim: usize, + num_rows: usize, + validity: Validity, +) -> VortexResult { + let dim_u32 = u32::try_from(dim).vortex_expect("dimension fits u32"); + let mut output = BufferMut::::with_capacity(num_rows * dim); + let mut unrotated = vec![0.0f32; padded_dim]; + + for row in 0..num_rows { + let row_data = &f32_elements[row * padded_dim..(row + 1) * padded_dim]; + + rotation.inverse_rotate(row_data, &mut unrotated); + + for idx in 0..dim { + // SAFETY: We allocated enough memory above. + unsafe { output.push_unchecked(float_from_f32::(unrotated[idx])) }; + } + } + + let elements = PrimitiveArray::new::(output.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new(elements.into_array(), dim_u32, validity, num_rows)?; + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) +} diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 4d78597c962..325a530ee46 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -13,8 +13,10 @@ use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; +use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; @@ -41,6 +43,40 @@ pub fn validate_tensor_float_input(input_dtype: &DType) -> VortexResult`. +/// +/// Several operations in this crate (SORF transform, TurboQuant quantization) work exclusively +/// in f32. This function handles the cast from any float ptype: +/// +/// - f16: losslessly widened to f32. +/// - f32: zero-copy buffer extraction. +/// - f64: truncated to f32 precision. Values outside f32 range become +/- infinity. This is +/// acceptable because callers of this function operate in f32 and document this constraint. +pub fn cast_to_f32(prim: PrimitiveArray) -> VortexResult> { + match prim.ptype() { + PType::F16 => Ok(prim + .as_slice::() + .iter() + .map(|&v| f32::from(v)) + .collect()), + PType::F32 => Ok(prim.into_buffer()), + PType::F64 => Ok(prim + .as_slice::() + .iter() + .map(|&v| { + #[expect( + clippy::cast_possible_truncation, + reason = "f64 values outside f32 range become infinity, which is acceptable \ + because callers operate in f32 and document this constraint" + )] + let v = v as f32; + v + }) + .collect()), + other => vortex_bail!("expected float elements, got {other:?}"), + } +} + /// The flat primitive elements of a tensor storage array, with typed row access. /// /// This struct hides the stride detail that arises from the [`ConstantArray`] optimization: a diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index b3bcd8334bc..9b7e069d963 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -450,7 +450,6 @@ mod turboquant_benches { use vortex_buffer::BufferMut; use vortex_tensor::encodings::turboquant::TurboQuantConfig; use vortex_tensor::encodings::turboquant::turboquant_encode_unchecked; - use vortex_tensor::scalar_fns::ApproxOptions; use vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm; use vortex_tensor::vector::Vector; @@ -498,7 +497,7 @@ mod turboquant_benches { fn setup_normalized_vector_ext(dim: usize) -> ExtensionArray { let ext = setup_vector_ext(dim); let mut ctx = SESSION.create_execution_ctx(); - let normalized = normalize_as_l2_denorm(&ApproxOptions::Exact, ext.into_array(), &mut ctx) + let normalized = normalize_as_l2_denorm(ext.into_array(), &mut ctx) .unwrap() .child_at(0) .clone(); From 3bdb3764e3559a77849d155de87c0ce7058c3f21 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 10 Apr 2026 15:16:59 -0400 Subject: [PATCH 011/250] add vector similarity benchmark (#7391) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Adds basic benchmarking setup for vector similarity. Right now it is just a bunch of random vectors. Note that the numbers dont really mean anything right now as we have yet to optimize anything (namely I have not yet added the inner product / cosine similarity optimizations pushed through both the SORF transform and the dictionary for constant array). In the future we will add proper benchmarking on real datasets (likely in `vortex-bench`, and also maybe we will integrate https://github.com/zilliztech/vectordbbench ## Testing N/A Signed-off-by: Connor Tsui --- Cargo.lock | 3 + vortex-tensor/Cargo.toml | 8 + vortex-tensor/benches/similarity_search.rs | 108 ++++++++ .../benches/similarity_search_common/mod.rs | 256 ++++++++++++++++++ 4 files changed, 375 insertions(+) create mode 100644 vortex-tensor/benches/similarity_search.rs create mode 100644 vortex-tensor/benches/similarity_search_common/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9fcd945926e..5ce1bc853cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10974,14 +10974,17 @@ dependencies = [ name = "vortex-tensor" version = "0.1.0" dependencies = [ + "codspeed-divan-compat", "half", "itertools 0.14.0", + "mimalloc", "num-traits", "prost 0.14.3", "rand 0.10.0", "rand_distr 0.6.0", "rstest", "vortex-array", + "vortex-btrblocks", "vortex-buffer", "vortex-compressor", "vortex-error", diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index 3dd463bf4df..b14e82287bb 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -31,6 +31,14 @@ num-traits = { workspace = true } prost = { workspace = true } [dev-dependencies] +divan = { workspace = true } +mimalloc = { workspace = true } rand = { workspace = true } rand_distr = { workspace = true } rstest = { workspace = true } +vortex-btrblocks = { path = "../vortex-btrblocks" } + +[[bench]] +name = "similarity_search" +harness = false +test = false diff --git a/vortex-tensor/benches/similarity_search.rs b/vortex-tensor/benches/similarity_search.rs new file mode 100644 index 00000000000..695eae75025 --- /dev/null +++ b/vortex-tensor/benches/similarity_search.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! End-to-end similarity-search execution benchmark. +//! +//! For each of three compression strategies (uncompressed, default BtrBlocks, TurboQuant), this +//! bench: +//! +//! 1. Generates a deterministic random `Vector` batch. +//! 2. Applies the compression strategy *outside* the timed region. +//! 3. Builds the lazy +//! `Binary(Gt, [CosineSimilarity(data, query), threshold])` +//! tree *outside* the timed region. +//! 4. Times *only* `tree.execute::(&mut ctx)`. +//! +//! Run with: `cargo bench -p vortex-tensor --bench similarity_search` + +use divan::Bencher; +use mimalloc::MiMalloc; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_error::VortexExpect; + +#[path = "similarity_search_common/mod.rs"] +mod common; + +use common::Variant; +use common::build_similarity_search_tree; +use common::extract_row_as_query; +use common::generate_random_vectors; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +/// Number of vectors in the benchmark dataset. +const NUM_ROWS: usize = 10_000; + +/// Dimensionality of each vector. Must be `>= vortex_tensor::encodings::turboquant::MIN_DIMENSION` +/// (128) for the TurboQuant variant to work. +const DIM: u32 = 768; + +/// Deterministic PRNG seed for the generated dataset. +const SEED: u64 = 0xC0FFEE; + +/// Cosine similarity threshold for the "greater than" filter. Random f32 vectors from N(0, 1) at +/// this dimension have near-zero pairwise similarity, so picking a row of the dataset as the +/// query guarantees at least that row matches. +const THRESHOLD: f32 = 0.8; + +fn main() { + divan::main(); +} + +/// Runs one end-to-end execution of the similarity-search tree for the given variant. All dataset +/// generation and tree construction happens in the bench setup closure so only the execution of +/// the lazy tree is timed. +fn bench_variant(bencher: Bencher<'_, '_>, variant: Variant) { + bencher + .with_inputs(|| { + let mut ctx = common::SESSION.create_execution_ctx(); + + // Use row 0 of the uncompressed data as the query so we always have at least one + // match. Keeping the query extraction separate from the compressed-data build keeps + // the query identical across all three variants. + let raw = generate_random_vectors(NUM_ROWS, DIM, SEED); + let query = extract_row_as_query(&raw, 0, DIM); + let data = match variant { + Variant::Uncompressed => raw, + Variant::DefaultCompression => { + common::compress_default(raw).vortex_expect("default compression succeeds") + } + Variant::TurboQuant => common::compress_turboquant(raw, &mut ctx) + .vortex_expect("turboquant compression succeeds"), + }; + + // println!( + // "\n\n{}: {}\n\n", + // variant, + // data.display_tree_encodings_only() + // ); + + let tree = build_similarity_search_tree(data, &query, THRESHOLD) + .vortex_expect("tree construction succeeds"); + + (tree, ctx) + }) + .bench_values(|(tree, mut ctx)| { + // Hot path: only the .execute() call is timed. The result is a BoolArray of length + // NUM_ROWS with true at positions where cosine_similarity > THRESHOLD. + tree.execute::(&mut ctx) + .vortex_expect("similarity search tree executes to a BoolArray") + }); +} + +#[divan::bench] +fn execute_uncompressed(bencher: Bencher) { + bench_variant(bencher, Variant::Uncompressed); +} + +#[divan::bench] +fn execute_default_compression(bencher: Bencher) { + bench_variant(bencher, Variant::DefaultCompression); +} + +#[divan::bench] +fn execute_turboquant(bencher: Bencher) { + bench_variant(bencher, Variant::TurboQuant); +} diff --git a/vortex-tensor/benches/similarity_search_common/mod.rs b/vortex-tensor/benches/similarity_search_common/mod.rs new file mode 100644 index 00000000000..c22cb5a9f08 --- /dev/null +++ b/vortex-tensor/benches/similarity_search_common/mod.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Shared helpers for the similarity-search benchmark and example. +//! +//! This module is included from both `vortex-tensor/benches/similarity_search.rs` and +//! `vortex-tensor/examples/similarity_search.rs` via an explicit `#[path = ...]` so both targets +//! use the exact same array-tree builder. +//! +//! The three main entry points are: +//! +//! - [`generate_random_vectors`] to build a deterministic random [`Vector`] extension array. +//! - [`build_variant`] to take a raw vector array and apply the requested compression strategy +//! (uncompressed, default BtrBlocks, or TurboQuant). +//! - [`build_similarity_search_tree`] to wire a cosine-similarity + threshold expression on top of +//! a prepared data array and a single-row query vector. +//! +//! [`Vector`]: vortex_tensor::vector::Vector + +#![allow(dead_code)] + +use std::fmt; +use std::sync::LazyLock; + +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand_distr::Distribution; +use rand_distr::Normal; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Extension; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeListArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::extension::EmptyMetadata; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::session::ArraySession; +use vortex_array::validity::Validity; +use vortex_btrblocks::BtrBlocksCompressor; +use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_tensor::encodings::turboquant::TurboQuantConfig; +use vortex_tensor::encodings::turboquant::turboquant_encode_unchecked; +use vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity; +use vortex_tensor::scalar_fns::l2_denorm::L2Denorm; +use vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use vortex_tensor::vector::Vector; + +/// A shared [`VortexSession`] pre-loaded with the builtin [`ArraySession`] so both bench and +/// example can create execution contexts cheaply. +pub static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + +/// The three compression strategies the benchmark and example exercise. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Variant { + /// Raw `Vector` with no compression applied. + Uncompressed, + /// `BtrBlocksCompressor::default()` walks into the extension array and compresses the + /// underlying FSL storage child with the default scheme set (no TurboQuant). + DefaultCompression, + /// TurboQuant: normalize, quantize to `FSL(Dict)`, wrap in SORF + `L2Denorm`. + TurboQuant, +} + +impl fmt::Display for Variant { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Uncompressed => f.write_str("Uncompressed"), + Self::DefaultCompression => f.write_str("DefaultCompression"), + Self::TurboQuant => f.write_str("TurboQuant"), + } + } +} + +/// Generate `num_rows` random f32 vectors of dimension `dim`, wrapped in a [`Vector`] extension +/// array. The values are drawn from a standard normal distribution seeded by `seed` so results +/// are reproducible across runs. +/// +/// [`Vector`]: vortex_tensor::vector::Vector +pub fn generate_random_vectors(num_rows: usize, dim: u32, seed: u64) -> ArrayRef { + let mut rng = StdRng::seed_from_u64(seed); + // `Normal::new(0, 1)` is infallible for these parameters. `rand_distr::NormalError` does + // not implement `Into`, so we cannot use `vortex_expect` here; fall back to + // `vortex_panic!` on the (impossible) error path instead. + let normal = + Normal::new(0.0f32, 1.0).unwrap_or_else(|_| vortex_panic!("Normal(0, 1) is well-defined")); + + let dim_usize = dim as usize; + let mut buf = BufferMut::::with_capacity(num_rows * dim_usize); + for _ in 0..(num_rows * dim_usize) { + buf.push(normal.sample(&mut rng)); + } + + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = + FixedSizeListArray::try_new(elements.into_array(), dim, Validity::NonNullable, num_rows) + .vortex_expect("FSL with valid shape and matching children length"); + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) + .vortex_expect("Vector extension dtype is valid for an f32 FSL") + .erased(); + ExtensionArray::new(ext_dtype, fsl.into_array()).into_array() +} + +/// Pull the `row`-th vector out of a `Vector` extension array as a plain `Vec`. +/// +/// Used to extract a single query vector from a batch of generated data. The input must already +/// be fully materialized (no lazy scalar-fn wrappers); pass a raw array from +/// [`generate_random_vectors`], not a compressed variant. +pub fn extract_row_as_query(vectors: &ArrayRef, row: usize, dim: u32) -> Vec { + let ext = vectors + .as_opt::() + .vortex_expect("data must be a Vector extension array"); + + let mut ctx = SESSION.create_execution_ctx(); + let fsl: FixedSizeListArray = ext + .storage_array() + .clone() + .execute(&mut ctx) + .vortex_expect("storage array executes to an FSL"); + let elements: PrimitiveArray = fsl + .elements() + .clone() + .execute(&mut ctx) + .vortex_expect("FSL elements execute to a PrimitiveArray"); + + let slice = elements.as_slice::(); + let dim_usize = dim as usize; + let start = row * dim_usize; + slice[start..start + dim_usize].to_vec() +} + +/// Build a `Vector` extension array whose storage is a [`ConstantArray`] broadcasting a +/// single query vector across `num_rows` rows. This is how we hand a single query vector to +/// `CosineSimilarity` on the `rhs` side -- `ScalarFnArray` requires both children to have the +/// same length, so we broadcast the query instead of hand-rolling a 1-row input. +fn build_constant_query_vector(query: &[f32], num_rows: usize) -> VortexResult { + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + + let children: Vec = query + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + + let storage = ConstantArray::new(storage_scalar, num_rows).into_array(); + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, storage).into_array()) +} + +/// Compresses a raw `Vector` array with the default BtrBlocks pipeline. +/// +/// [`BtrBlocksCompressor`] walks into the extension array and recursively compresses the +/// underlying FSL storage child. TurboQuant is *not* exercised by this path -- it is not +/// registered in the default scheme set -- so this measures "generic" lossless compression +/// applied to float vectors. +pub fn compress_default(data: ArrayRef) -> VortexResult { + BtrBlocksCompressor::default().compress(&data) +} + +/// Compresses a raw `Vector` array with the TurboQuant pipeline by hand, producing the +/// same tree shape that +/// [`vortex_tensor::encodings::turboquant::TurboQuantScheme`] would: +/// +/// ```text +/// L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms) +/// ``` +/// +/// Calling the encode helpers directly (instead of going through +/// `BtrBlocksCompressorBuilder::with_turboquant()`) lets this example avoid depending on the +/// `unstable_encodings` feature flag. +/// +/// See `vortex-tensor/src/encodings/turboquant/tests/mod.rs::normalize_and_encode` for the same +/// canonical recipe. +pub fn compress_turboquant(data: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let l2_denorm = normalize_as_l2_denorm(data, ctx)?; + let normalized = l2_denorm.child_at(0).clone(); + let norms = l2_denorm.child_at(1).clone(); + let num_rows = l2_denorm.len(); + + let normalized_ext = normalized + .as_opt::() + .vortex_expect("normalized child should be an Extension array"); + + let config = TurboQuantConfig::default(); + // SAFETY: `normalize_as_l2_denorm` guarantees every row is unit-norm (or zero), which is the + // invariant `turboquant_encode_unchecked` expects. + let tq = unsafe { turboquant_encode_unchecked(normalized_ext, &config, ctx) }?; + + Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) +} + +/// Dispatch helper that builds the data array for the requested [`Variant`], starting from a +/// single random-vector generation. Always returns an `ArrayRef` whose logical dtype is +/// `Vector`. +pub fn build_variant( + variant: Variant, + num_rows: usize, + dim: u32, + seed: u64, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let raw = generate_random_vectors(num_rows, dim, seed); + match variant { + Variant::Uncompressed => Ok(raw), + Variant::DefaultCompression => compress_default(raw), + Variant::TurboQuant => compress_turboquant(raw, ctx), + } +} + +/// Build the lazy similarity-search array tree for a prepared data array and a single query +/// vector. The returned tree is a boolean array of length `data.len()` where position `i` is +/// `true` iff `cosine_similarity(data[i], query) > threshold`. +/// +/// The tree shape is: +/// +/// ```text +/// Binary(Gt, [ +/// CosineSimilarity([data, ConstantArray(query_vec, n)]), +/// ConstantArray(threshold, n), +/// ]) +/// ``` +/// +/// This function does no execution; it is safe to call inside a benchmark setup closure. +pub fn build_similarity_search_tree( + data: ArrayRef, + query: &[f32], + threshold: f32, +) -> VortexResult { + let num_rows = data.len(); + let query_vec = build_constant_query_vector(query, num_rows)?; + + let cosine = CosineSimilarity::try_new_array(data, query_vec, num_rows)?.into_array(); + + let threshold_scalar = Scalar::primitive(threshold, Nullability::NonNullable); + let threshold_array = ConstantArray::new(threshold_scalar, num_rows).into_array(); + + cosine.binary(threshold_array, Operator::Gt) +} From 4aa1c15c10d7b5fa89d2acc2f5642a5b34d8aca4 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Sat, 11 Apr 2026 19:24:57 +0100 Subject: [PATCH 012/250] perf: `Arc::ptr_eq` short-circuit to DType eq comparisons (#7398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FieldName, FieldNames, and StructFields all wrap Arc types but their derived PartialEq always dereferences through the Arc to compare contents. Since DTypes are frequently cloned (every slice, filter, execute copies the DType), the cloned Arcs share pointers - making pointer equality a reliable fast path. DataFusion ClickBench full-suite [apmc](https://github.com/0ax1/apmc) measurement for vortex, averaged over two runs: - Cycles: -5.7% (861B → 812B) — fewer total CPU cycles - IPC: +5.8% (1.88 → 1.99) — more instructions per cycle - MAP_STALL: -5.5% (360B → 340B) — less CPU stalls waiting on memory Signed-off-by: Alexander Droste --- vortex-array/public-api.lock | 12 +++--------- vortex-array/src/dtype/field_names.rs | 18 ++++++++++++++++-- vortex-array/src/dtype/struct_.rs | 9 ++++++++- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index d5ab9cc61f8..317f0869dac 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -9758,7 +9758,7 @@ pub fn vortex_array::dtype::FieldName::cmp(&self, other: &vortex_array::dtype::F impl core::cmp::PartialEq for vortex_array::dtype::FieldName -pub fn vortex_array::dtype::FieldName::eq(&self, other: &vortex_array::dtype::FieldName) -> bool +pub fn vortex_array::dtype::FieldName::eq(&self, other: &Self) -> bool impl core::cmp::PartialEq<&alloc::string::String> for vortex_array::dtype::FieldName @@ -9832,8 +9832,6 @@ impl core::hash::Hash for vortex_array::dtype::FieldName pub fn vortex_array::dtype::FieldName::hash<__H: core::hash::Hasher>(&self, state: &mut __H) -impl core::marker::StructuralPartialEq for vortex_array::dtype::FieldName - pub struct vortex_array::dtype::FieldNames(_) impl vortex_array::dtype::FieldNames @@ -9858,7 +9856,7 @@ impl core::cmp::Eq for vortex_array::dtype::FieldNames impl core::cmp::PartialEq for vortex_array::dtype::FieldNames -pub fn vortex_array::dtype::FieldNames::eq(&self, other: &vortex_array::dtype::FieldNames) -> bool +pub fn vortex_array::dtype::FieldNames::eq(&self, other: &Self) -> bool impl core::cmp::PartialEq<&[&str]> for &vortex_array::dtype::FieldNames @@ -9928,8 +9926,6 @@ pub type vortex_array::dtype::FieldNames::Item = vortex_array::dtype::FieldName pub fn vortex_array::dtype::FieldNames::into_iter(self) -> Self::IntoIter -impl core::marker::StructuralPartialEq for vortex_array::dtype::FieldNames - impl core::ops::index::Index for vortex_array::dtype::FieldNames pub type vortex_array::dtype::FieldNames::Output = vortex_array::dtype::FieldName @@ -10170,7 +10166,7 @@ impl core::cmp::Eq for vortex_array::dtype::StructFields impl core::cmp::PartialEq for vortex_array::dtype::StructFields -pub fn vortex_array::dtype::StructFields::eq(&self, other: &vortex_array::dtype::StructFields) -> bool +pub fn vortex_array::dtype::StructFields::eq(&self, other: &Self) -> bool impl core::default::Default for vortex_array::dtype::StructFields @@ -10188,8 +10184,6 @@ impl core::hash::Hash for vortex_array::dtype::StructFields pub fn vortex_array::dtype::StructFields::hash<__H: core::hash::Hasher>(&self, state: &mut __H) -impl core::marker::StructuralPartialEq for vortex_array::dtype::StructFields - impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::fields::Fields> for vortex_array::dtype::StructFields pub fn vortex_array::dtype::StructFields::from_arrow(value: &arrow_schema::fields::Fields) -> Self diff --git a/vortex-array/src/dtype/field_names.rs b/vortex-array/src/dtype/field_names.rs index 737ad0cfd52..331d70aecd6 100644 --- a/vortex-array/src/dtype/field_names.rs +++ b/vortex-array/src/dtype/field_names.rs @@ -10,9 +10,16 @@ use itertools::Itertools; use vortex_utils::aliases::StringEscape; /// A name for a field in a struct. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Clone, Debug, Eq, PartialOrd, Ord, Hash)] +#[allow(clippy::derived_hash_with_manual_eq)] // manual PartialEq adds Arc::ptr_eq fast path only pub struct FieldName(Arc); +impl PartialEq for FieldName { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) || *self.0 == *other.0 + } +} + impl FieldName { /// Returns a reference to the inner string pub fn inner(&self) -> &Arc { @@ -139,10 +146,17 @@ impl From for Arc { } /// An ordered list of field names in a struct. -#[derive(Clone, PartialEq, Eq, Debug, Default, Hash)] +#[derive(Clone, Eq, Debug, Default, Hash)] +#[allow(clippy::derived_hash_with_manual_eq)] // manual PartialEq adds Arc::ptr_eq fast path only #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct FieldNames(Arc<[FieldName]>); +impl PartialEq for FieldNames { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) || *self.0 == *other.0 + } +} + impl fmt::Display for FieldNames { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( diff --git a/vortex-array/src/dtype/struct_.rs b/vortex-array/src/dtype/struct_.rs index b824caf22dd..637c261b479 100644 --- a/vortex-array/src/dtype/struct_.rs +++ b/vortex-array/src/dtype/struct_.rs @@ -138,9 +138,16 @@ impl FieldDTypeInner { /// // Accessing a field by name will yield the first /// assert_eq!(fields.field("int_col").unwrap(), DType::Primitive(PType::I32, Nullability::Nullable)); /// ``` -#[derive(Clone, PartialEq, Eq, Hash)] +#[derive(Clone, Eq, Hash)] +#[allow(clippy::derived_hash_with_manual_eq)] // manual PartialEq adds Arc::ptr_eq fast path only pub struct StructFields(Arc); +impl PartialEq for StructFields { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) || self.0 == other.0 + } +} + impl std::fmt::Debug for StructFields { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("StructFields") From cc297a6fb8258c945f4608b76f87808650bb0c13 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 01:27:33 +0000 Subject: [PATCH 013/250] Update dependency io.netty:netty-bom to v4.2.12.Final (#7403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [io.netty:netty-bom](https://netty.io/) ([source](https://redirect.github.com/netty/netty)) | `4.2.7.Final` → `4.2.12.Final` | ![age](https://developer.mend.io/api/mc/badges/age/maven/io.netty:netty-bom/4.2.12.Final?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/io.netty:netty-bom/4.2.7.Final/4.2.12.Final?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index c881e141de9..6e93cb5e96c 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -8,7 +8,7 @@ guava = "33.5.0-jre" immutables = "2.12.1" junit-jupiter = "6.0.3" logback = "1.5.32" -netty = "4.2.7.Final" +netty = "4.2.12.Final" nopen = "1.0.1" protobuf = "4.33.5" slf4j = "2.0.17" From 15090e85c1f2a4323550c0ebdb60c7296f054b32 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 02:12:39 +0000 Subject: [PATCH 014/250] Update react monorepo to v19.2.5 (#7404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [react](https://react.dev/) ([source](https://redirect.github.com/facebook/react/tree/HEAD/packages/react)) | [`19.2.4` → `19.2.5`](https://renovatebot.com/diffs/npm/react/19.2.4/19.2.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/react/19.2.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/react/19.2.4/19.2.5?slim=true) | | [react-dom](https://react.dev/) ([source](https://redirect.github.com/facebook/react/tree/HEAD/packages/react-dom)) | [`19.2.4` → `19.2.5`](https://renovatebot.com/diffs/npm/react-dom/19.2.4/19.2.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/react-dom/19.2.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/react-dom/19.2.4/19.2.5?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes

facebook/react (react) ### [`v19.2.5`](https://redirect.github.com/facebook/react/releases/tag/v19.2.5): 19.2.5 (April 8th, 2026) [Compare Source](https://redirect.github.com/facebook/react/compare/v19.2.4...v19.2.5) ##### React Server Components - Add more cycle protections ([#​36236](https://redirect.github.com/facebook/react/pull/36236) by [@​eps1lon](https://redirect.github.com/eps1lon) and [@​unstubbable](https://redirect.github.com/unstubbable))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 1fd57153b0f..013d3ff0bec 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -4671,9 +4671,9 @@ } }, "node_modules/react": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", - "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", + "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4712,15 +4712,15 @@ } }, "node_modules/react-dom": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", - "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.5.tgz", + "integrity": "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag==", "license": "MIT", "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.2.4" + "react": "^19.2.5" } }, "node_modules/react-is": { From f83a367121053d33dd7f7e7c9900e14f860da682 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 02:13:22 +0000 Subject: [PATCH 015/250] Update rust-wasm-bindgen monorepo (#7406) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [js-sys](https://wasm-bindgen.github.io/wasm-bindgen/) ([source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/tree/HEAD/crates/js-sys)) | dependencies | patch | `0.3.91` → `0.3.95` | | [wasm-bindgen](https://wasm-bindgen.github.io/wasm-bindgen) ([source](https://redirect.github.com/wasm-bindgen/wasm-bindgen)) | dependencies | patch | `0.2.114` → `0.2.118` | | [wasm-bindgen-futures](https://wasm-bindgen.github.io/wasm-bindgen/) ([source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/tree/HEAD/crates/futures)) | workspace.dependencies | patch | `0.4.64` → `0.4.68` | | [web-sys](https://wasm-bindgen.github.io/wasm-bindgen/web-sys/index.html) ([source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/tree/HEAD/crates/web-sys)) | dependencies | patch | `0.3.91` → `0.3.95` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
wasm-bindgen/wasm-bindgen (wasm-bindgen) ### [`v0.2.118`](https://redirect.github.com/wasm-bindgen/wasm-bindgen/blob/HEAD/CHANGELOG.md#02118) [Compare Source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/compare/0.2.117...0.2.118) ##### Added - Added `Error::stack_trace_limit()` and `Error::set_stack_trace_limit()` bindings to `js-sys` for the non-standard V8 `Error.stackTraceLimit` property. [#​5082](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5082) - Added support for multiple `#[wasm_bindgen(start)]` functions, which are chained together at initialization, as well as a new `#[wasm_bindgen(start, private)]` to register a start function without exporting it as a public export. [#​5081](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5081) - Reinitialization is no longer automatically applied when using `panic=unwind` and `--experimental-reset-state-function`, instead it is triggered by any use of the `handler::schedule_reinit()` function under `panic=unwind`, which is supported from within the `on_abort` handler for reinit workflows. Renamed `handler::reinit()` to `handler::schedule_reinit()` and removed the `set_on_reinit()` handler. The `__instance_terminated` address is now always a simple boolean (`0` = live, `1` = terminated). [#​5083](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5083) - `handler::schedule_reinit()` now works under `panic=abort` builds. Previously it was a no-op; it now sets the JS-side reinit flag and the next export call transparently creates a fresh `WebAssembly.Instance`. [#​5099](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5099) ##### Changed - MSRV bump from 1.71 to 1.76 for the CLI, and 1.82 to 1.86 for the API [#​5102](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5102) ##### Fixed - ES module `import` statements are now hoisted to the top of generated JS files, placed right after the `@ts-self-types` directive. This ensures valid ES module output since `import` declarations must precede other statements. [#​5103](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5103) - Fixed two CLI issues affecting WASM modules built by rustc 1.94+. First, a panic (`failed to find N in function table`) caused by lld emitting element segment offsets as `global.get $__table_base` or extended const expressions instead of plain `i32.const N` for large function tables; the fix adds a const-expression evaluator in `get_function_table_entry` and guards against integer underflow in multi-segment tables. Second, the descriptor interpreter now routes all global reads/writes through a single `globals` HashMap seeded from the module's own globals, and mirrors the module's actual linear memory rather than a fixed 32KB buffer, so the stack pointer's real value is valid without any override. This fixes panics like `failed to find 32752 in function table` caused by `GOT.func.internal.*` globals being misidentified as the stack pointer. [#​5076](https://redirect.github.com/wasm-bindgen/wasm-bindgen/issues/5076) [#​5080](https://redirect.github.com/wasm-bindgen/wasm-bindgen/issues/5080) [#​5093](https://redirect.github.com/wasm-bindgen/wasm-bindgen/issues/5093) [#​5095](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5095) ### [`v0.2.117`](https://redirect.github.com/wasm-bindgen/wasm-bindgen/blob/HEAD/CHANGELOG.md#02117) [Compare Source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/compare/0.2.116...0.2.117) ##### Fixed - Fixed a regression introduced in [#​5026](https://redirect.github.com/wasm-bindgen/wasm-bindgen/issues/5026) where stable `web-sys` methods that accept a union type containing a `[WbgGeneric]` interface (e.g. `ImageBitmapSource`, which includes `VideoFrame`) incorrectly applied typed generics to all union expansions rather than only those whose argument type is itself `[WbgGeneric]`. In practice this caused `Window::create_image_bitmap_with_*` and the corresponding `WorkerGlobalScope` overloads to return `Promise` instead of `Promise` for the stable (non-`VideoFrame`) call sites, breaking `JsFuture::from(promise).await?`. [#​5064](https://redirect.github.com/wasm-bindgen/wasm-bindgen/issues/5064) [#​5073](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5073) - Fixed handling logic for environment variable `WASM_BINDGEN_TEST_ADDRESS` in the test runner, when running tests in headless mode. [#​5087](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5087) ### [`v0.2.116`](https://redirect.github.com/wasm-bindgen/wasm-bindgen/blob/HEAD/CHANGELOG.md#02116) [Compare Source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/compare/0.2.115...0.2.116) ##### Added - Added `js_sys::Float16Array` bindings, `DataView` float16 accessors using `f32`, and raw `[u16]` helper APIs for interoperability with binary16 representations such as `half::f16`. [#​5033](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5033) ##### Changed - Updated to Walrus 0.26.1 for deterministic type section ordering. [#​5069](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5069) - The `#[wasm_bindgen]` macro now emits `&mut (impl FnMut(...) + MaybeUnwindSafe)` / `&(impl Fn(...) + MaybeUnwindSafe)` for raw `&mut dyn FnMut` / `&dyn Fn` import arguments instead of a hidden generic parameter and where-clause. The generated signature is cleaner and the `MaybeUnwindSafe` bound is visible directly in the argument position. The ABI and wire format are unchanged. When building with `panic=unwind`, closures that capture non-`UnwindSafe` values (e.g. `&mut T`, `Cell`) must wrap them in `AssertUnwindSafe` before capture; on all other targets `MaybeUnwindSafe` is a no-op blanket impl. [#​5056](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5056) ### [`v0.2.115`](https://redirect.github.com/wasm-bindgen/wasm-bindgen/blob/HEAD/CHANGELOG.md#02115) [Compare Source](https://redirect.github.com/wasm-bindgen/wasm-bindgen/compare/0.2.114...0.2.115) ##### Added - `console.debug/log/info/warn/error` output from user-spawned `Worker` and `SharedWorker` instances is now forwarded to the CLI test runner during headless browser tests, just like output from the main thread. Works for blob URL workers, module workers, URL-based workers (importScripts), nested workers, and shared workers (including logs emitted before the first port connection). Non-cloneable arguments are serialized via `String()` rather than crashing the worker. The `--nocapture` flag is respected. [#​5037](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5037) - `js_sys::Promise` now implements `IntoFuture`, enabling direct `.await` on any JS promise without a wrapper type. The `wasm-bindgen-futures` implementation has been moved into `js-sys` behind an optional `futures` feature, which is activated automatically when `wasm-bindgen-futures` is a dependency. All existing `wasm_bindgen_futures::*` import paths continue to work unchanged via re-exports. `js_sys::futures` is also available directly for users who want `promise.await` without depending on `wasm-bindgen-futures`. [#​5049](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5049) - Added `--target emscripten` support, generating a `library_bindgen.js` file for consumption by Emscripten at link time. Includes support for futures, JS closures, and TypeScript output. A new Emscripten-specific test runner is also included, along with CI integration. [#​4443](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4443) - Added `VideoFrame`, `VideoColorSpace`, and related WebCodecs dictionaries/enums to `web-sys`. [#​5008](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5008) - Added `wasm_bindgen::handler` module with `set_on_abort` and `set_on_reinit` hooks for `panic=unwind` builds. `set_on_abort` registers a callback invoked after the instance is terminated (hard abort, OOM, stack overflow). `set_on_reinit` registers a callback invoked after `reinit()` resets the WebAssembly instance via `--experimental-reset-state-function`. Handlers are stored as Wasm indirect-function-table indices so dispatch is safe even when linear memory is corrupt. ##### Changed - Replaced per-closure generic destructors with a single `__wbindgen_destroy_closure` export. [#​5019](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5019) - Refactored the headless browser test runner logging pipeline for dramatically improved performance (>400x faster on Chrome, >10x on Firefox, \~5x on Safari). Switched to incremental DOM scraping with `textContent.slice(offset)`, append-only output semantics, unified log capture across all log levels on failure, and browser-specific invisible-div optimizations (`display:none` for Chrome/Firefox, `visibility:hidden` for Safari). [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - TTY-gated status/clear output in the test runner shell to avoid `\r` control-character artifacts in non-interactive (CI) environments. [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - Added `bench_console_log_10mb` benchmark alongside the existing 1MB benchmark for the headless test runner. The main branch cannot complete this benchmark at any volume. [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - Updated to Walrus 0.26 [#​5057](https://redirect.github.com/wasm-bindgen/walrus/pull/5057) ##### Fixed - Fixed argument order when calling multi-parameter functions in the `wasm-bindgen` interpreter by reversing the args collected from the stack. [#​5047](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5047) - Added support for per-operation `[WbgGeneric]` in WebIDL, restoring typed generic return types (e.g. `Promise`) for `createImageBitmap` on `Window` and `WorkerGlobalScope` that were lost after the `VideoFrame` stabilization. [#​5026](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5026) - Fixed missing `#[cfg(feature = "...")]` gates on deprecated dictionary builder methods and getters for union-typed fields (e.g. `{Open,Save,Directory}FilePickerOptions::start_in()`), and fixed per-setter doc requirements to list each setter's own required features. [#​5039](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5039) - Fixed `JsOption::new()` to use `undefined` instead of `null`, to be compatible with `Option::None` and JS default parameters. [#​5023](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5023) - Fixed unsound `unsafe` transmutes in `JsOption::wrap`, `as_option`, and `into_option` by replacing `transmute_copy` with `unchecked_into()`. Also tightened the `JsGeneric` trait bound and `JsOption` impl block to require `T: JsGeneric` (which implies `JsCast`), preventing use with arbitrary non-JS types. [#​5030](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5030) - Fixed headless test runner emitting `\r` carriage-return sequences in non-TTY environments, which polluted captured logs in CI and complicated output-matching tests. [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - Fixed headless test runner printing incomplete and out-of-order log output on test failures by merging all five log levels into a single unified output div. [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - Fixed large test outputs (10MB+) causing oversized WebDriver responses that were either extremely slow or crashed completely, by switching to incremental streaming output collection. [#​4960](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/4960) - Fixed a duplciate wasm export in node ESM atomics, when compiled in debug mode [#​5028](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5028) - Fixed a type inference regression (`E0283: type annotations needed`) introduced in v0.2.109 where the stable `FromIterator` and `Extend` impls on `js_sys::Array` were changed from `A: AsRef` to `A: AsRef`. Because `#[wasm_bindgen]` generates multiple `AsRef` impls per type, the compiler could not uniquely resolve `T`, breaking code like `Array::from_iter([my_wasm_value])` without explicit annotations. The stable impls are restored to `A: AsRef` (returning `Array`); the generic `A: AsRef` forms remain available under `js_sys_unstable_apis`. [#​5052](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5052) - Fixed `skip_typescript` not being respected when using `reexport`, causing TypeScript definitions to be incorrectly emitted for re-exported items marked with `#[wasm_bindgen(skip_typescript)]`. [#​5051](https://redirect.github.com/wasm-bindgen/wasm-bindgen/pull/5051) ##### Removed
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ce1bc853cc..754ab735d3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5090,10 +5090,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -11139,9 +11141,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -11152,23 +11154,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11176,9 +11174,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -11189,9 +11187,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -11245,9 +11243,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", From 5060020c0549cc3bb153b381d56b76a086b0030a Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 13 Apr 2026 03:05:06 -0400 Subject: [PATCH 016/250] bump rand to 0.10.1 (#7405) ## Summary Bumps `rand` to `0.10.1`. - https://rustsec.org/advisories/RUSTSEC-2026-0097 - https://github.com/rust-random/rand/pull/1763 ## Testing N/A Signed-off-by: Connor Tsui --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 754ab735d3c..9576e32adb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7520,9 +7520,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", @@ -7600,7 +7600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" dependencies = [ "num-traits", - "rand 0.10.0", + "rand 0.10.1", ] [[package]] @@ -7629,7 +7629,7 @@ dependencies = [ "clap", "indicatif", "lance-bench", - "rand 0.10.0", + "rand 0.10.1", "rand_distr 0.6.0", "tokio", "vortex-bench", @@ -10071,7 +10071,7 @@ dependencies = [ "mimalloc", "parquet 58.0.0", "paste", - "rand 0.10.0", + "rand 0.10.1", "rand_distr 0.6.0", "serde_json", "tokio", @@ -10116,7 +10116,7 @@ dependencies = [ "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rstest", "rustc-hash", "vortex-array", @@ -10167,7 +10167,7 @@ dependencies = [ "pin-project-lite", "primitive-types", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rand_distr 0.6.0", "rstest", "rstest_reuse", @@ -10224,7 +10224,7 @@ dependencies = [ "noodles-vcf", "parking_lot", "parquet 58.0.0", - "rand 0.10.0", + "rand 0.10.1", "regex", "reqwest 0.12.28", "serde", @@ -10254,7 +10254,7 @@ dependencies = [ "itertools 0.14.0", "num-traits", "pco", - "rand 0.10.0", + "rand 0.10.1", "rstest", "rustc-hash", "test-with", @@ -10343,7 +10343,7 @@ dependencies = [ "itertools 0.14.0", "num-traits", "parking_lot", - "rand 0.10.0", + "rand 0.10.1", "rstest", "rustc-hash", "tracing", @@ -10536,7 +10536,7 @@ dependencies = [ "lending-iterator", "num-traits", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rstest", "vortex-alp", "vortex-array", @@ -10629,7 +10629,7 @@ dependencies = [ "codspeed-divan-compat", "fsst-rs", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rstest", "vortex-array", "vortex-buffer", @@ -10885,7 +10885,7 @@ dependencies = [ "itertools 0.14.0", "num-traits", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rstest", "vortex-array", "vortex-buffer", @@ -10982,7 +10982,7 @@ dependencies = [ "mimalloc", "num-traits", "prost 0.14.3", - "rand 0.10.0", + "rand 0.10.1", "rand_distr 0.6.0", "rstest", "vortex-array", diff --git a/Cargo.toml b/Cargo.toml index 9853cf94ed9..f486a90b3c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,7 @@ pyo3-bytes = "0.6" pyo3-log = "0.13.0" pyo3-object_store = "0.9.0" quote = "1.0.44" -rand = "0.10.0" +rand = "0.10.1" rand_distr = "0.6" ratatui = { version = "0.30", default-features = false } regex = "1.11.0" From 93746cf4b050391c764b5d11db655a4a78dc7ad0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:55:51 +0000 Subject: [PATCH 017/250] Update all patch updates (#7402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---|---|---| | [arc-swap](https://redirect.github.com/vorner/arc-swap) | workspace.dependencies | patch | `1.9.0` → `1.9.1` | ![age](https://developer.mend.io/api/mc/badges/age/crate/arc-swap/1.9.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/arc-swap/1.9.0/1.9.1?slim=true) | | [cc](https://redirect.github.com/rust-lang/cc-rs) | workspace.dependencies | patch | `1.2.57` → `1.2.60` | ![age](https://developer.mend.io/api/mc/badges/age/crate/cc/1.2.60?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/cc/1.2.57/1.2.60?slim=true) | | [custom-labels](https://polarsignals.com) ([source](https://redirect.github.com/polarsignals/custom-labels)) | workspace.dependencies | patch | `0.4.5` → `0.4.6` | ![age](https://developer.mend.io/api/mc/badges/age/crate/custom-labels/0.4.6?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/custom-labels/0.4.5/0.4.6?slim=true) | | [env_logger](https://redirect.github.com/rust-cli/env_logger) | workspace.dependencies | patch | `0.11.9` → `0.11.10` | ![age](https://developer.mend.io/api/mc/badges/age/crate/env_logger/0.11.10?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/env_logger/0.11.9/0.11.10?slim=true) | | [fsst-rs](https://redirect.github.com/spiraldb/fsst) | workspace.dependencies | patch | `0.5.9` → `0.5.10` | ![age](https://developer.mend.io/api/mc/badges/age/crate/fsst-rs/0.5.10?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/fsst-rs/0.5.9/0.5.10?slim=true) | | [inventory](https://redirect.github.com/dtolnay/inventory) | workspace.dependencies | patch | `0.3.22` → `0.3.24` | ![age](https://developer.mend.io/api/mc/badges/age/crate/inventory/0.3.24?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/inventory/0.3.22/0.3.24?slim=true) | | [object_store](https://redirect.github.com/apache/arrow-rs-object-store) | workspace.dependencies | patch | `0.13.1` → `0.13.2` | ![age](https://developer.mend.io/api/mc/badges/age/crate/object_store/0.13.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/object_store/0.13.1/0.13.2?slim=true) | | [prettier](https://prettier.io) ([source](https://redirect.github.com/prettier/prettier)) | devDependencies | patch | [`3.8.1` → `3.8.2`](https://renovatebot.com/diffs/npm/prettier/3.8.1/3.8.2) | ![age](https://developer.mend.io/api/mc/badges/age/npm/prettier/3.8.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/prettier/3.8.1/3.8.2?slim=true) | | [pyo3](https://redirect.github.com/pyo3/pyo3) | workspace.dependencies | patch | `0.28.2` → `0.28.3` | ![age](https://developer.mend.io/api/mc/badges/age/crate/pyo3/0.28.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/pyo3/0.28.2/0.28.3?slim=true) | | [rustc-hash](https://redirect.github.com/rust-lang/rustc-hash) | workspace.dependencies | patch | `2.1.1` → `2.1.2` | ![age](https://developer.mend.io/api/mc/badges/age/crate/rustc-hash/2.1.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/rustc-hash/2.1.1/2.1.2?slim=true) | | com.gradleup.shadow | plugin | patch | `9.4.0` → `9.4.1` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.gradleup.shadow:com.gradleup.shadow.gradle.plugin/9.4.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.gradleup.shadow:com.gradleup.shadow.gradle.plugin/9.4.0/9.4.1?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
vorner/arc-swap (arc-swap) ### [`v1.9.1`](https://redirect.github.com/vorner/arc-swap/blob/HEAD/CHANGELOG.md#191) [Compare Source](https://redirect.github.com/vorner/arc-swap/compare/v1.9.0...v1.9.1) - One more SeqCst :-| ([#​204](https://redirect.github.com/vorner/arc-swap/issues/204)).
rust-lang/cc-rs (cc) ### [`v1.2.60`](https://redirect.github.com/rust-lang/cc-rs/blob/HEAD/CHANGELOG.md#1260---2026-04-10) [Compare Source](https://redirect.github.com/rust-lang/cc-rs/compare/cc-v1.2.59...cc-v1.2.60) ##### Fixed - *(ar)* suppress warnings from `D` modifier probe ([#​1700](https://redirect.github.com/rust-lang/cc-rs/pull/1700)) ### [`v1.2.59`](https://redirect.github.com/rust-lang/cc-rs/blob/HEAD/CHANGELOG.md#1259---2026-04-03) [Compare Source](https://redirect.github.com/rust-lang/cc-rs/compare/cc-v1.2.58...cc-v1.2.59) ##### Fixed - *(ar)* deterministic archives with `D` modifier ([#​1697](https://redirect.github.com/rust-lang/cc-rs/pull/1697)) ##### Other - Regenerate target info ([#​1698](https://redirect.github.com/rust-lang/cc-rs/pull/1698)) - Fix target abi parsing for sanitiser targets ([#​1695](https://redirect.github.com/rust-lang/cc-rs/pull/1695)) ### [`v1.2.58`](https://redirect.github.com/rust-lang/cc-rs/blob/HEAD/CHANGELOG.md#1258---2026-03-27) [Compare Source](https://redirect.github.com/rust-lang/cc-rs/compare/cc-v1.2.57...cc-v1.2.58) ##### Other - Update Compile-time Requirements to add info about clang-cl.exe ([#​1693](https://redirect.github.com/rust-lang/cc-rs/pull/1693))
rust-cli/env_logger (env_logger) ### [`v0.11.10`](https://redirect.github.com/rust-cli/env_logger/blob/HEAD/CHANGELOG.md#01110---2026-03-23) [Compare Source](https://redirect.github.com/rust-cli/env_logger/compare/v0.11.9...v0.11.10) ##### Internal - Update dependencies
spiraldb/fsst (fsst-rs) ### [`v0.5.10`](https://redirect.github.com/spiraldb/fsst/releases/tag/0.5.10) [Compare Source](https://redirect.github.com/spiraldb/fsst/compare/0.5.9...0.5.10) #### Changes - feat: prune low-value symbols from table on small inputs ([#​203](https://redirect.github.com/spiraldb/fsst/issues/203)) [@​CommanderStorm](https://redirect.github.com/CommanderStorm) - chore(deps): update codspeedhq/action digest to [`db35df7`](https://redirect.github.com/spiraldb/fsst/commit/db35df7) ([#​201](https://redirect.github.com/spiraldb/fsst/issues/201)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update taiki-e/cache-cargo-install-action digest to [`511847d`](https://redirect.github.com/spiraldb/fsst/commit/511847d) ([#​198](https://redirect.github.com/spiraldb/fsst/issues/198)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): pin dtolnay/rust-toolchain action to [`3c5f7ea`](https://redirect.github.com/spiraldb/fsst/commit/3c5f7ea) ([#​197](https://redirect.github.com/spiraldb/fsst/issues/197)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): lock file maintenance ([#​196](https://redirect.github.com/spiraldb/fsst/issues/196)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): pin release-drafter/release-drafter action to [`139054a`](https://redirect.github.com/spiraldb/fsst/commit/139054a) ([#​194](https://redirect.github.com/spiraldb/fsst/issues/194)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update taiki-e/cache-cargo-install-action digest to [`7824a3d`](https://redirect.github.com/spiraldb/fsst/commit/7824a3d) ([#​195](https://redirect.github.com/spiraldb/fsst/issues/195)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - Update commments in the decompress method ([#​193](https://redirect.github.com/spiraldb/fsst/issues/193)) [@​robert3005](https://redirect.github.com/robert3005)
dtolnay/inventory (inventory) ### [`v0.3.24`](https://redirect.github.com/dtolnay/inventory/releases/tag/0.3.24) [Compare Source](https://redirect.github.com/dtolnay/inventory/compare/0.3.23...0.3.24) - Add support for VxWorks targets ([#​89](https://redirect.github.com/dtolnay/inventory/issues/89), thanks [@​elBoberido](https://redirect.github.com/elBoberido)) ### [`v0.3.23`](https://redirect.github.com/dtolnay/inventory/releases/tag/0.3.23) [Compare Source](https://redirect.github.com/dtolnay/inventory/compare/0.3.22...0.3.23) - Avoid triggering clippy::disallowed\_types in downstream projects that use Loom ([#​88](https://redirect.github.com/dtolnay/inventory/issues/88), thanks [@​elBoberido](https://redirect.github.com/elBoberido))
apache/arrow-rs-object-store (object_store) ### [`v0.13.2`](https://redirect.github.com/apache/arrow-rs-object-store/blob/HEAD/CHANGELOG.md#v0132-2026-03-19) [Compare Source](https://redirect.github.com/apache/arrow-rs-object-store/compare/v0.13.1...v0.13.2) [Full Changelog](https://redirect.github.com/apache/arrow-rs-object-store/compare/v0.12.5...v0.13.2) **Implemented enhancements:** - `Path::join(Self, &PathPart) -> Self` [#​665](https://redirect.github.com/apache/arrow-rs-object-store/issues/665) - Support for AWS Encryption Client encryption [#​647](https://redirect.github.com/apache/arrow-rs-object-store/issues/647) - `LocalFileSystem`: use `read_at` instead of seek + read [#​622](https://redirect.github.com/apache/arrow-rs-object-store/issues/622) - Avoid reading metadata for `LocalFileSystem::read_ranges` (and other methods) [#​614](https://redirect.github.com/apache/arrow-rs-object-store/issues/614) - expose `Inner` from `HttpRequestBody` [#​606](https://redirect.github.com/apache/arrow-rs-object-store/issues/606) - Release object store `0.13.1` (maintenance) - Target Jan 2026 [#​598](https://redirect.github.com/apache/arrow-rs-object-store/issues/598) - Support AWS\_ENDPOINT\_URL\_S3 in aws backend [#​589](https://redirect.github.com/apache/arrow-rs-object-store/issues/589) - Release object store `0.12.5` (maintenance) - Target Dec 2025 [#​582](https://redirect.github.com/apache/arrow-rs-object-store/issues/582) - Support upper-case configuration options in parse\_url\_opts [#​529](https://redirect.github.com/apache/arrow-rs-object-store/issues/529) - object\_store: Support `{az,abfs,abfss}://container@account.blob.{core.windows.net,fabric.microsoft.com}` URLs [#​430](https://redirect.github.com/apache/arrow-rs-object-store/issues/430) - Support `Transfer-Encoding: chunked` responses in HttpStore [#​340](https://redirect.github.com/apache/arrow-rs-object-store/issues/340) - Use reconstructed ListBlobs marker to provide list offset support in `MicrosoftAzure` store [#​461](https://redirect.github.com/apache/arrow-rs-object-store/issues/461) **Fixed bugs:** - Azure Fabric: Unsigned integer underflow when fetching token causes integer overflow panic [#​640](https://redirect.github.com/apache/arrow-rs-object-store/issues/640) - Error body missing for 5xx errors after retry exhausted [#​617](https://redirect.github.com/apache/arrow-rs-object-store/issues/617) - Heavy contention on credentials cache [#​541](https://redirect.github.com/apache/arrow-rs-object-store/issues/541) - AWS/S3 Default Headers are not considered for signature calculation [#​484](https://redirect.github.com/apache/arrow-rs-object-store/issues/484) - az:// \ not work as expected [#​443](https://redirect.github.com/apache/arrow-rs-object-store/issues/443) **Performance improvements:** - Preallocate single `Vec` in `get_ranges` for LocalFilesystem [#​634](https://redirect.github.com/apache/arrow-rs-object-store/issues/634) - Use platform specific `read_at` when available [#​628](https://redirect.github.com/apache/arrow-rs-object-store/pull/628) ([AdamGS](https://redirect.github.com/AdamGS)) - Avoid metadata lookup for `LocalFileSystem::read_ranges` and `chunked_stream` [#​621](https://redirect.github.com/apache/arrow-rs-object-store/pull/621) ([Dandandan](https://redirect.github.com/Dandandan)) **Closed issues:** - \[Security Alert] Exposed API key(s) detected: AWS Access Key [#​659](https://redirect.github.com/apache/arrow-rs-object-store/issues/659) - AWS S3 token expired on multi-threaded app with Arc usage [#​655](https://redirect.github.com/apache/arrow-rs-object-store/issues/655) - Emulator tests fail in CI due to an unsupported service version header in Azurite [#​626](https://redirect.github.com/apache/arrow-rs-object-store/issues/626) **Merged pull requests:** - Replace `Path::child` with `Path::join` [#​666](https://redirect.github.com/apache/arrow-rs-object-store/pull/666) ([Kinrany](https://redirect.github.com/Kinrany)) - Support --xa-s3 suffix for S3 Express One Zone bucket access points [#​663](https://redirect.github.com/apache/arrow-rs-object-store/pull/663) ([pdeva](https://redirect.github.com/pdeva)) - docs: clarify `Clone` behavior [#​656](https://redirect.github.com/apache/arrow-rs-object-store/pull/656) ([crepererum](https://redirect.github.com/crepererum)) - Implement Clone for local and memory stores [#​653](https://redirect.github.com/apache/arrow-rs-object-store/pull/653) ([DoumanAsh](https://redirect.github.com/DoumanAsh)) - Unify `from_env` behaviours [#​652](https://redirect.github.com/apache/arrow-rs-object-store/pull/652) ([miraclx](https://redirect.github.com/miraclx)) - docs: add examples to the aws docs where appropriate [#​651](https://redirect.github.com/apache/arrow-rs-object-store/pull/651) ([CommanderStorm](https://redirect.github.com/CommanderStorm)) - Switch TokenCache to RWLock [#​648](https://redirect.github.com/apache/arrow-rs-object-store/pull/648) ([tustvold](https://redirect.github.com/tustvold)) - Minimize futures dependency into relevant sub-crates [#​646](https://redirect.github.com/apache/arrow-rs-object-store/pull/646) ([AdamGS](https://redirect.github.com/AdamGS)) - Clarify ShuffleResolver doc-comments [#​645](https://redirect.github.com/apache/arrow-rs-object-store/pull/645) ([jkosh44](https://redirect.github.com/jkosh44)) - Introduce a "tokio" to allow pulling a trait-only build [#​644](https://redirect.github.com/apache/arrow-rs-object-store/pull/644) ([AdamGS](https://redirect.github.com/AdamGS)) - fix(azure): fix integer overflow in Fabric token expiry check [#​641](https://redirect.github.com/apache/arrow-rs-object-store/pull/641) ([desmondcheongzx](https://redirect.github.com/desmondcheongzx)) - chore: upgrade to `rand` 0.10 [#​637](https://redirect.github.com/apache/arrow-rs-object-store/pull/637) ([crepererum](https://redirect.github.com/crepererum)) - fix(aws): Include default headers in signature calculation ([#​484](https://redirect.github.com/apache/arrow-rs-object-store/issues/484)) [#​636](https://redirect.github.com/apache/arrow-rs-object-store/pull/636) ([singhsaabir](https://redirect.github.com/singhsaabir)) - fix(azure): correct Microsoft Fabric blob endpoint domain [#​631](https://redirect.github.com/apache/arrow-rs-object-store/pull/631) ([kevinjqliu](https://redirect.github.com/kevinjqliu)) - Unblock emulator based tests [#​627](https://redirect.github.com/apache/arrow-rs-object-store/pull/627) ([AdamGS](https://redirect.github.com/AdamGS)) - Azure ADLS list\_with\_offset support [#​623](https://redirect.github.com/apache/arrow-rs-object-store/pull/623) ([omar](https://redirect.github.com/omar)) - Implement tests for range and partial content responses [#​619](https://redirect.github.com/apache/arrow-rs-object-store/pull/619) ([vitoordaz](https://redirect.github.com/vitoordaz)) - fix: missing 5xx error body when retry exhausted [#​618](https://redirect.github.com/apache/arrow-rs-object-store/pull/618) ([jackye1995](https://redirect.github.com/jackye1995)) - build(deps): update nix requirement from 0.30.0 to 0.31.1 [#​616](https://redirect.github.com/apache/arrow-rs-object-store/pull/616) ([dependabot\[bot\]](https://redirect.github.com/apps/dependabot)) - Clarify behavior of `parse_url_opts` with regards to case sensitivity [#​613](https://redirect.github.com/apache/arrow-rs-object-store/pull/613) ([AdamGS](https://redirect.github.com/AdamGS)) - Fix logical format conflict [#​605](https://redirect.github.com/apache/arrow-rs-object-store/pull/605) ([tustvold](https://redirect.github.com/tustvold)) - Fix Azure URL parsing [#​604](https://redirect.github.com/apache/arrow-rs-object-store/pull/604) ([tustvold](https://redirect.github.com/tustvold)) - build(deps): update quick-xml requirement from 0.38.0 to 0.39.0 [#​602](https://redirect.github.com/apache/arrow-rs-object-store/pull/602) ([dependabot\[bot\]](https://redirect.github.com/apps/dependabot)) - Only read file metadata once in `LocalFileSystem::read_ranges` [#​595](https://redirect.github.com/apache/arrow-rs-object-store/pull/595) ([AdamGS](https://redirect.github.com/AdamGS)) - feat: Add support for AWS\_ENDPOINT\_URL\_S3 environment variable [#​590](https://redirect.github.com/apache/arrow-rs-object-store/pull/590) ([rajatgoel](https://redirect.github.com/rajatgoel)) - feat: impl MultipartStore for PrefixStore [#​587](https://redirect.github.com/apache/arrow-rs-object-store/pull/587) ([ddupg](https://redirect.github.com/ddupg)) - Implement typos-cli [#​570](https://redirect.github.com/apache/arrow-rs-object-store/pull/570) ([jayvdb](https://redirect.github.com/jayvdb)) - feat (azure): support for '.blob.core.windows.net' in "az://" scheme [#​431](https://redirect.github.com/apache/arrow-rs-object-store/pull/431) ([vladidobro](https://redirect.github.com/vladidobro)) \* *This Changelog was automatically generated by [github\_changelog\_generator](https://redirect.github.com/github-changelog-generator/github-changelog-generator)*
prettier/prettier (prettier) ### [`v3.8.2`](https://redirect.github.com/prettier/prettier/compare/3.8.1...fbf300f9d89820364ddc9b2efa05b92b8c01b692) [Compare Source](https://redirect.github.com/prettier/prettier/compare/3.8.1...3.8.2)
pyo3/pyo3 (pyo3) ### [`v0.28.3`](https://redirect.github.com/pyo3/pyo3/blob/HEAD/CHANGELOG.md#0283---2026-04-02) [Compare Source](https://redirect.github.com/pyo3/pyo3/compare/v0.28.2...v0.28.3) ##### Fixed - Fix compile error with `#[pyclass(get_all)]` on a type named `Probe`. [#​5837](https://redirect.github.com/PyO3/pyo3/pull/5837) - Fix compile error in debug builds related to `_Py_NegativeRefcount` with Python < 3.12. [#​5847](https://redirect.github.com/PyO3/pyo3/pull/5847) - Fix a race condition where `Python::attach` or `try_attach` could return before `site.py` had finished running. [#​5903](https://redirect.github.com/PyO3/pyo3/pull/5903) - Fix unsoundness in `PyBytesWriter::write_vectored` with Python 3.15 prerelease versions. [#​5907](https://redirect.github.com/PyO3/pyo3/pull/5907) - Fix deadlock in `.into_pyobject()` implementation for C-like `#[pyclass]` enums. [#​5928](https://redirect.github.com/PyO3/pyo3/pull/5928)
rust-lang/rustc-hash (rustc-hash) ### [`v2.1.2`](https://redirect.github.com/rust-lang/rustc-hash/blob/HEAD/CHANGELOG.md#212) [Compare Source](https://redirect.github.com/rust-lang/rustc-hash/compare/v2.1.1...v2.1.2) - [Refactor byte hashing to remove unreachable panic](https://redirect.github.com/rust-lang/rustc-hash/pull/65)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 252 +++++++++-------------------- java/vortex-jni/build.gradle.kts | 2 +- java/vortex-spark/build.gradle.kts | 2 +- vortex-web/package-lock.json | 6 +- 4 files changed, 83 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9576e32adb6..0b84a2f407a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,21 +89,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" -[[package]] -name = "anstream" -version = "0.6.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" -dependencies = [ - "anstyle", - "anstyle-parse 0.2.7", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstream" version = "1.0.0" @@ -111,7 +96,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", - "anstyle-parse 1.0.0", + "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", @@ -125,15 +110,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - [[package]] name = "anstyle-parse" version = "1.0.0" @@ -224,9 +200,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "arc-swap" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] @@ -985,7 +961,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -1306,9 +1282,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -1448,7 +1424,7 @@ version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ - "anstream 1.0.0", + "anstream", "anstyle", "clap_lex", "strsim", @@ -1966,9 +1942,9 @@ dependencies = [ [[package]] name = "custom-labels" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a750ea4bdb7dbf9584b5d5c668bfa3835f88275781a947b5ea0212945bbdd41f" +checksum = "4121f4f539659ad975da5bc6702b7d7d6de59ff1bbaec328dfe7dd5058052ef2" dependencies = [ "bindgen", "cc", @@ -2179,7 +2155,7 @@ dependencies = [ "itertools 0.14.0", "liblzma", "log", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "parquet 58.0.0", "rand 0.9.2", @@ -2204,7 +2180,7 @@ dependencies = [ "datafusion-physical-plan 53.0.0", "futures", "itertools 0.14.0", - "object_store 0.13.1", + "object_store 0.13.2", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -2263,7 +2239,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "tokio", ] @@ -2312,7 +2288,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.1", + "object_store 0.13.2", ] [[package]] @@ -2354,7 +2330,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store 0.13.1", + "object_store 0.13.2", "parquet 58.0.0", "paste", "recursive", @@ -2441,7 +2417,7 @@ dependencies = [ "itertools 0.14.0", "liblzma", "log", - "object_store 0.13.1", + "object_store 0.13.2", "rand 0.9.2", "tokio", "tokio-util", @@ -2493,7 +2469,7 @@ dependencies = [ "datafusion-session 53.0.0", "futures", "itertools 0.14.0", - "object_store 0.13.1", + "object_store 0.13.2", "tokio", ] @@ -2514,7 +2490,7 @@ dependencies = [ "datafusion-session 53.0.0", "futures", "num-traits", - "object_store 0.13.1", + "object_store 0.13.2", ] [[package]] @@ -2558,7 +2534,7 @@ dependencies = [ "datafusion-physical-plan 53.0.0", "datafusion-session 53.0.0", "futures", - "object_store 0.13.1", + "object_store 0.13.2", "regex", "tokio", ] @@ -2603,7 +2579,7 @@ dependencies = [ "datafusion-physical-plan 53.0.0", "datafusion-session 53.0.0", "futures", - "object_store 0.13.1", + "object_store 0.13.2", "serde_json", "tokio", "tokio-stream", @@ -2633,7 +2609,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "parquet 58.0.0", "tokio", @@ -2687,7 +2663,7 @@ dependencies = [ "datafusion-physical-expr-common 53.0.0", "futures", "log", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "rand 0.9.2", "tempfile", @@ -3443,7 +3419,7 @@ dependencies = [ "indicatif", "itertools 0.14.0", "log", - "object_store 0.13.1", + "object_store 0.13.2", "sqllogictest", "sqlparser 0.61.0", "tempfile", @@ -3463,7 +3439,7 @@ dependencies = [ "datafusion 53.0.0", "half", "itertools 0.14.0", - "object_store 0.13.1", + "object_store 0.13.2", "pbjson-types", "prost 0.14.3", "substrait", @@ -3558,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3726,11 +3702,11 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ - "anstream 0.6.21", + "anstream", "anstyle", "env_filter", "jiff", @@ -3750,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3973,9 +3949,9 @@ dependencies = [ [[package]] name = "fsst-rs" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf65e16e100438be0030d113042e07a62bed67203998640ca6fae0404eed71e" +checksum = "3bf53d7c403a2b76873d4d66ba7d79c54bde2784cdaba6083f223d6e33270708" dependencies = [ "rustc-hash", ] @@ -4891,9 +4867,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -4922,7 +4898,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5003,7 +4979,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5823,7 +5799,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" dependencies = [ - "anstream 1.0.0", + "anstream", "anstyle", "clap", "escape8259", @@ -6311,7 +6287,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6454,16 +6430,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2858065e55c148d294a9f3aae3b0fa9458edadb41a108397094566f4e3c0dfb" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http", "http-body-util", "httparse", @@ -6474,7 +6452,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.2", + "rand 0.10.1", "reqwest 0.12.28", "ring", "rustls-pki-types", @@ -6767,7 +6745,7 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "object_store 0.13.1", + "object_store 0.13.2", "paste", "seq-macro", "simdutf8", @@ -6980,9 +6958,9 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.9.2", + "rand 0.8.5", "socket2", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -7189,7 +7167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph", @@ -7221,7 +7199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -7282,9 +7260,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "chrono", "indexmap", @@ -7312,9 +7290,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon", ] @@ -7331,9 +7309,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", @@ -7352,9 +7330,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -7364,9 +7342,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", @@ -7388,7 +7366,7 @@ dependencies = [ "http", "humantime", "itertools 0.14.0", - "object_store 0.13.1", + "object_store 0.13.2", "percent-encoding", "pyo3", "pyo3-async-runtimes", @@ -7406,9 +7384,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" dependencies = [ "memchr", "serde", @@ -7467,7 +7445,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8124,9 +8102,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc-hex" @@ -8176,7 +8154,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8234,7 +8212,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9313,7 +9291,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9332,7 +9310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -10379,7 +10357,7 @@ dependencies = [ "futures", "itertools 0.14.0", "kanal", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "prost 0.14.3", "rstest", @@ -10440,7 +10418,7 @@ dependencies = [ "futures", "insta", "itertools 0.14.0", - "object_store 0.13.1", + "object_store 0.13.2", "rstest", "tempfile", "tokio", @@ -10496,7 +10474,7 @@ dependencies = [ "jiff", "kanal", "num-traits", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "paste", "reqwest 0.12.28", @@ -10519,7 +10497,7 @@ dependencies = [ "arrow-schema 58.0.0", "flatbuffers", "jiff", - "object_store 0.13.1", + "object_store 0.13.2", "prost 0.14.3", "serial_test", "temp-env", @@ -10556,7 +10534,7 @@ dependencies = [ "futures", "itertools 0.14.0", "mimalloc", - "object_store 0.13.1", + "object_store 0.13.2", "paste", "prost 0.14.3", "tempfile", @@ -10579,7 +10557,7 @@ dependencies = [ "itertools 0.14.0", "kanal", "moka", - "object_store 0.13.1", + "object_store 0.13.2", "oneshot 0.2.1", "parking_lot", "pin-project-lite", @@ -10680,7 +10658,7 @@ dependencies = [ "handle", "itertools 0.14.0", "kanal", - "object_store 0.13.1", + "object_store 0.13.2", "oneshot 0.2.1", "parking_lot", "pin-project-lite", @@ -10723,7 +10701,7 @@ dependencies = [ "arrow-schema 58.0.0", "futures", "jni", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "prost 0.14.3", "thiserror 2.0.18", @@ -10862,7 +10840,7 @@ dependencies = [ "itertools 0.14.0", "log", "mimalloc", - "object_store 0.13.1", + "object_store 0.13.2", "parking_lot", "pyo3", "pyo3-bytes", @@ -11310,7 +11288,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11458,15 +11436,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -11500,30 +11469,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows-threading" version = "0.2.1" @@ -11545,12 +11497,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -11563,12 +11509,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -11581,24 +11521,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -11611,12 +11539,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -11629,12 +11551,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -11647,12 +11563,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -11665,12 +11575,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.7.15" diff --git a/java/vortex-jni/build.gradle.kts b/java/vortex-jni/build.gradle.kts index 7bf389683c9..148d0cf18af 100644 --- a/java/vortex-jni/build.gradle.kts +++ b/java/vortex-jni/build.gradle.kts @@ -8,7 +8,7 @@ plugins { `java-library` `jvm-test-suite` id("com.google.protobuf") - id("com.gradleup.shadow") version "9.4.0" + id("com.gradleup.shadow") version "9.4.1" } dependencies { diff --git a/java/vortex-spark/build.gradle.kts b/java/vortex-spark/build.gradle.kts index f5fb83fe7ba..12cd759ec5e 100644 --- a/java/vortex-spark/build.gradle.kts +++ b/java/vortex-spark/build.gradle.kts @@ -6,7 +6,7 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar plugins { `java-library` `jvm-test-suite` - id("com.gradleup.shadow") version "9.4.0" + id("com.gradleup.shadow") version "9.4.1" } // Derive Scala and Spark versions from the Gradle project name (vortex-spark_2.12 or vortex-spark_2.13) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 013d3ff0bec..0bd6691a913 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -4615,9 +4615,9 @@ } }, "node_modules/prettier": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", - "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.2.tgz", + "integrity": "sha512-8c3mgTe0ASwWAJK+78dpviD+A8EqhndQPUBpNUIPt6+xWlIigCwfN01lWr9MAede4uqXGTEKeQWTvzb3vjia0Q==", "dev": true, "license": "MIT", "bin": { From ceb8638e9627042c62753f2d96cf51162fa59420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Mon, 13 Apr 2026 13:22:29 +0200 Subject: [PATCH 018/250] chore[vortex-array]: add integer casting benchmark (#7400) The motivation is to compare an upcoming perf improvement. ## Summary ## Testing Signed-off-by: Alfonso Subiotto Marques Co-authored-by: Adam Gutglick --- vortex-array/Cargo.toml | 4 +++ vortex-array/benches/cast_primitive.rs | 37 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 vortex-array/benches/cast_primitive.rs diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 462e85a1364..74fa772820c 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -96,6 +96,10 @@ serde_json = { workspace = true } serde_test = { workspace = true } vortex-array = { path = ".", features = ["_test-harness", "table-display"] } +[[bench]] +name = "cast_primitive" +harness = false + [[bench]] name = "search_sorted" harness = false diff --git a/vortex-array/benches/cast_primitive.rs b/vortex-array/benches/cast_primitive.rs new file mode 100644 index 00000000000..3e1b0831736 --- /dev/null +++ b/vortex-array/benches/cast_primitive.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use divan::Bencher; +use rand::prelude::*; +use vortex_array::IntoArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; + +fn main() { + divan::main(); +} + +const N: usize = 100_000; + +#[divan::bench] +fn cast_u16_to_u32(bencher: Bencher) { + let mut rng = StdRng::seed_from_u64(42); + #[expect(clippy::cast_possible_truncation)] + let arr = PrimitiveArray::from_option_iter((0..N).map(|i| { + if rng.random_bool(0.5) { + None + } else { + Some(i as u16) + } + })) + .into_array(); + bencher.with_inputs(|| arr.clone()).bench_refs(|a| { + #[expect(clippy::unwrap_used)] + a.cast(DType::Primitive(PType::U32, Nullability::Nullable)) + .unwrap() + .to_canonical() + }); +} From 75b1062d5e86dc9ed36f743f6ff93f2d369bb932 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 13 Apr 2026 13:53:32 +0100 Subject: [PATCH 019/250] Ignore dead_code warnings on scalar bit transpose code (#7408) Likely we shouldn't use expect here but we also don't test on architectures where this wouldn't be dead code Signed-off-by: Robert Kruszewski --- encodings/fastlanes/src/bit_transpose/scalar.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/encodings/fastlanes/src/bit_transpose/scalar.rs b/encodings/fastlanes/src/bit_transpose/scalar.rs index 152b2a830c1..425e74087ab 100644 --- a/encodings/fastlanes/src/bit_transpose/scalar.rs +++ b/encodings/fastlanes/src/bit_transpose/scalar.rs @@ -12,6 +12,7 @@ use crate::bit_transpose::TRANSPOSE_8X8; /// This version uses 64-bit gather + parallel bit operations instead of /// extracting bits one by one. Typically 5-10x faster than the basic scalar version. #[inline(never)] +#[allow(dead_code)] pub fn transpose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { // Helper to perform 8x8 bit transpose on a u64 (each byte becomes a row) fn transpose_8x8(mut x: u64) -> u64 { @@ -84,6 +85,7 @@ pub fn transpose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { /// Fast scalar untranspose using the 8x8 bit matrix transpose algorithm. #[inline(never)] +#[allow(dead_code)] pub fn untranspose_bits_scalar(input: &[u8; 128], output: &mut [u8; 128]) { fn transpose_8x8(mut x: u64) -> u64 { let t = (x ^ (x >> 7)) & TRANSPOSE_2X2; From d4e7dca9ec3423fb25edfc648e620fe1b099b80d Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Mon, 13 Apr 2026 08:56:12 -0400 Subject: [PATCH 020/250] support multiple globs in DuckDB and MultiFileDataSource (#7390) ## Summary Adds support for multiple globs in DuckDB, similar to Parquet, E.g. instead of `read_vortex('*.vortex')` now you can also do `read_vortex(['s3://bucket1/path/*.vortex', '/a/b/*.vortex'])` Note that multiple file systems are supported in a single scan. This is implemented and wired through into the underlying Rust MultiFileDataSource. ## Testing We add new DuckDB e2e tests for multi-glob Couldn't figure out a great way to test multiple file systems, just multiple globs --------- Signed-off-by: Andrew Duffy --- benchmarks/datafusion-bench/src/main.rs | 3 +- vortex-duckdb/cpp/table_function.cpp | 2 + .../src/e2e_test/vortex_scan_test.rs | 34 +++++ vortex-duckdb/src/lib.rs | 4 + vortex-duckdb/src/multi_file.rs | 98 +++++++++++--- vortex-file/public-api.lock | 4 +- vortex-file/src/multi/mod.rs | 120 +++++++++++------- 7 files changed, 197 insertions(+), 68 deletions(-) diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index 920d4346361..ad0df8ea74a 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -317,8 +317,7 @@ async fn register_v2_tables( }; let multi_ds = MultiFileDataSource::new(SESSION.clone()) - .with_filesystem(fs) - .with_glob(glob_pattern) + .with_glob(glob_pattern, Some(fs)) .build() .await?; diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index fc4ac1b66da..6154e91f738 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -383,6 +383,8 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d auto &system_catalog = Catalog::GetSystemCatalog(*db); auto data = CatalogTransaction::GetSystemTransaction(*db); CreateTableFunctionInfo tf_info(tf); + // Allow registering multiple overloads with the same name but different parameter types. + tf_info.on_conflict = OnCreateConflict::ALTER_ON_CONFLICT; system_catalog.CreateFunction(data, tf_info); } catch (...) { return DuckDBError; diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs index 4e03cd07cd7..ed057eb709a 100644 --- a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs +++ b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs @@ -350,6 +350,40 @@ fn test_vortex_scan_multiple_files() { assert_eq!(total_sum, 21); } +#[test] +fn test_vortex_scan_multiple_globs() { + // Test scanning multiple directories using a list of glob patterns. + let (tempdir1, tempdir2, _file1, _file2, _file3) = RUNTIME.block_on(async { + let tempdir1 = tempfile::tempdir().unwrap(); + let tempdir2 = tempfile::tempdir().unwrap(); + + let file1 = write_vortex_file_to_dir(tempdir1.path(), "numbers", buffer![1i32, 2, 3]).await; + let file2 = write_vortex_file_to_dir(tempdir1.path(), "numbers", buffer![4i32, 5, 6]).await; + let file3 = + write_vortex_file_to_dir(tempdir2.path(), "numbers", buffer![7i32, 8, 9, 10]).await; + + (tempdir1, tempdir2, file1, file2, file3) + }); + + // Create glob patterns for each directory. + let glob_pattern1 = format!("{}/*.vortex", tempdir1.path().display()); + let glob_pattern2 = format!("{}/*.vortex", tempdir2.path().display()); + + // Scan files from both directories using a list of globs. + let conn = database_connection(); + let result = conn + .query(&format!( + "SELECT SUM(numbers) FROM read_vortex(['{glob_pattern1}', '{glob_pattern2}'])" + )) + .unwrap(); + let chunk = result.into_iter().next().unwrap(); + let vec = chunk.get_vector(0); + let total_sum = vec.as_slice_with_len::(chunk.len().as_())[0]; + + // 1+2+3 + 4+5+6 + 7+8+9+10 = 55 + assert_eq!(total_sum, 55); +} + #[test] fn test_vortex_scan_over_http() { let file = RUNTIME.block_on(async { diff --git a/vortex-duckdb/src/lib.rs b/vortex-duckdb/src/lib.rs index 2e45685d8d1..413a71c611e 100644 --- a/vortex-duckdb/src/lib.rs +++ b/vortex-duckdb/src/lib.rs @@ -21,6 +21,7 @@ use crate::duckdb::DatabaseRef; use crate::duckdb::LogicalType; use crate::duckdb::Value; use crate::multi_file::VortexMultiFileScan; +use crate::multi_file::VortexMultiFileScanList; mod convert; mod datasource; @@ -56,6 +57,9 @@ pub fn initialize(db: &DatabaseRef) -> VortexResult<()> { )?; db.register_table_function::(c"vortex_scan")?; db.register_table_function::(c"read_vortex")?; + // Register list overloads for multi-glob scanning (e.g., read_vortex(['a.vortex', 'b.vortex'])) + db.register_table_function::(c"vortex_scan")?; + db.register_table_function::(c"read_vortex")?; db.register_copy_function::(c"vortex", c"vortex") } diff --git a/vortex-duckdb/src/multi_file.rs b/vortex-duckdb/src/multi_file.rs index 84a843e09cf..14c8e02befb 100644 --- a/vortex-duckdb/src/multi_file.rs +++ b/vortex-duckdb/src/multi_file.rs @@ -5,18 +5,23 @@ use std::path::Path; use std::path::absolute; use std::sync::Arc; +use itertools::Itertools; use url::Url; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_err; use vortex::file::multi::MultiFileDataSource; +use vortex::io::filesystem::FileSystemRef; use vortex::io::runtime::BlockingRuntime; use vortex::scan::DataSourceRef; +use vortex_utils::aliases::hash_map::HashMap; use crate::RUNTIME; use crate::SESSION; use crate::datasource::DataSourceTableFunction; use crate::duckdb::BindInputRef; use crate::duckdb::ClientContextRef; +use crate::duckdb::ExtractedValue; use crate::duckdb::LogicalType; use crate::filesystem::resolve_filesystem; @@ -60,34 +65,95 @@ fn normalize_path(path: std::path::PathBuf) -> std::path::PathBuf { #[derive(Debug)] pub struct VortexMultiFileScan; +/// Variant of [`VortexMultiFileScan`] that accepts a list of globs. +/// +/// This is registered as a separate overload to handle `read_vortex(['a.vortex', 'b.vortex'])`. +#[derive(Debug)] +pub struct VortexMultiFileScanList; + impl DataSourceTableFunction for VortexMultiFileScan { fn parameters() -> Vec { vec![LogicalType::varchar()] } fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { - let glob_url_parameter = input - .get_parameter(0) - .ok_or_else(|| vortex_err!("Missing file glob parameter"))?; + bind_multi_file_scan(ctx, input) + } +} + +impl DataSourceTableFunction for VortexMultiFileScanList { + fn parameters() -> Vec { + vec![ + LogicalType::list_type(LogicalType::varchar()) + .unwrap_or_else(|_| unreachable!("creating list type should not fail")), + ] + } + + fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { + bind_multi_file_scan(ctx, input) + } +} - // Parse the URL and separate the base URL (keep scheme, host, etc.) from the path. - let glob_url_string = glob_url_parameter.as_string(); - let glob_url_str = glob_url_string.as_str(); - let glob_url = parse_glob_url(glob_url_str)?; +/// Shared bind logic for both single-glob and multi-glob variants. +fn bind_multi_file_scan( + ctx: &ClientContextRef, + input: &BindInputRef, +) -> VortexResult { + let glob_url_parameter = input + .get_parameter(0) + .ok_or_else(|| vortex_err!("Missing file glob parameter"))?; + // The input to the table function can either be a single glob, or a List of glob patterns. + let glob_strings: Vec = match glob_url_parameter.extract() { + ExtractedValue::Varchar(glob) => { + vec![glob.to_string()] + } + ExtractedValue::List(globs) => globs + .into_iter() + .map(|glob| { + let ExtractedValue::Varchar(string) = glob.extract() else { + vortex_bail!("list element must be Varchar type") + }; + + Ok(string.to_string()) + }) + .try_collect()?, + _ => vortex_bail!("Invalid argument to read_vortex table function"), + }; + + // Parse each glob URL and resolve its filesystem. + let mut glob_urls: Vec = Vec::with_capacity(glob_strings.len()); + for glob_str in &glob_strings { + glob_urls.push(parse_glob_url(glob_str)?); + } + + // Cache filesystems by base URL to avoid resolving the same filesystem multiple times. + let mut fs_cache: HashMap = HashMap::new(); + for glob_url in &glob_urls { let mut base_url = glob_url.clone(); base_url.set_path(""); + if !fs_cache.contains_key(&base_url) { + let fs = resolve_filesystem(&base_url, ctx)?; + fs_cache.insert(base_url, fs); + } + } - let fs = resolve_filesystem(&base_url, ctx)?; + RUNTIME.block_on(async { + let mut builder = MultiFileDataSource::new(SESSION.clone()); - RUNTIME.block_on(async { - let builder = MultiFileDataSource::new(SESSION.clone()) - .with_filesystem(fs) - .with_glob(glob_url.path()); - let ds = builder.build().await?; - VortexResult::Ok(Arc::new(ds) as DataSourceRef) - }) - } + for glob_url in &glob_urls { + let mut base_url = glob_url.clone(); + base_url.set_path(""); + let fs = fs_cache + .get(&base_url) + .map(Arc::clone) + .unwrap_or_else(|| unreachable!("fs should be cached for all base URLs")); + builder = builder.with_glob(glob_url.path(), Some(fs)); + } + + let ds = builder.build().await?; + Ok(Arc::new(ds) as DataSourceRef) + }) } #[cfg(test)] diff --git a/vortex-file/public-api.lock b/vortex-file/public-api.lock index 59ec83b7fe9..f5be6aa8f23 100644 --- a/vortex-file/public-api.lock +++ b/vortex-file/public-api.lock @@ -10,9 +10,7 @@ pub async fn vortex_file::multi::MultiFileDataSource::build(self) -> vortex_erro pub fn vortex_file::multi::MultiFileDataSource::new(session: vortex_session::VortexSession) -> Self -pub fn vortex_file::multi::MultiFileDataSource::with_filesystem(self, fs: vortex_io::filesystem::FileSystemRef) -> Self - -pub fn vortex_file::multi::MultiFileDataSource::with_glob(self, glob: impl core::convert::Into) -> Self +pub fn vortex_file::multi::MultiFileDataSource::with_glob(self, glob: impl core::convert::Into, fs: core::option::Option) -> Self pub fn vortex_file::multi::MultiFileDataSource::with_open_options(self, f: impl core::ops::function::Fn(vortex_file::VortexOpenOptions) -> vortex_file::VortexOpenOptions + core::marker::Send + core::marker::Sync + 'static) -> Self diff --git a/vortex-file/src/multi/mod.rs b/vortex-file/src/multi/mod.rs index 2e0c3735499..bf687f97ec1 100644 --- a/vortex-file/src/multi/mod.rs +++ b/vortex-file/src/multi/mod.rs @@ -13,7 +13,6 @@ use session::MultiFileSessionExt; use tracing::debug; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_err; use vortex_io::filesystem::FileListing; use vortex_io::filesystem::FileSystemRef; use vortex_layout::LayoutReaderRef; @@ -26,33 +25,41 @@ use crate::OpenOptionsSessionExt; use crate::VortexOpenOptions; use crate::v2::FileStatsLayoutReader; -/// A builder that discovers multiple Vortex files from a glob pattern and constructs a +/// A builder that discovers multiple Vortex files from glob patterns and constructs a /// [`MultiLayoutDataSource`] to scan them as a single data source. /// -/// The primary interface is [`Self::with_glob`], which accepts a glob -/// pattern (optionally prefixed with `file://`). For non-local filesystems (S3, GCS, etc.), -/// callers must also provide a [`FileSystemRef`] via [`Self::with_filesystem`]). +/// The primary interface is [`Self::with_glob`], which accepts a glob pattern and an optional +/// filesystem. For non-local filesystems (S3, GCS, etc.), callers must provide a [`FileSystemRef`]. +/// For local files, pass `None` and a local filesystem will be created automatically. /// /// # Examples /// /// ```ignore /// // Local files — filesystem is auto-created: /// let ds = MultiFileDataSource::new(session) -/// .with_glob("/data/warehouse/*.vortex") +/// .with_glob("/data/warehouse/*.vortex", None) /// .build() /// .await?; /// /// // S3 — caller provides the filesystem: /// let ds = MultiFileDataSource::new(session) -/// .with_filesystem(s3_fs) -/// .with_glob("prefix/*.vortex") +/// .with_glob("prefix/*.vortex", Some(s3_fs)) +/// .build() +/// .await?; +/// +/// // Mixed filesystems — multiple globs with different filesystems: +/// let ds = MultiFileDataSource::new(session) +/// .with_glob("bucket-a/*.vortex", Some(s3_fs.clone())) +/// .with_glob("bucket-b/*.vortex", Some(s3_fs)) +/// .with_glob("gcs-bucket/*.vortex", Some(gcs_fs)) /// .build() /// .await?; /// ``` pub struct MultiFileDataSource { session: VortexSession, - fs: Option, - glob: Option, + /// List of (glob, optional filesystem) pairs to resolve. + /// When the filesystem is None, a local filesystem will be created in build(). + glob_sources: Vec<(String, Option)>, open_options_fn: Arc VortexOpenOptions + Send + Sync>, } @@ -61,26 +68,18 @@ impl MultiFileDataSource { pub fn new(session: VortexSession) -> Self { Self { session, - fs: None, - glob: None, + glob_sources: Vec::new(), open_options_fn: Arc::new(|opts| opts), } } - /// Set the path glob for file discovery. + /// Add a path glob for file discovery. /// - /// This path should be relative to the filesystem's base URL. - pub fn with_glob(mut self, glob: impl Into) -> Self { - self.glob = Some(glob.into().trim_start_matches("/").to_string()); - self - } - - /// Set the filesystem to use for file discovery and reading. - /// - /// Required for non-local URLs (S3, GCS, etc.). For `file://` or bare path URLs, - /// a local filesystem is created automatically if none is provided. - pub fn with_filesystem(mut self, fs: FileSystemRef) -> Self { - self.fs = Some(fs); + /// The glob path should be relative to the filesystem's base URL. Pass `None` for the + /// filesystem to use the local filesystem (auto-created in [`Self::build`]). + pub fn with_glob(mut self, glob: impl Into, fs: Option) -> Self { + let glob_str = glob.into().trim_start_matches('/').to_string(); + self.glob_sources.push((glob_str, fs)); self } @@ -99,36 +98,63 @@ impl MultiFileDataSource { /// /// Discovers files via glob, opens the first file eagerly to determine the schema, /// and creates lazy factories for the remaining files. - pub async fn build(mut self) -> VortexResult { - let glob = self - .glob - .take() - .ok_or_else(|| vortex_err!("MultiFileDataSource requires a glob URL"))?; - - let fs = match self.fs { - Some(fs) => fs, - None => create_local_filesystem(&self.session)?, - }; - let files: Vec = fs.glob(&glob)?.try_collect().await?; - - if files.is_empty() { - vortex_bail!("No files matched the glob pattern '{}'", glob); + pub async fn build(self) -> VortexResult { + if self.glob_sources.is_empty() { + vortex_bail!("MultiFileDataSource requires at least one glob pattern"); + } + + // Create local filesystem lazily if needed (only if any glob lacks a filesystem). + let local_fs: Option = self + .glob_sources + .iter() + .any(|(_, fs)| fs.is_none()) + .then(|| create_local_filesystem(&self.session)) + .transpose()?; + + // Collect files from all glob sources. + let mut all_files: Vec<(FileListing, FileSystemRef)> = Vec::new(); + for (glob, maybe_fs) in &self.glob_sources { + // Use the provided filesystem, or fall back to the local filesystem. + // We know local_fs is Some when maybe_fs is None (by construction above). + let fs = maybe_fs + .as_ref() + .or(local_fs.as_ref()) + .map(Arc::clone) + .unwrap_or_else(|| { + unreachable!("local_fs is set when any glob lacks a filesystem") + }); + let files: Vec = fs.glob(glob)?.try_collect().await?; + for file in files { + all_files.push((file, Arc::clone(&fs))); + } + } + + if all_files.is_empty() { + let globs: Vec<_> = self.glob_sources.iter().map(|(g, _)| g.as_str()).collect(); + vortex_bail!("No files matched the glob pattern(s): {:?}", globs); } - let file_count = files.len(); - debug!(file_count, glob = %glob, "discovered files"); + let file_count = all_files.len(); + let globs: Vec<_> = self.glob_sources.iter().map(|(g, _)| g.as_str()).collect(); + debug!(file_count, glob = ?globs, "discovered files"); // Open first file eagerly for dtype. - let first_file = - open_file(&fs, &files[0], &self.session, self.open_options_fn.as_ref()).await?; + let (first_file_listing, first_fs) = &all_files[0]; + let first_file = open_file( + first_fs, + first_file_listing, + &self.session, + self.open_options_fn.as_ref(), + ) + .await?; let first_reader = layout_reader_with_stats(&first_file)?; - let factories: Vec> = files[1..] + let factories: Vec> = all_files[1..] .iter() - .map(|f| { + .map(|(file, fs)| { Arc::new(VortexFileReaderFactory { - fs: Arc::clone(&fs), - file: f.clone(), + fs: Arc::clone(fs), + file: file.clone(), session: self.session.clone(), open_options_fn: Arc::clone(&self.open_options_fn), }) as Arc From fbfa072b5ae7b9a3efb7e16b67d73dc637d3bc13 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 13 Apr 2026 09:59:19 -0400 Subject: [PATCH 021/250] Optimize math expressions with `Constant` children (#7394) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Adds some constant array optimizations to the tensor crate expressions `L2Norm`, `L2Denorm`, and `CosineSimilarity`. The remaining expressions `InnerProduct` and `SorfTransform` are a bit more complicated and deserve their own PR. ## Testing Adds more tests for these optimizations. --------- Signed-off-by: Connor Tsui --- .../src/scalar_fns/cosine_similarity.rs | 120 +++++++- vortex-tensor/src/scalar_fns/l2_denorm.rs | 289 +++++++++++++++++- vortex-tensor/src/scalar_fns/l2_norm.rs | 13 +- vortex-tensor/src/utils.rs | 2 + 4 files changed, 407 insertions(+), 17 deletions(-) diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index f4f129c2b27..2a717b90b7b 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -32,6 +32,7 @@ use vortex_error::vortex_ensure; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::try_build_constant_l2_denorm; use crate::scalar_fns::l2_norm::L2Norm; use crate::utils::extract_l2_denorm_children; use crate::utils::validate_tensor_float_input; @@ -133,6 +134,20 @@ impl ScalarFnVTable for CosineSimilarity { let mut rhs_ref = args.get(1)?; let len = args.row_count(); + // If either side is a constant tensor-like extension array, eagerly normalize the single + // stored row and re-wrap it as an `L2Denorm` whose children are both [`ConstantArray`]s. + // The L2Denorm fast path below then picks it up. + if let Some(lhs_constant) = + try_build_constant_l2_denorm(&lhs_ref, len, ctx)?.map(|sfn| sfn.into_array()) + { + lhs_ref = lhs_constant; + } + if let Some(rhs_constant) = + try_build_constant_l2_denorm(&rhs_ref, len, ctx)?.map(|sfn| sfn.into_array()) + { + rhs_ref = rhs_constant; + } + // Check if any of our children have be already normalized. { let lhs_is_denorm = lhs_ref.is::>(); @@ -249,8 +264,9 @@ impl CosineSimilarity { let (normalized, _) = extract_l2_denorm_children(denorm_ref); let dot_arr = InnerProduct::try_new_array(normalized, plain_ref.clone(), len)?; - let norm_arr = L2Norm::try_new_array(plain_ref.clone(), len)?; let dot: PrimitiveArray = dot_arr.into_array().execute(ctx)?; + + let norm_arr = L2Norm::try_new_array(plain_ref.clone(), len)?; let plain_norm: PrimitiveArray = norm_arr.into_array().execute(ctx)?; // TODO(connor): Ideally we would have a `SafeDiv` binary numeric operation. @@ -575,4 +591,106 @@ mod tests { assert_close(&[prim.as_slice::()[0]], &[1.0]); Ok(()) } + + #[test] + fn constant_lhs_matches_plain_tensor() -> VortexResult<()> { + // The constant query `[1, 2, 2]` has norm 3, so its normalized form is `[1/3, 2/3, 2/3]`. + // Expected cosine similarity against each row is `dot([1, 2, 2], row) / (3 * ||row||)`. + let lhs = constant_tensor_array(&[3], &[1.0, 2.0, 2.0], 4)?; + let rhs = tensor_array( + &[3], + &[ + 1.0, 0.0, 0.0, // dot=1, ||rhs||=1, expected=1/3 + 1.0, 2.0, 2.0, // dot=9, ||rhs||=3, expected=1 + 0.0, 0.0, 1.0, // dot=2, ||rhs||=1, expected=2/3 + 2.0, 1.0, 2.0, // dot=8, ||rhs||=3, expected=8/9 + ], + )?; + assert_close( + &eval_cosine_similarity(lhs, rhs, 4)?, + &[1.0 / 3.0, 1.0, 2.0 / 3.0, 8.0 / 9.0], + ); + Ok(()) + } + + #[test] + fn constant_rhs_matches_plain_tensor() -> VortexResult<()> { + // Mirror of `constant_lhs_matches_plain_tensor` with the constant on the right. + let lhs = tensor_array( + &[3], + &[ + 1.0, 0.0, 0.0, // + 1.0, 2.0, 2.0, // + 0.0, 0.0, 1.0, // + 2.0, 1.0, 2.0, // + ], + )?; + let rhs = constant_tensor_array(&[3], &[1.0, 2.0, 2.0], 4)?; + assert_close( + &eval_cosine_similarity(lhs, rhs, 4)?, + &[1.0 / 3.0, 1.0, 2.0 / 3.0, 8.0 / 9.0], + ); + Ok(()) + } + + #[test] + fn both_constant_tensors() -> VortexResult<()> { + // `[1, 0, 0]` vs `[1, 1, 0]`. dot=1, ||lhs||=1, ||rhs||=sqrt(2), expected=1/sqrt(2). + let lhs = constant_tensor_array(&[3], &[1.0, 0.0, 0.0], 3)?; + let rhs = constant_tensor_array(&[3], &[1.0, 1.0, 0.0], 3)?; + let expected = 1.0 / 2.0_f64.sqrt(); + assert_close( + &eval_cosine_similarity(lhs, rhs, 3)?, + &[expected, expected, expected], + ); + Ok(()) + } + + #[test] + fn constant_zero_norm_query() -> VortexResult<()> { + // A zero-norm constant query must produce `0.0` for every row via the zero-norm guard in + // `execute_one_denorm` and `execute_both_denorm`. + let lhs = constant_tensor_array(&[3], &[0.0, 0.0, 0.0], 3)?; + let rhs = tensor_array( + &[3], + &[ + 1.0, 2.0, 3.0, // + 4.0, 5.0, 6.0, // + 7.0, 8.0, 9.0, // + ], + )?; + assert_close(&eval_cosine_similarity(lhs, rhs, 3)?, &[0.0, 0.0, 0.0]); + Ok(()) + } + + #[test] + fn constant_self_similarity_nonunit() -> VortexResult<()> { + // A non-unit constant query compared to itself must produce `1.0`. This exercises the + // helper's division: after normalization, both sides must be exactly unit so the + // L2Denorm fast path's inner product yields 1. + let lhs = constant_tensor_array(&[3], &[3.0, 4.0, 0.0], 5)?; + let rhs = constant_tensor_array(&[3], &[3.0, 4.0, 0.0], 5)?; + assert_close(&eval_cosine_similarity(lhs, rhs, 5)?, &[1.0; 5]); + Ok(()) + } + + #[test] + fn vector_constant_matches_plain() -> VortexResult<()> { + // Exercise the `Vector` extension variant through the new pre-pass. + let lhs = constant_vector_array(&[1.0, 2.0, 2.0], 4)?; + let rhs = vector_array( + 3, + &[ + 1.0, 0.0, 0.0, // + 1.0, 2.0, 2.0, // + 0.0, 0.0, 1.0, // + 2.0, 1.0, 2.0, // + ], + )?; + assert_close( + &eval_cosine_similarity(lhs, rhs, 4)?, + &[1.0 / 3.0, 1.0, 2.0 / 3.0, 8.0 / 9.0], + ); + Ok(()) + } } diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index d72940c8da8..c0f0f54fb12 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -5,22 +5,31 @@ use std::fmt::Formatter; +use num_traits::Float; use num_traits::ToPrimitive; use num_traits::Zero; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::expr::Expression; use vortex_array::expr::and; use vortex_array::match_each_float_ptype; +use vortex_array::scalar::Scalar; +use vortex_array::scalar::ScalarValue; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; @@ -28,6 +37,7 @@ use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -183,8 +193,31 @@ impl ScalarFnVTable for L2Denorm { args: &dyn ExecutionArgs, ctx: &mut ExecutionCtx, ) -> VortexResult { - let normalized: ExtensionArray = args.get(0)?.execute(ctx)?; - let norms: PrimitiveArray = args.get(1)?.execute(ctx)?; + let normalized_ref = args.get(0)?; + let norms_ref = args.get(1)?; + let output_dtype = normalized_ref + .dtype() + .union_nullability(norms_ref.dtype().nullability()); + let validity = normalized_ref.validity()?.and(norms_ref.validity()?)?; + + if let Some(const_norms) = norms_ref.as_opt::() { + let norm_scalar = const_norms.scalar(); + vortex_ensure!(norm_scalar.dtype().is_float()); + + if let Some(norm_value) = norm_scalar.value() { + return execute_l2_denorm_constant_norms( + normalized_ref, + norm_scalar, + norm_value, + output_dtype, + validity, + ctx, + ); + } + } + + let normalized: ExtensionArray = normalized_ref.execute(ctx)?; + let norms: PrimitiveArray = norms_ref.execute(ctx)?; let row_count = args.row_count(); let tensor_match = normalized @@ -194,10 +227,6 @@ impl ScalarFnVTable for L2Denorm { .vortex_expect("we already validated this in `return_dtype`"); let tensor_flat_size = tensor_match.list_size(); - let validity = normalized.as_ref().validity()?.and(norms.validity()?)?; - let output_dtype = normalized - .dtype() - .union_nullability(norms.dtype().nullability()); let flat = extract_flat_elements(normalized.storage_array(), tensor_flat_size, ctx)?; // TODO(connor): Theoretically we could model this as a multiplication between the @@ -243,6 +272,55 @@ impl ScalarFnVTable for L2Denorm { } } +/// Optimized execution when the norms array is constant. +fn execute_l2_denorm_constant_norms( + normalized_ref: ArrayRef, + norm_scalar: &Scalar, + norm_value: &ScalarValue, + output_dtype: DType, + new_validity: Validity, + ctx: &mut ExecutionCtx, +) -> VortexResult { + // If the norms are all equal to 1 then we don't need to do anything. + let err = norm_value + .as_primitive() + .as_f64() + .vortex_expect("we know that this is a float, so it must fit in f64") + - 1.0f64; + + let tolerance = unit_norm_tolerance(norm_scalar.dtype().as_ptype()); + if err.abs() < tolerance { + return Ok(normalized_ref); + } + + // Even if the norms are not all 1, if they are all the same then we can multiply + // the entire elements array by the same number. + let normalized: ExtensionArray = normalized_ref.execute(ctx)?; + let storage_fsl: FixedSizeListArray = normalized.storage_array().clone().execute(ctx)?; + + // Replace the elements array with an array that multiplies it by the constant + // norms array (with length multiplied by the dimensions of the vectors). + let const_array = + ConstantArray::new(norm_scalar.clone(), storage_fsl.elements().len()).into_array(); + let mult_elements = storage_fsl + .elements() + .clone() + .binary(const_array, Operator::Mul)?; + + // SAFETY: We just updated the elements of the array with a scalar fn, so all + // invariants still hold. + let new_fsl = unsafe { + FixedSizeListArray::new_unchecked( + mult_elements, + storage_fsl.list_size(), + new_validity, + storage_fsl.len(), + ) + }; + + Ok(ExtensionArray::new(output_dtype.as_extension().clone(), new_fsl.into_array()).into_array()) +} + /// Builds an unexecuted [`L2Denorm`] expression by normalizing `input` and reattaching the exact /// norms as the norms child. /// @@ -274,17 +352,25 @@ pub fn normalize_as_l2_denorm( let tensor_match = validate_tensor_float_input(input.dtype())?; let tensor_flat_size = tensor_match.list_size(); + // Constant fast path: if the input is a constant-backed extension, normalize the single + // stored row once and return an `L2Denorm` whose children are both `ConstantArray`s. + if let Some(wrapped) = try_build_constant_l2_denorm(&input, row_count, ctx)? { + return Ok(wrapped); + } + + // Calculate the norms of the vectors. let norms_sfn = L2Norm::try_new_array(input.clone(), row_count)?; let norms_array: ArrayRef = norms_sfn.into_array().execute(ctx)?; - let norms: PrimitiveArray = norms_array.clone().execute(ctx)?; - let norms_validity = norms.validity()?; + let primitive_norms: PrimitiveArray = norms_array.clone().execute(ctx)?; + let norms_validity = primitive_norms.validity()?; let input: ExtensionArray = input.execute(ctx)?; let normalized_dtype = input.dtype().as_nonnullable(); let flat = extract_flat_elements(input.storage_array(), tensor_flat_size, ctx)?; + // Normalize all of the vectors. let normalized = match_each_float_ptype!(flat.ptype(), |T| { - let norm_values = norms.as_slice::(); + let norm_values = primitive_norms.as_slice::(); let total_elements = row_count * tensor_flat_size; let mut elements = BufferMut::::with_capacity(total_elements); @@ -305,6 +391,8 @@ pub fn normalize_as_l2_denorm( } } + // Since L2Denorm's validity is the `and` of its child validities, we can make the + // normalized array non-nullable. build_tensor_array( normalized_dtype, tensor_flat_size, @@ -325,6 +413,90 @@ pub fn normalize_as_l2_denorm( unsafe { L2Denorm::new_array_unchecked(normalized, norms_array, row_count) } } +/// Attempts to build an [`L2Denorm`] whose two children are both [`ConstantArray`]s by eagerly +/// normalizing `input`'s single stored row. +/// +/// Returns `Ok(None)` when `input` is not a tensor-like extension array whose storage is a +/// [`ConstantArray`] with a non-null fixed-size-list scalar. +/// +/// When `input` matches, the returned [`ScalarFnArray`] is equivalent to [`normalize_as_l2_denorm`] +/// but runs in `O(list_size)` time instead of `O(row_count * list_size)`. +/// +/// This is helpful in some of the reduction steps for cosine similarity execution into inner +/// product execution. +pub(crate) fn try_build_constant_l2_denorm( + input: &ArrayRef, + len: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + let Some(ext) = input.as_opt::() else { + return Ok(None); + }; + let storage = ext.storage_array(); + let Some(const_storage) = storage.as_opt::() else { + return Ok(None); + }; + if const_storage.scalar().is_null() { + return Ok(None); + } + + // The caller is expected to have already validated that `input` is an `AnyTensor` + // extension dtype. + let tensor_match = input + .dtype() + .as_extension() + .metadata_opt::() + .vortex_expect("caller validated input has AnyTensor metadata"); + let list_size = tensor_match.list_size(); + let original_nullability = input.dtype().nullability(); + let ext_dtype = input.dtype().as_extension().clone(); + let storage_fsl_nullability = storage.dtype().nullability(); + + // `extract_flat_elements` takes the stride-0 single-row path for `Constant` storage, so + // this is cheap and does not expand the constant to the full column length. + let flat = extract_flat_elements(storage, list_size, ctx)?; + + let (normalized_fsl_scalar, norms_scalar) = match_each_float_ptype!(flat.ptype(), |T| { + let row = flat.row::(0); + + let mut sum_sq = T::zero(); + for &x in row { + sum_sq += x * x; + } + let norm_t: T = sum_sq.sqrt(); + + // Zero-norm rows must be stored as all-zeros so [`L2Denorm`]'s unit-norm-or-zero + // invariant holds. This mirrors the per-row logic in `normalize_as_l2_denorm`. + let element_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable); + let children: Vec = if norm_t == T::zero() { + (0..list_size) + .map(|_| Scalar::zero_value(&element_dtype)) + .collect() + } else { + row.iter() + .map(|&v| Scalar::primitive(v / norm_t, Nullability::NonNullable)) + .collect() + }; + + // The rebuilt FSL scalar preserves the original storage FSL's nullability so the + // resulting `ExtensionArray::new` call accepts the same extension dtype. + let fsl_scalar = Scalar::fixed_size_list(element_dtype, children, storage_fsl_nullability); + let norms_scalar = Scalar::primitive(norm_t, original_nullability); + (fsl_scalar, norms_scalar) + }); + + let normalized_storage = ConstantArray::new(normalized_fsl_scalar, len).into_array(); + let normalized_ext = ExtensionArray::new(ext_dtype, normalized_storage).into_array(); + let norms_array = ConstantArray::new(norms_scalar, len).into_array(); + + // SAFETY: Each row of `normalized_ext` is either `v / ||v||` (unit norm within floating + // point tolerance) or all zeros when `||v|| == 0`. Stored norms are non-negative by + // construction (`sqrt`). These are exactly the invariants required by + // [`L2Denorm::new_array_unchecked`]. + let wrapped = unsafe { L2Denorm::new_array_unchecked(normalized_ext, norms_array, len)? }; + Ok(Some(wrapped)) +} + /// Rebuilds a tensor-like extension array from flat primitive elements. /// /// # Errors @@ -403,6 +575,7 @@ fn validate_l2_normalized_rows_impl( let normalized: ExtensionArray = normalized.clone().execute(ctx)?; let normalized_validity = normalized.as_ref().validity()?; + let flat = extract_flat_elements(normalized.storage_array(), tensor_flat_size, ctx)?; let norms = norms .map(|norms| norms.clone().execute::(ctx)) @@ -463,6 +636,9 @@ mod tests { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::MaskedArray; @@ -471,10 +647,12 @@ mod tests { use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; use vortex_array::extension::EmptyMetadata; use vortex_array::extension::datetime::Date; use vortex_array::extension::datetime::TimeUnit; + use vortex_array::scalar::Scalar; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; @@ -747,6 +925,44 @@ mod tests { Ok(()) } + #[test] + fn normalize_as_l2_denorm_constant_input_has_constant_children() -> VortexResult<()> { + // The constant fast path in `normalize_as_l2_denorm` must produce an `L2Denorm` whose + // normalized storage and norms child are both still `ConstantArray`s. This is what + // allows downstream ops (cosine similarity, inner product) to short-circuit. + let input = constant_vector_array(&[3.0, 4.0], 16)?; + let mut ctx = SESSION.create_execution_ctx(); + let roundtrip = normalize_as_l2_denorm(input, &mut ctx)?; + + // The normalized child must be an extension array whose storage is still constant. + let normalized = roundtrip.child_at(0).clone(); + let normalized_ext = normalized + .as_opt::() + .expect("normalized child should be an Extension array"); + assert!( + normalized_ext + .storage_array() + .as_opt::() + .is_some(), + "normalized storage should stay constant after the fast path" + ); + + // The norms child must itself be a ConstantArray with the exact precomputed norm. + let norms = roundtrip.child_at(1).clone(); + let norms_const = norms + .as_opt::() + .expect("norms child should be a ConstantArray"); + assert_close( + &[norms_const + .scalar() + .as_primitive() + .typed_value::() + .expect("norms scalar")], + &[5.0], + ); + Ok(()) + } + #[test] fn normalize_as_l2_denorm_uses_zero_rows_for_zero_norms() -> VortexResult<()> { let input = vector_array(2, &[0.0, 0.0, 3.0, 4.0])?; @@ -761,4 +977,59 @@ mod tests { assert_tensor_arrays_eq(actual, input)?; Ok(()) } + + /// Builds a non-nullable constant f64 norms array of length `len`. + fn constant_f64_norms(value: f64, len: usize) -> ArrayRef { + ConstantArray::new(Scalar::primitive(value, Nullability::NonNullable), len).into_array() + } + + #[test] + fn l2_denorm_constant_unit_norms_is_noop() -> VortexResult<()> { + // Every stored norm is exactly 1.0, so the constant fast path must short-circuit and + // return the normalized child unchanged. + let normalized = vector_array(3, &[1.0, 0.0, 0.0, 0.0, 1.0, 0.0])?; + let norms = constant_f64_norms(1.0, 2); + + let actual = eval_l2_denorm(normalized.clone(), norms, 2)?; + assert_tensor_arrays_eq(actual, normalized)?; + Ok(()) + } + + #[test] + fn l2_denorm_constant_near_unit_norms_is_noop() -> VortexResult<()> { + // A norm that differs from 1.0 by less than the f64 unit-norm tolerance must still + // hit the fast path and return the normalized child unchanged. + let normalized = vector_array(3, &[1.0, 0.0, 0.0, 0.0, 1.0, 0.0])?; + let norms = constant_f64_norms(1.0 + 1e-12, 2); + + let actual = eval_l2_denorm(normalized.clone(), norms, 2)?; + assert_tensor_arrays_eq(actual, normalized)?; + Ok(()) + } + + #[test] + fn l2_denorm_constant_nonunit_norms_scales_vectors() -> VortexResult<()> { + // A constant norm that is not 1.0 must scale every element of every row by the same + // factor via the backing elements multiplication path. + let normalized = vector_array(3, &[0.6, 0.8, 0.0, 1.0, 0.0, 0.0])?; + let norms = constant_f64_norms(5.0, 2); + + let actual = eval_l2_denorm(normalized, norms, 2)?; + let expected = vector_array(3, &[3.0, 4.0, 0.0, 5.0, 0.0, 0.0])?; + assert_tensor_arrays_eq(actual, expected)?; + Ok(()) + } + + #[test] + fn l2_denorm_constant_nonunit_norms_scales_fixed_shape_tensors() -> VortexResult<()> { + // The same constant-scaling fast path must also cover multi-dimensional fixed-shape + // tensors, where the backing elements buffer spans more than one slot per row. + let normalized = tensor_array(&[2, 2], &[0.5, 0.5, 0.5, 0.5, 1.0, 0.0, 0.0, 0.0])?; + let norms = constant_f64_norms(4.0, 2); + + let actual = eval_l2_denorm(normalized, norms, 2)?; + let expected = tensor_array(&[2, 2], &[2.0, 2.0, 2.0, 2.0, 4.0, 0.0, 0.0, 0.0])?; + assert_tensor_arrays_eq(actual, expected)?; + Ok(()) + } } diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 170ac800d10..59bd9d77009 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -12,9 +12,8 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; -use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayVTable; use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; @@ -125,17 +124,17 @@ impl ScalarFnVTable for L2Norm { // L2Norm(L2Denorm(normalized, norms)) is defined to read back the authoritative stored // norms. Exact callers of lossy encodings like TurboQuant opt into that storage semantics // instead of forcing a decode-and-recompute path here. - if let Some(sfn) = input_ref.as_opt::() - && sfn.scalar_fn().as_opt::().is_some() - { - let norms: PrimitiveArray = sfn.child_at(1).clone().execute(ctx)?; + if let Some(sfn) = input_ref.as_opt::>() { + let norms = sfn + .nth_child(1) + .vortex_expect("L2Denom must have at 2 children"); vortex_ensure_eq!( norms.dtype(), &DType::Primitive(element_ptype, input_ref.dtype().nullability()) ); - return Ok(norms.into_array()); + return Ok(norms); } let input: ExtensionArray = input_ref.execute(ctx)?; diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 325a530ee46..76d65ce9eef 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -103,6 +103,8 @@ impl FlatElements { } } +// TODO(connor): Usage of this function is sometimes incorrect / not performant. +// Make sure to fix them. /// Extracts the flat primitive elements from a tensor storage array (FixedSizeList). /// /// When the input is a [`ConstantArray`] (e.g., a literal query vector), only a single row is From a004afab4520a2a0185ef325df7a56caa44c2ac2 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:45:34 -0400 Subject: [PATCH 022/250] TurboQuant `InnerProduct` optimizations (#7396) _Stacked on top of https://github.com/vortex-data/vortex/pull/7394._ ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Implements the final pushdown / reduction rules needed to make the TurboQuant quantization scheme make sense. TODO ## Testing More tests --------- Signed-off-by: Connor Tsui Signed-off-by: Claude Co-authored-by: Claude --- vortex-tensor/benches/similarity_search.rs | 2 +- vortex-tensor/src/scalar_fns/inner_product.rs | 1067 +++++++++++++++++ 2 files changed, 1068 insertions(+), 1 deletion(-) diff --git a/vortex-tensor/benches/similarity_search.rs b/vortex-tensor/benches/similarity_search.rs index 695eae75025..1112eda11b2 100644 --- a/vortex-tensor/benches/similarity_search.rs +++ b/vortex-tensor/benches/similarity_search.rs @@ -33,7 +33,7 @@ use common::generate_random_vectors; static GLOBAL: MiMalloc = MiMalloc; /// Number of vectors in the benchmark dataset. -const NUM_ROWS: usize = 10_000; +const NUM_ROWS: usize = 100_000; /// Dimensionality of each vector. Must be `>= vortex_tensor::encodings::turboquant::MIN_DIMENSION` /// (128) for the TurboQuant variant to work. diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 71288c241ae..c883cf00982 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -4,22 +4,34 @@ //! Inner product expression for tensor-like types. use std::fmt::Formatter; +use std::sync::Arc; use num_traits::Float; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Dict; +use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::FixedSizeList; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; +use vortex_array::dtype::PType; +use vortex_array::dtype::extension::ExtDType; use vortex_array::expr::Expression; use vortex_array::expr::and; +use vortex_array::extension::EmptyMetadata; use vortex_array::match_each_float_ptype; +use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; @@ -28,6 +40,7 @@ use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -35,8 +48,11 @@ use vortex_error::vortex_err; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::sorf_transform::SorfMatrix; +use crate::scalar_fns::sorf_transform::SorfTransform; use crate::utils::extract_flat_elements; use crate::utils::extract_l2_denorm_children; +use crate::vector::Vector; /// Inner product (dot product) between two columns. /// @@ -161,6 +177,20 @@ impl ScalarFnVTable for InnerProduct { } } + // Reduction case 1: `InnerProduct(SorfTransform(x), const)` rewrites to + // `InnerProduct(x, forward_rotate(zero_pad(const)))`. Re-executes recursively so + // case 2 can fire on the rewritten tree. + if let Some(rewritten) = self.try_execute_sorf_constant(&lhs_ref, &rhs_ref, len, ctx)? { + return Ok(rewritten); + } + + // Reduction case 2: `InnerProduct(Vector[FSL(Dict(u8, f32))], const)` is computed by + // gather-summing `q[j] * values[codes[j] as usize]` per row, reading the codebook + // directly instead of decoding the column into dense vectors. + if let Some(result) = self.try_execute_dict_constant(&lhs_ref, &rhs_ref, len, ctx)? { + return Ok(result); + } + // Compute combined validity. let validity = lhs_ref.validity()?.and(rhs_ref.validity()?)?; @@ -275,6 +305,245 @@ impl InnerProduct { Ok(unsafe { PrimitiveArray::new_unchecked(buffer, validity) }.into_array()) }) } + + /// Fast path when one side is `ExactScalarFn` and the other side is a + /// constant-backed tensor-like extension. Rewrites to + /// `InnerProduct(sorf_child, forward_rotate(zero_pad(const_query)))` because SORF is + /// orthogonal, so ` = ` where `T` is the truncation + /// from `padded_dim` to `dim` applied by `SorfTransform` and `R` is the SORF forward + /// matrix. See the proof in the crate-level docs and in the plan file. + /// + /// Returns `Ok(None)` if neither side matches or when `element_ptype` is not `F32`. The + /// caller is expected to fall through to the standard path in that case. + /// + /// # TODO(connor): + /// + /// This rewrite is only sound for `PType::F32` because `SorfTransform` applies an + /// `f32 -> element_ptype` cast at the end of its execute (see `sorf_transform/vtable.rs` + /// line ~218). For F16/F64 the cast changes the inner product's rounding and would + /// change the semantics of the rewrite. Until we push the cast through `InnerProduct`, + /// this path only fires for F32. + fn try_execute_sorf_constant( + &self, + lhs_ref: &ArrayRef, + rhs_ref: &ArrayRef, + len: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // Identify which side is the SorfTransform, if any. + let (sorf_view, const_ref) = + if let Some(view) = lhs_ref.as_opt::>() { + (view, rhs_ref) + } else if let Some(view) = rhs_ref.as_opt::>() { + (view, lhs_ref) + } else { + return Ok(None); + }; + + // TODO(connor): pull-through is only sound for F32 because SorfTransform applies an + // `f32 -> element_ptype` cast at the end of its execute. For F16/F64 the rewrite + // would change the inner product's rounding semantics. Fall through so the standard + // path (which does the cast before inner product) handles it. + if sorf_view.options.element_ptype != PType::F32 { + return Ok(None); + } + + // The other side must be a constant-backed tensor-like extension whose scalar is + // non-null. + let Some(const_ext) = const_ref.as_opt::() else { + return Ok(None); + }; + let const_storage = const_ext.storage_array(); + let Some(const_backing) = const_storage.as_opt::() else { + return Ok(None); + }; + if const_backing.scalar().is_null() { + return Ok(None); + } + + let dim = sorf_view.options.dimension as usize; + let num_rounds = sorf_view.options.num_rounds as usize; + let seed = sorf_view.options.seed; + let padded_dim = dim.next_power_of_two(); + + // Extract the single stored row of the constant via the stride-0 short-circuit. + let flat = extract_flat_elements(const_storage, dim, ctx)?; + if flat.ptype() != PType::F32 { + // TODO(connor): as above, f16/f64 are not supported by this rewrite yet. The + // standard path handles them correctly. + return Ok(None); + } + + // Zero-pad the query from `dim` to `padded_dim` and forward-rotate. + let mut padded_query = vec![0.0f32; padded_dim]; + padded_query[..dim].copy_from_slice(flat.row::(0)); + + let rotation = SorfMatrix::try_new(seed, dim, num_rounds)?; + let mut rotated_query = vec![0.0f32; padded_dim]; + rotation.rotate(&padded_query, &mut rotated_query); + + // Build the rewritten constant as a `Vector` extension wrapping a + // `ConstantArray` of length `len`. We reuse the original storage FSL nullability so + // the new extension dtype stays consistent with whatever the original tree expected. + let storage_fsl_nullability = const_storage.dtype().nullability(); + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let children: Vec = rotated_query + .into_iter() + .map(|v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let fsl_scalar = + Scalar::fixed_size_list(element_dtype.clone(), children, storage_fsl_nullability); + let new_storage = ConstantArray::new(fsl_scalar, len).into_array(); + + // Build a fresh `Vector` extension dtype. We cannot reuse the + // original extension dtype because that one has `dim`, not `padded_dim`. + let padded_dim_u32 = u32::try_from(padded_dim).vortex_expect("padded_dim fits u32"); + let new_fsl_dtype = DType::FixedSizeList( + Arc::new(element_dtype), + padded_dim_u32, + storage_fsl_nullability, + ); + let new_ext_dtype = ExtDType::::try_new(EmptyMetadata, new_fsl_dtype)?.erased(); + let new_constant = ExtensionArray::new(new_ext_dtype, new_storage).into_array(); + + // Extract the SorfTransform child (the already-padded Vector). + let sorf_child = sorf_view + .nth_child(0) + .vortex_expect("SorfTransform must have exactly one child"); + + // Recursively execute the rewritten inner product. This allows case 2 to fire on + // the rewritten tree if the sorf child is `Vector[FSL(Dict)]`. Termination is + // guaranteed because the rewrite strictly removes a `SorfTransform` scalar-fn node + // from the tree and SORFs cannot be nested. + let rewritten = InnerProduct::try_new_array(sorf_child, new_constant, len)? + .into_array() + .execute(ctx)?; + Ok(Some(rewritten)) + } + + /// Fast path when one side is an extension whose storage is `FSL(Dict(u8, f32))` and + /// the other side is a constant-backed tensor extension with an F32 element ptype. + /// + /// Computes each row's inner product as + /// `out[i] = sum_{j in 0..padded_dim} q[j] * values[codes[i * padded_dim + j] as usize]` + /// using a direct codebook lookup in the hot loop. An explicit product table + /// `P[j, k] = q[j] * values[k]` (size `padded_dim * num_centroids * 4B`, ~1 MiB for the + /// common 1024/256 case) was tried and measured ~10% *slower* on the + /// `similarity_search` bench because the 1 KiB `values` table stays in L1 across all + /// rows, while the 1 MiB product table does not. + /// + /// Returns `Ok(None)` when the pattern doesn't match; the caller should fall through to + /// the standard path. + fn try_execute_dict_constant( + &self, + lhs_ref: &ArrayRef, + rhs_ref: &ArrayRef, + len: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // Try each orientation. The oriented helper navigates each side exactly once, so + // the only redundant work here is the failed navigation of the first side when the + // dict happens to be on the right. + if let Some(result) = self.try_execute_dict_constant_oriented(lhs_ref, rhs_ref, len, ctx)? { + return Ok(Some(result)); + } + self.try_execute_dict_constant_oriented(rhs_ref, lhs_ref, len, ctx) + } + + /// Orientation-specific helper for [`Self::try_execute_dict_constant`]. `dict_candidate` + /// is tried as `Extension[FSL[Dict]]`; `const_candidate` is tried as a constant-backed + /// tensor extension. Returns `Ok(None)` if either navigation fails or any gate rejects. + fn try_execute_dict_constant_oriented( + &self, + dict_candidate: &ArrayRef, + const_candidate: &ArrayRef, + len: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // Navigate the dict side. + let Some(dict_ext) = dict_candidate.as_opt::() else { + return Ok(None); + }; + let Some(fsl) = dict_ext.storage_array().as_opt::() else { + return Ok(None); + }; + let Some(dict) = fsl.elements().as_opt::() else { + return Ok(None); + }; + + // Navigate the constant side and require its scalar be non-null. + let Some(const_ext) = const_candidate.as_opt::() else { + return Ok(None); + }; + let const_storage = const_ext.storage_array(); + let Some(const_backing) = const_storage.as_opt::() else { + return Ok(None); + }; + if const_backing.scalar().is_null() { + return Ok(None); + } + + // Canonicalize codes and values. Codes may be e.g. BitPacked; executing is cheaper + // than falling through to the standard path (which would also canonicalize). + let codes_prim: PrimitiveArray = dict.codes().clone().execute(ctx)?; + let values_prim: PrimitiveArray = dict.values().clone().execute(ctx)?; + + // Gate: u8 codes and f32 centroids. + if codes_prim.ptype() != PType::U8 { + // TODO(connor): support wider code widths (u16, u32). TurboQuant only emits u8 + // codes today, so this is the only path we need for now. + return Ok(None); + } + if values_prim.ptype() != PType::F32 { + // TODO(connor): direct-lookup path only supports f32 centroids. SorfTransform + // forces f32 anyway, so this is the only shape we need for now. + return Ok(None); + } + + let padded_dim = usize::try_from(fsl.list_size()).vortex_expect("fsl list_size fits usize"); + + let flat = extract_flat_elements(const_storage, padded_dim, ctx)?; + if flat.ptype() != PType::F32 { + // TODO(connor): case 2 is f32-only. For f16/f64 we fall through to the standard + // path, which computes the inner product with the correct element type. + return Ok(None); + } + + // Combine the input validities up front; the per-row arithmetic may write garbage + // into null rows but the validity mask hides it (matching the standard path). + let validity = dict_candidate + .validity()? + .and(const_candidate.validity()?)?; + + // Fast path for the empty case: skip allocating and touching the codes buffer. + if len == 0 { + let empty = PrimitiveArray::empty::(validity.nullability()); + return Ok(Some(empty.into_array())); + } + + let q: &[f32] = flat.row::(0); + debug_assert_eq!(q.len(), padded_dim); + let codes: &[u8] = codes_prim.as_slice::(); + let values: &[f32] = values_prim.as_slice::(); + debug_assert_eq!(codes.len(), len * padded_dim); + + // Direct codebook lookup in the hot loop. See the function doc comment for why this + // beats an explicit product table here. + let mut out = BufferMut::::with_capacity(len); + for row in 0..len { + let row_codes = &codes[row * padded_dim..(row + 1) * padded_dim]; + let mut acc = 0.0f32; + for j in 0..padded_dim { + acc += q[j] * values[row_codes[j] as usize]; + } + // SAFETY: we reserved `len` slots above and push exactly once per row. + unsafe { out.push_unchecked(acc) }; + } + + // SAFETY: the buffer length equals `len`, which matches the validity length. + let result = unsafe { PrimitiveArray::new_unchecked(out.freeze(), validity) }.into_array(); + Ok(Some(result)) + } } /// Computes the inner product (dot product) of two equal-length float slices. @@ -506,4 +775,802 @@ mod tests { assert_close(&[prim.as_slice::()[0]], &[25.0]); Ok(()) } + + // ---- Tests for the `SorfTransform + constant` and `Dict + constant` fast paths ---- + + #[allow( + clippy::cast_possible_truncation, + reason = "tests build small fixtures with deterministic in-range indices" + )] + mod constant_query_optimizations { + use std::sync::LazyLock; + + use rstest::rstest; + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::ExtensionArray; + use vortex_array::arrays::FixedSizeListArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::ScalarFnArray; + use vortex_array::arrays::dict::DictArray; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::extension::EmptyMetadata; + use vortex_array::scalar::Scalar; + use vortex_array::session::ArraySession; + use vortex_array::validity::Validity; + use vortex_buffer::Buffer; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + + use crate::scalar_fns::inner_product::InnerProduct; + use crate::scalar_fns::sorf_transform::SorfMatrix; + use crate::scalar_fns::sorf_transform::SorfOptions; + use crate::scalar_fns::sorf_transform::SorfTransform; + use crate::vector::Vector; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + + /// Compact f32 Vector extension over a column-major `elements` slice. + fn vector_f32(dim: u32, elements: &[f32]) -> VortexResult { + let row_count = elements.len() / dim as usize; + let elems: ArrayRef = Buffer::copy_from(elements).into_array(); + let fsl = FixedSizeListArray::new(elems, dim, Validity::NonNullable, row_count); + let ext_dtype = + ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + } + + /// Compact constant-backed f32 Vector extension with a single stored row. + fn constant_vector_f32(elements: &[f32], len: usize) -> VortexResult { + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let children: Vec = elements + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = + Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + let storage = ConstantArray::new(storage_scalar, len).into_array(); + let ext_dtype = + ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, storage).into_array()) + } + + /// Build an `ExtensionArray>` whose storage is + /// `FSL(DictArray(codes: u8, values: f32))`. This mirrors the shape that + /// TurboQuant produces as the SorfTransform child. + fn dict_vector_f32(list_size: u32, codes: &[u8], values: &[f32]) -> VortexResult { + let num_rows = codes.len() / list_size as usize; + let codes_arr = + PrimitiveArray::new::(Buffer::copy_from(codes), Validity::NonNullable) + .into_array(); + let values_arr = + PrimitiveArray::new::(Buffer::copy_from(values), Validity::NonNullable) + .into_array(); + let dict = DictArray::try_new(codes_arr, values_arr)?; + let fsl = FixedSizeListArray::try_new( + dict.into_array(), + list_size, + Validity::NonNullable, + num_rows, + )?; + let ext_dtype = + ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + } + + /// Execute an inner product and return the flat `f32` results. + fn eval_ip_f32(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { + let scalar_fn = InnerProduct::new().erased(); + let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], len)?; + let mut ctx = SESSION.create_execution_ctx(); + let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; + Ok(prim.as_slice::().to_vec()) + } + + fn assert_close_f32(actual: &[f32], expected: &[f32], tol: f32) { + assert_eq!(actual.len(), expected.len(), "length mismatch"); + for (i, (a, e)) in actual.iter().zip(expected).enumerate() { + assert!( + (a - e).abs() < tol, + "row {i}: got {a}, expected {e} (diff = {})", + (a - e).abs() + ); + } + } + + /// Build a SorfTransform ScalarFnArray whose child is a `Vector` + /// wrapping `FSL(Dict(codes, values))`. Returns `(sorf_array, codes, values, + /// padded_dim)`. + fn build_sorf_with_dict_child( + dim: u32, + num_rows: usize, + seed: u64, + num_rounds: u8, + ) -> VortexResult<(ArrayRef, Vec, Vec, usize)> { + let padded_dim = (dim as usize).next_power_of_two(); + // Small hand-picked codebook of 8 f32 centroids. + let values: Vec = vec![-1.5, -1.0, -0.5, -0.1, 0.1, 0.5, 1.0, 1.5]; + // Deterministic codes in 0..values.len() covering every position. + let codes: Vec = (0..num_rows * padded_dim) + .map(|i| (i as u8) % (values.len() as u8)) + .collect(); + + let padded_vector = dict_vector_f32(padded_dim as u32, &codes, &values)?; + let sorf_options = SorfOptions { + seed, + num_rounds, + dimension: dim, + element_ptype: PType::F32, + }; + let sorf = + SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array(); + Ok((sorf, codes, values, padded_dim)) + } + + /// Decode a SorfTransform-wrapped dict-vector to a flat `Vec` of `num_rows * + /// dim` post-rotation, post-truncation values. This is the ground truth against + /// which we compare the fast-path result. + fn decode_sorf_dict( + codes: &[u8], + values: &[f32], + padded_dim: usize, + dim: usize, + num_rows: usize, + seed: u64, + num_rounds: u8, + ) -> VortexResult> { + let rotation = SorfMatrix::try_new(seed, dim, num_rounds as usize)?; + let mut padded = vec![0.0f32; padded_dim]; + let mut rotated = vec![0.0f32; padded_dim]; + let mut out = Vec::with_capacity(num_rows * dim); + for row in 0..num_rows { + for j in 0..padded_dim { + padded[j] = values[codes[row * padded_dim + j] as usize]; + } + rotation.inverse_rotate(&padded, &mut rotated); + out.extend_from_slice(&rotated[..dim]); + } + Ok(out) + } + + fn naive_dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(&x, &y)| x * y).sum() + } + + // ---- Case 1: SorfTransform + Constant pull-through ---- + + /// Case 1: SorfTransform on LHS, constant query on RHS, with `dim < padded_dim` + /// so the zero-padding branch is exercised. + #[test] + fn case1_sorf_lhs_constant_rhs_padded_gt_dim() -> VortexResult<()> { + let dim: u32 = 100; + let num_rows = 7usize; + let seed = 42u64; + let num_rounds = 3u8; + let padded_dim = (dim as usize).next_power_of_two(); + assert!(padded_dim > dim as usize, "test must exercise padding"); + + let (sorf_lhs, codes, values, padded_dim_computed) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + assert_eq!(padded_dim_computed, padded_dim); + + // Query has `dim` elements. + let query_elems: Vec = (0..dim).map(|i| (i as f32 * 0.1).sin()).collect(); + let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + + // Ground truth: decode LHS to plain f32 vectors, dot each with the query. + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| { + naive_dot( + &decoded[i * dim as usize..(i + 1) * dim as usize], + &query_elems, + ) + }) + .collect(); + + let actual = eval_ip_f32(sorf_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-3); + Ok(()) + } + + /// Case 1: SorfTransform on RHS, constant query on LHS (mirrored). + #[test] + fn case1_constant_lhs_sorf_rhs_mirrored() -> VortexResult<()> { + let dim: u32 = 100; + let num_rows = 5usize; + let seed = 7u64; + let num_rounds = 3u8; + + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + let query_elems: Vec = (0..dim).map(|i| (i as f32 * 0.2).cos()).collect(); + let const_lhs = constant_vector_f32(&query_elems, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| { + naive_dot( + &decoded[i * dim as usize..(i + 1) * dim as usize], + &query_elems, + ) + }) + .collect(); + + let actual = eval_ip_f32(const_lhs, sorf, num_rows)?; + assert_close_f32(&actual, &expected, 1e-3); + Ok(()) + } + + /// Case 1: `dim == padded_dim` (power-of-two, no zero padding). + #[test] + fn case1_padded_equals_dim() -> VortexResult<()> { + let dim: u32 = 128; + let num_rows = 4usize; + let seed = 11u64; + let num_rounds = 3u8; + + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + assert_eq!(padded_dim, dim as usize); + + let query_elems: Vec = (0..dim).map(|i| i as f32 * 0.01 - 0.5).collect(); + let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| { + naive_dot( + &decoded[i * dim as usize..(i + 1) * dim as usize], + &query_elems, + ) + }) + .collect(); + + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-3); + Ok(()) + } + + /// Case 1: empty `len == 0`. The fast path should handle this without exploding. + #[test] + fn case1_empty_len_zero() -> VortexResult<()> { + let dim: u32 = 100; + let num_rows = 0usize; + let seed = 42u64; + let num_rounds = 3u8; + + let (sorf, _codes, _values, _padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + let query_elems: Vec = vec![0.0; dim as usize]; + let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_eq!(actual.len(), 0); + Ok(()) + } + + // ---- Case 2: Dict + Constant direct-lookup path ---- + + /// Case 2: Vector[FSL[Dict(u8, f32)]] on LHS, constant query on RHS. + #[test] + fn case2_dict_lhs_constant_rhs_matches_naive() -> VortexResult<()> { + let list_size: u32 = 8; + let num_rows = 10usize; + // 8 centroids, tiny table. + let values: Vec = vec![-1.0, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1.0]; + // Deterministic codes. + let codes: Vec = (0..num_rows * list_size as usize) + .map(|i| (i as u8) % (values.len() as u8)) + .collect(); + let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; + + let query: Vec = (0..list_size).map(|i| (i as f32 + 1.0) * 0.3).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let expected: Vec = (0..num_rows) + .map(|row| { + let mut acc = 0.0f32; + for j in 0..list_size as usize { + let k = codes[row * list_size as usize + j] as usize; + acc += query[j] * values[k]; + } + acc + }) + .collect(); + + let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-5); + Ok(()) + } + + /// Case 2: constant query on LHS, dict column on RHS (mirrored). + #[test] + fn case2_constant_lhs_dict_rhs_mirrored() -> VortexResult<()> { + let list_size: u32 = 4; + let num_rows = 6usize; + let values: Vec = vec![0.1, 0.4, 0.7, 1.0]; + let codes: Vec = (0..num_rows * list_size as usize) + .map(|i| ((i * 3) as u8) % (values.len() as u8)) + .collect(); + let dict_rhs = dict_vector_f32(list_size, &codes, &values)?; + + let query: Vec = vec![0.5, -1.0, 2.5, -0.25]; + let const_lhs = constant_vector_f32(&query, num_rows)?; + + let expected: Vec = (0..num_rows) + .map(|row| { + let mut acc = 0.0f32; + for j in 0..list_size as usize { + let k = codes[row * list_size as usize + j] as usize; + acc += query[j] * values[k]; + } + acc + }) + .collect(); + + let actual = eval_ip_f32(const_lhs, dict_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-5); + Ok(()) + } + + /// Case 2: dict with `u16` codes (and hence more than 256 values) falls through to + /// the standard path but still produces the correct result. The direct-lookup path + /// only handles `u8` codes today. + #[test] + fn case2_u16_codes_falls_through() -> VortexResult<()> { + let list_size: u32 = 4; + let num_rows = 3usize; + let num_values = 300usize; + let values: Vec = (0..num_values).map(|i| i as f32 * 0.01).collect(); + // Codes must be u16 because 300 > 255. dict_vector_f32 only supports u8 so we + // build the dict by hand here. + let codes_u16: Vec = (0..(num_rows * 4)) + .map(|i| (i % num_values) as u16) + .collect(); + let codes_arr = + PrimitiveArray::new::(Buffer::copy_from(codes_u16), Validity::NonNullable) + .into_array(); + let values_arr = + PrimitiveArray::new::(Buffer::copy_from(&values), Validity::NonNullable) + .into_array(); + let dict = DictArray::try_new(codes_arr, values_arr)?; + let fsl = FixedSizeListArray::try_new( + dict.into_array(), + list_size, + Validity::NonNullable, + num_rows, + )?; + let ext_dtype = + ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + let dict_lhs = ExtensionArray::new(ext_dtype, fsl.into_array()).into_array(); + + let query: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let const_rhs = constant_vector_f32(&query, num_rows)?; + + // Build expected by decoding by hand. + let expected: Vec = (0..num_rows) + .map(|row| { + let mut acc = 0.0f32; + for j in 0..4 { + let code = (row * 4 + j) % num_values; + acc += query[j] * values[code]; + } + acc + }) + .collect(); + + let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-5); + Ok(()) + } + + /// Case 2: plain (non-dict) FSL with a constant RHS falls through to the standard + /// path and produces the correct result. + #[test] + fn case2_plain_fsl_falls_through() -> VortexResult<()> { + let dim: u32 = 4; + let num_rows = 3usize; + let lhs_elems: Vec = (0..num_rows * dim as usize) + .map(|i| i as f32 * 0.25) + .collect(); + let plain_lhs = vector_f32(dim, &lhs_elems)?; + + let query: Vec = vec![1.0, 2.0, 3.0, 4.0]; + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let expected: Vec = (0..num_rows) + .map(|row| { + naive_dot( + &lhs_elems[row * dim as usize..(row + 1) * dim as usize], + &query, + ) + }) + .collect(); + + let actual = eval_ip_f32(plain_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-5); + Ok(()) + } + + /// Case 2: empty `len == 0` fast path returns an empty primitive array without + /// touching the codes buffer. + #[test] + fn case2_empty_len_zero() -> VortexResult<()> { + let list_size: u32 = 4; + let num_rows = 0usize; + let values: Vec = vec![0.0, 1.0, 2.0, 3.0]; + let codes: Vec = Vec::new(); + let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; + + let query: Vec = vec![0.0; 4]; + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; + assert_eq!(actual.len(), 0); + Ok(()) + } + + /// Case 1 + Case 2 end-to-end: the SorfTransform-wrapped dict column hits Case 1 + /// then Case 2 via recursive execution. + #[test] + fn end_to_end_sorf_plus_dict_cosine_path() -> VortexResult<()> { + let dim: u32 = 100; + let num_rows = 9usize; + let seed = 99u64; + let num_rounds = 3u8; + + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + let query_elems: Vec = (0..dim).map(|i| ((i as f32) * 0.15).sin() * 0.4).collect(); + let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + + // Ground truth via full decode + naive dot. + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| { + naive_dot( + &decoded[i * dim as usize..(i + 1) * dim as usize], + &query_elems, + ) + }) + .collect(); + + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-3); + Ok(()) + } + + // ---- Additional correctness / stress tests (all with loose tolerances) ---- + + /// A tiny in-place xorshift64 PRNG so these tests don't depend on `rand`. Producing + /// deterministic pseudo-random f32 values lets the correctness checks exercise + /// realistic data instead of smooth sin/cos patterns. + struct XorShift64(u64); + + impl XorShift64 { + fn new(seed: u64) -> Self { + // Any nonzero seed is fine; xorshift fixed-points at 0. + Self(seed.wrapping_add(0x9E37_79B9_7F4A_7C15)) + } + + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + /// Uniform f32 in `[-1.0, 1.0)`. + fn next_f32(&mut self) -> f32 { + // Top 24 bits -> mantissa in [0, 1), then shift to [-1, 1). + let bits = (self.next_u64() >> 40) as u32; // 24 bits + (bits as f32) / (1u32 << 24) as f32 * 2.0 - 1.0 + } + } + + /// Case 2 stress: u8-coded dict with 200 centroids (formerly blocked by the + /// `values.len() <= 256` gate). The direct-lookup path must now handle it. + #[test] + fn case2_large_u8_codebook_direct_lookup() -> VortexResult<()> { + let list_size: u32 = 16; + let num_rows = 20usize; + let num_centroids = 200usize; + assert!(num_centroids > 8 && num_centroids <= 256); + + let mut rng = XorShift64::new(0xDEAD_BEEF); + let values: Vec = (0..num_centroids).map(|_| rng.next_f32()).collect(); + let codes: Vec = (0..num_rows * list_size as usize) + .map(|_| (rng.next_u64() % num_centroids as u64) as u8) + .collect(); + + let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; + let query: Vec = (0..list_size).map(|_| rng.next_f32()).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let expected: Vec = (0..num_rows) + .map(|row| { + let mut acc = 0.0f32; + for j in 0..list_size as usize { + let k = codes[row * list_size as usize + j] as usize; + acc += query[j] * values[k]; + } + acc + }) + .collect(); + + let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-4); + Ok(()) + } + + /// Parameterized sweep over the full `InnerProduct(SorfTransform(Vector[FSL(Dict)]), + /// ConstantArray)` tree, exercising the case 1 + case 2 chain for a realistic mix + /// of dimensions, row counts, seeds, and number of SORF rounds. Tolerance is + /// deliberately loose because the rewrite introduces an f32-domain rotation that + /// accumulates a small numerical drift versus a naive decode. + #[rstest] + #[case::small_no_pad(128, 11, 1, 1)] + #[case::small_no_pad_rounds3(128, 23, 1_234, 3)] + #[case::small_padded(100, 17, 42, 3)] + #[case::mid_padded(200, 13, 2024, 3)] + #[case::mid_power_of_two(256, 31, 7, 3)] + #[case::larger_padded(300, 9, 99, 3)] + #[case::max_rounds(128, 5, 31_415, 5)] + fn case1_sorf_random_sweep( + #[case] dim: u32, + #[case] num_rows: usize, + #[case] seed: u64, + #[case] num_rounds: u8, + ) -> VortexResult<()> { + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + // Use a pseudo-random query with both positive and negative entries so the sum + // has cancellation. + let mut rng = XorShift64::new(seed ^ 0xABCD_1234); + let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| naive_dot(&decoded[i * dim as usize..(i + 1) * dim as usize], &query)) + .collect(); + + // Loose tolerance: the sorf transform works in f32 with a k-round butterfly, so + // the rewrite path and the decoded path accumulate slightly different rounding + // even though the math is equivalent in exact arithmetic. + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-2); + Ok(()) + } + + /// Parameterized sweep over plain `Vector[FSL(Dict(u8, f32))]` + constant query, + /// without SorfTransform in the mix. This directly exercises case 2 across a + /// variety of list sizes, num_rows, and codebook sizes including large ones that + /// the old `<= 256` gate would have rejected. + #[rstest] + #[case::small(4, 7, 8)] + #[case::medium(16, 50, 64)] + #[case::larger(32, 100, 150)] + #[case::very_large_codebook(8, 25, 250)] + fn case2_random_sweep( + #[case] list_size: u32, + #[case] num_rows: usize, + #[case] num_centroids: usize, + ) -> VortexResult<()> { + let mut rng = XorShift64::new((list_size as u64) * 31 + num_rows as u64); + let values: Vec = (0..num_centroids).map(|_| rng.next_f32()).collect(); + assert!(num_centroids <= 256, "u8 codes cap at 256 centroids"); + let codes: Vec = (0..num_rows * list_size as usize) + .map(|_| (rng.next_u64() % num_centroids as u64) as u8) + .collect(); + + let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; + let query: Vec = (0..list_size).map(|_| rng.next_f32()).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let expected: Vec = (0..num_rows) + .map(|row| { + let mut acc = 0.0f32; + for j in 0..list_size as usize { + let k = codes[row * list_size as usize + j] as usize; + acc += query[j] * values[k]; + } + acc + }) + .collect(); + + // Tight tolerance here because no SorfTransform rotation is involved — the + // arithmetic should agree bit-for-bit up to float reassociation. + let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-4); + Ok(()) + } + + /// End-to-end regression: for a plausible vector-search configuration (SORF rounds + /// = 3, dim = 128, num_rows = 64, u8 codes, 64 centroids), the fast-path result + /// must track a fully naive computation within 1e-2. + #[test] + fn end_to_end_dim128_rows64_bit6_regression() -> VortexResult<()> { + let dim: u32 = 128; + let num_rows = 64usize; + let seed = 0xFACE_F00D; + let num_rounds = 3u8; + + // Use 64 centroids (6 bits), a typical TurboQuant configuration. + let num_centroids = 64usize; + let padded_dim = (dim as usize).next_power_of_two(); + let mut rng = XorShift64::new(seed); + let values: Vec = (0..num_centroids).map(|_| rng.next_f32()).collect(); + let codes: Vec = (0..num_rows * padded_dim) + .map(|_| (rng.next_u64() % num_centroids as u64) as u8) + .collect(); + + let padded_vector = dict_vector_f32(padded_dim as u32, &codes, &values)?; + let sorf_options = SorfOptions { + seed, + num_rounds, + dimension: dim, + element_ptype: PType::F32, + }; + let sorf = + SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array(); + + let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| naive_dot(&decoded[i * dim as usize..(i + 1) * dim as usize], &query)) + .collect(); + + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-2); + + // Also verify the max relative error is small. The SORF rotation does not + // amplify error, so both measures should be bounded. + for (i, (a, e)) in actual.iter().zip(expected.iter()).enumerate() { + let denom = e.abs().max(1.0); + let rel = (a - e).abs() / denom; + assert!( + rel < 1e-3, + "row {i}: rel err {rel} too large (a={a}, e={e})" + ); + } + Ok(()) + } + + /// Case 1 + Case 2 end-to-end with varying `num_rounds`. The rotation becomes + /// progressively more chaotic as rounds increase, so this catches any off-by-one + /// bug in the round-indexing that would not show up in the 3-round default. + #[rstest] + #[case(1)] + #[case(2)] + #[case(3)] + #[case(4)] + #[case(5)] + fn case1_various_num_rounds(#[case] num_rounds: u8) -> VortexResult<()> { + let dim: u32 = 128; + let num_rows = 8usize; + let seed = 0x1234_5678; + + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + let mut rng = XorShift64::new(seed ^ (num_rounds as u64)); + let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); + let const_rhs = constant_vector_f32(&query, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| naive_dot(&decoded[i * dim as usize..(i + 1) * dim as usize], &query)) + .collect(); + + let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; + assert_close_f32(&actual, &expected, 1e-2); + Ok(()) + } + + /// Swap LHS and RHS on the full tree to prove the side-detection and the scalar + /// argument-order handling are symmetric for both cases simultaneously. + #[test] + fn end_to_end_constant_lhs_sorf_rhs_mirrored() -> VortexResult<()> { + let dim: u32 = 256; + let num_rows = 12usize; + let seed = 0xBEEF_CAFE; + let num_rounds = 3u8; + + let (sorf, codes, values, padded_dim) = + build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; + + let mut rng = XorShift64::new(seed); + let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); + let const_lhs = constant_vector_f32(&query, num_rows)?; + + let decoded = decode_sorf_dict( + &codes, + &values, + padded_dim, + dim as usize, + num_rows, + seed, + num_rounds, + )?; + let expected: Vec = (0..num_rows) + .map(|i| naive_dot(&decoded[i * dim as usize..(i + 1) * dim as usize], &query)) + .collect(); + + let actual = eval_ip_f32(const_lhs, sorf, num_rows)?; + assert_close_f32(&actual, &expected, 1e-2); + Ok(()) + } + } } From 44c511d3121ce46f5048df1996519d86d6737062 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 13 Apr 2026 16:35:44 +0100 Subject: [PATCH 023/250] duckdb: Exit earlier on Validity::AllFalse (#7411) 1. Make duckdb all-invalid exporter a unit type: If we set underlying vector's validity to "all false", duckdb won't read underlying values so you don't need to fill them 2. Exit early on exporter branches if validity is all false. Saves a ConstantArray creation + execute Signed-off-by: Mikhail Kot --- vortex-duckdb/src/exporter/all_invalid.rs | 33 +++++-------------- vortex-duckdb/src/exporter/bool.rs | 9 ++--- vortex-duckdb/src/exporter/constant.rs | 4 +-- vortex-duckdb/src/exporter/decimal.rs | 11 +++---- vortex-duckdb/src/exporter/dict.rs | 4 +-- vortex-duckdb/src/exporter/fixed_size_list.rs | 13 ++++---- vortex-duckdb/src/exporter/list.rs | 10 +++--- vortex-duckdb/src/exporter/list_view.rs | 10 +++--- vortex-duckdb/src/exporter/mod.rs | 3 +- vortex-duckdb/src/exporter/primitive.rs | 16 ++++----- vortex-duckdb/src/exporter/sequence.rs | 1 + vortex-duckdb/src/exporter/struct_.rs | 13 ++++---- vortex-duckdb/src/exporter/varbinview.rs | 12 +++---- vortex-duckdb/src/exporter/vector.rs | 4 +-- 14 files changed, 57 insertions(+), 86 deletions(-) diff --git a/vortex-duckdb/src/exporter/all_invalid.rs b/vortex-duckdb/src/exporter/all_invalid.rs index 873dde3c089..cc5d5da9efb 100644 --- a/vortex-duckdb/src/exporter/all_invalid.rs +++ b/vortex-duckdb/src/exporter/all_invalid.rs @@ -3,39 +3,25 @@ use vortex::array::ExecutionCtx; use vortex::error::VortexResult; -use vortex::error::vortex_ensure; -use crate::duckdb::LogicalTypeRef; -use crate::duckdb::Value; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; -struct AllInvalidExporter { - len: usize, - null_value: Value, -} +struct AllInvalidExporter; -pub(crate) fn new_exporter(len: usize, logical_type: &LogicalTypeRef) -> Box { - Box::new(AllInvalidExporter { - len, - null_value: Value::null(logical_type), - }) +pub(crate) fn new_exporter() -> Box { + Box::new(AllInvalidExporter {}) } impl ColumnExporter for AllInvalidExporter { fn export( &self, - offset: usize, - len: usize, + _offset: usize, + _len: usize, vector: &mut VectorRef, _ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - vortex_ensure!( - offset + len <= self.len, - "invalid exporter: offset + len must be less than or equal to len" - ); - - vector.reference_value(&self.null_value); + vector.set_all_false_validity(); Ok(()) } } @@ -43,7 +29,6 @@ impl ColumnExporter for AllInvalidExporter { #[cfg(test)] mod tests { use vortex::array::VortexSessionExecute; - use vortex::array::arrays::PrimitiveArray; use super::*; use crate::SESSION; @@ -52,12 +37,10 @@ mod tests { #[test] fn all_null_array() { - let arr = PrimitiveArray::from_option_iter::([None, None, None]); let ltype = LogicalType::int32(); + let mut chunk = DataChunk::new([ltype]); - let mut chunk = DataChunk::new([ltype.clone()]); - - new_exporter(arr.len(), <ype) + new_exporter() .export( 0, 3, diff --git a/vortex-duckdb/src/exporter/bool.rs b/vortex-duckdb/src/exporter/bool.rs index cb488ce5297..84fd17f0789 100644 --- a/vortex-duckdb/src/exporter/bool.rs +++ b/vortex-duckdb/src/exporter/bool.rs @@ -4,11 +4,11 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::BoolArray; use vortex::array::arrays::bool::BoolArrayExt; +use vortex::array::validity::Validity; use vortex::buffer::BitBuffer; use vortex::error::VortexResult; use vortex::mask::Mask; -use crate::duckdb::LogicalType; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; use crate::exporter::all_invalid; @@ -24,11 +24,12 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let len = array.len(); let bits = array.to_bit_buffer(); - let validity = array.validity()?.to_array(len).execute::(ctx)?; - if validity.all_false() { - return Ok(all_invalid::new_exporter(len, &LogicalType::bool())); + let validity = array.validity()?; + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let validity = validity.to_array(len).execute::(ctx)?; Ok(validity::new_exporter( validity, diff --git a/vortex-duckdb/src/exporter/constant.rs b/vortex-duckdb/src/exporter/constant.rs index 8bb05ac1edc..ade5fece825 100644 --- a/vortex-duckdb/src/exporter/constant.rs +++ b/vortex-duckdb/src/exporter/constant.rs @@ -65,14 +65,14 @@ impl ColumnExporter for ConstantExporter { fn export( &self, _offset: usize, - len: usize, + _len: usize, vector: &mut VectorRef, _ctx: &mut ExecutionCtx, ) -> VortexResult<()> { match self.value.as_ref() { None => { // TODO(ngates): would be good if DuckDB supported constant null vectors. - unsafe { vector.set_validity(&Mask::AllFalse(len), 0, len) }; + vector.set_all_false_validity(); } Some(value) => { vector.reference_value(value); diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 2fe226b7f31..5b52f6d6c53 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -8,9 +8,9 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::decimal::DecimalDataParts; use vortex::array::match_each_decimal_value_type; +use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::BigCast; -use vortex::dtype::DType; use vortex::dtype::DecimalDType; use vortex::dtype::DecimalType; use vortex::dtype::NativeDecimalType; @@ -19,7 +19,6 @@ use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::mask::Mask; -use crate::duckdb::LogicalType; use crate::duckdb::VectorBuffer; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; @@ -49,13 +48,11 @@ pub(crate) fn new_exporter( values, } = array.into_data_parts(); let dest_values_type = precision_to_duckdb_storage_size(&decimal_dtype)?; - let nullability = validity.nullability(); - let validity = validity.to_array(len).execute::(ctx)?; - if validity.all_false() { - let ltype = LogicalType::try_from(DType::Decimal(decimal_dtype, nullability))?; - return Ok(all_invalid::new_exporter(len, <ype)); + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let validity = validity.to_array(len).execute::(ctx)?; let exporter = if values_type == dest_values_type { match_each_decimal_value_type!(values_type, |D| { diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index 961c7e35b84..6377906940c 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -17,7 +17,6 @@ use vortex::dtype::IntegerPType; use vortex::error::VortexResult; use vortex::mask::Mask; -use crate::duckdb::LogicalType; use crate::duckdb::ReusableDict; use crate::duckdb::SelectionVector; use crate::duckdb::VectorRef; @@ -43,7 +42,6 @@ pub(crate) fn new_exporter_with_flatten( ) -> VortexResult> { // Grab the cache dictionary values. let values = array.values(); - let values_type: LogicalType = values.dtype().try_into()?; if let Some(constant) = values.as_opt::() { return constant::new_exporter_with_mask( ConstantArray::new(constant.scalar().clone(), array.codes().len()), @@ -57,7 +55,7 @@ pub(crate) fn new_exporter_with_flatten( match codes_mask { Mask::AllTrue(_) => {} - Mask::AllFalse(len) => return Ok(all_invalid::new_exporter(len, &values_type)), + Mask::AllFalse(_) => return Ok(all_invalid::new_exporter()), Mask::Values(_) => { // duckdb cannot have a dictionary with validity in the codes, so flatten the array and // apply the validity mask there. diff --git a/vortex-duckdb/src/exporter/fixed_size_list.rs b/vortex-duckdb/src/exporter/fixed_size_list.rs index d7fbfa64647..ed93ad2b9c1 100644 --- a/vortex-duckdb/src/exporter/fixed_size_list.rs +++ b/vortex-duckdb/src/exporter/fixed_size_list.rs @@ -11,6 +11,7 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::FixedSizeListArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex::array::validity::Validity; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -18,7 +19,6 @@ use super::ConversionCache; use super::all_invalid; use super::new_array_exporter_with_flatten; use super::validity; -use crate::duckdb::LogicalType; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; @@ -42,15 +42,14 @@ pub(crate) fn new_exporter( let parts = array.into_data_parts(); let elements = parts.elements; let validity = parts.validity; - let dtype = parts.dtype; - let mask = validity.to_array(len).execute::(ctx)?; - let elements_exporter = new_array_exporter_with_flatten(elements, cache, ctx, true)?; - if mask.all_false() { - let ltype = LogicalType::try_from(dtype)?; - return Ok(all_invalid::new_exporter(len, <ype)); + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let mask = validity.to_array(len).execute::(ctx)?; + let elements_exporter = new_array_exporter_with_flatten(elements, cache, ctx, true)?; + Ok(validity::new_exporter( mask, Box::new(FixedSizeListExporter { diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index 1ae14869908..c532242fae7 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -10,6 +10,7 @@ use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::list::ListDataParts; use vortex::array::match_each_integer_ptype; +use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -49,15 +50,14 @@ pub(crate) fn new_exporter( elements, offsets, validity, - dtype, + dtype: _dtype, } = array.into_data_parts(); let num_elements = elements.len(); - let validity = validity.to_array(array_len).execute::(ctx)?; - if validity.all_false() { - let ltype = LogicalType::try_from(dtype)?; - return Ok(all_invalid::new_exporter(array_len, <ype)); + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let validity = validity.to_array(array_len).execute::(ctx)?; let values_key = elements.addr(); // Check if we have a cached vector and extract it if we do. diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index dcc7c66ce31..4e88d8f4199 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -10,7 +10,7 @@ use vortex::array::arrays::ListViewArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::listview::ListViewDataParts; use vortex::array::match_each_integer_ptype; -use vortex::dtype::DType; +use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -56,13 +56,11 @@ pub(crate) fn new_exporter( } = array.into_data_parts(); // Cache an `elements` vector up front so that future exports can reference it. let num_elements = elements.len(); - let nullability = validity.nullability(); - let validity = validity.to_array(len).execute::(ctx)?; - if validity.all_false() { - let ltype = LogicalType::try_from(DType::List(elements_dtype, nullability))?; - return Ok(all_invalid::new_exporter(len, <ype)); + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let validity = validity.to_array(len).execute::(ctx)?; let values_key = elements.addr(); // Check if we have a cached vector and extract it if we do. diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index b5b26340efe..f76b3e41124 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -37,7 +37,6 @@ use vortex::error::VortexResult; use vortex::error::vortex_bail; use crate::duckdb::DataChunkRef; -use crate::duckdb::LogicalType; use crate::duckdb::VectorRef; use crate::duckdb::duckdb_vector_size; @@ -166,7 +165,7 @@ fn new_array_exporter_with_flatten( // Otherwise, we fall back to canonical match array.execute::(ctx)? { - Canonical::Null(array) => Ok(all_invalid::new_exporter(array.len(), &LogicalType::null())), + Canonical::Null(_) => Ok(all_invalid::new_exporter()), Canonical::Bool(array) => bool::new_exporter(array, ctx), Canonical::Primitive(array) => primitive::new_exporter(array, ctx), Canonical::Decimal(array) => decimal::new_exporter(array, ctx), diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs index 7506cbdb93c..1bbec2d79c8 100644 --- a/vortex-duckdb/src/exporter/primitive.rs +++ b/vortex-duckdb/src/exporter/primitive.rs @@ -6,11 +6,11 @@ use std::marker::PhantomData; use vortex::array::ExecutionCtx; use vortex::array::arrays::PrimitiveArray; use vortex::array::match_each_native_ptype; +use vortex::array::validity::Validity; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; -use crate::duckdb::LogicalType; use crate::duckdb::VectorBuffer; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; @@ -28,15 +28,11 @@ pub fn new_exporter( array: PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let validity = array - .validity()? - .to_array(array.len()) - .execute::(ctx)?; - - if validity.all_false() { - let ltype = LogicalType::try_from(array.ptype())?; - return Ok(all_invalid::new_exporter(array.len(), <ype)); - } + let validity = array.validity()?; + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); + }; + let validity = validity.to_array(array.len()).execute::(ctx)?; match_each_native_ptype!(array.ptype(), |T| { let buffer = array.to_buffer::(); diff --git a/vortex-duckdb/src/exporter/sequence.rs b/vortex-duckdb/src/exporter/sequence.rs index a9a569d2a06..bc4b2731b40 100644 --- a/vortex-duckdb/src/exporter/sequence.rs +++ b/vortex-duckdb/src/exporter/sequence.rs @@ -35,6 +35,7 @@ impl ColumnExporter for SequenceExporter { ) -> VortexResult<()> { let offset = offset.as_i64(); let start = (offset * self.step) + self.start; + // TODO why don't we apply validity mask here? vector.to_sequence(start, self.step, len.as_u64()); Ok(()) diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index f53ec437bf2..9a07028686b 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -8,9 +8,9 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; +use vortex::array::validity::Validity; use vortex::error::VortexResult; -use crate::duckdb::LogicalType; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; use crate::exporter::ConversionCache; @@ -30,16 +30,15 @@ pub(crate) fn new_exporter( let len = array.len(); let StructDataParts { validity, - struct_fields, + struct_fields: _struct_fields, fields, .. } = array.into_data_parts(); - let validity = validity.to_array(len).execute::(ctx)?; - if validity.to_bit_buffer().true_count() == 0 { - let ltype = LogicalType::try_from(struct_fields)?; - return Ok(all_invalid::new_exporter(len, <ype)); - } + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); + }; + let validity = validity.to_array(len).execute::(ctx)?; let children = fields .iter() diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index 8067b08a856..e17f63e2e2a 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -9,12 +9,12 @@ use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; use vortex::mask::Mask; -use crate::duckdb::LogicalType; use crate::duckdb::VectorBuffer; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; @@ -34,15 +34,15 @@ pub(crate) fn new_exporter( let len = array.len(); let VarBinViewDataParts { validity, - dtype, + dtype: _dtype, views, buffers, } = array.into_data_parts(); - let validity = validity.to_array(len).execute::(ctx)?; - if validity.all_false() { - let ltype = LogicalType::try_from(dtype)?; - return Ok(all_invalid::new_exporter(len, <ype)); + + if matches!(validity, Validity::AllInvalid) { + return Ok(all_invalid::new_exporter()); } + let validity = validity.to_array(len).execute::(ctx)?; let buffers: Vec<_> = buffers.iter().cloned().map(|b| b.unwrap_host()).collect(); let buffers: Arc<[ByteBuffer]> = Arc::from(buffers); diff --git a/vortex-duckdb/src/exporter/vector.rs b/vortex-duckdb/src/exporter/vector.rs index c8b9e36f2eb..e6ada5c7dd7 100644 --- a/vortex-duckdb/src/exporter/vector.rs +++ b/vortex-duckdb/src/exporter/vector.rs @@ -48,11 +48,11 @@ impl VectorRef { } } - fn set_all_true_validity(&mut self) { + pub fn set_all_true_validity(&mut self) { unsafe { duckdb_vx_vector_set_all_valid(self.as_ptr()) }; } - fn set_all_false_validity(&mut self) { + pub fn set_all_false_validity(&mut self) { self.reference_value(&Value::null(&self.logical_type())); } } From 8dc8c96bca94aff9bd1f18c301a3ed4a7c9615b4 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Mon, 13 Apr 2026 17:01:28 +0100 Subject: [PATCH 024/250] perf: `Arc::ptr_eq` short-circuit for List & FixedSizeList DType eq (#7410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the `Arc::ptr_eq` fast-path from #7398 to cover the remaining Arc-containing DType variants. List and FixedSizeList hold a bare `Arc` in the enum variant, so the shortcut is applied in `DType`'s manual `PartialEq` impl. `StructFields` already handles its own `Arc::ptr_eq` internally. The mismatch arms enumerate every variant in the first position so that adding a new DType variant produces a non-exhaustive match compile error. DuckDB StatPopGen full-suite [apmc](https://github.com/0ax1/apmc) measurement for vortex, averaged over two runs: - Cycles: -5.4% (5.97B → 5.65B) - Instructions: -15.3% (15.5B → 13.1B) - L1D_CACHE_MISS_LD: -0.8% (56.2M → 55.7M) - MAP_STALL: -0.2% (1.35B → 1.34B) Signed-off-by: Alexander Droste --- vortex-array/public-api.lock | 4 +--- vortex-array/src/dtype/mod.rs | 39 ++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 317f0869dac..3edeffa8457 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -9166,7 +9166,7 @@ impl core::cmp::Eq for vortex_array::dtype::DType impl core::cmp::PartialEq for vortex_array::dtype::DType -pub fn vortex_array::dtype::DType::eq(&self, other: &vortex_array::dtype::DType) -> bool +pub fn vortex_array::dtype::DType::eq(&self, other: &Self) -> bool impl core::convert::From for vortex_array::dtype::FieldDType @@ -9216,8 +9216,6 @@ impl core::hash::Hash for vortex_array::dtype::DType pub fn vortex_array::dtype::DType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) -impl core::marker::StructuralPartialEq for vortex_array::dtype::DType - impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for vortex_array::dtype::DType pub fn vortex_array::dtype::DType::from_arrow(field: &arrow_schema::field::Field) -> Self diff --git a/vortex-array/src/dtype/mod.rs b/vortex-array/src/dtype/mod.rs index 8041faae414..62f0f1f757c 100644 --- a/vortex-array/src/dtype/mod.rs +++ b/vortex-array/src/dtype/mod.rs @@ -50,7 +50,8 @@ use std::sync::Arc; /// /// [`I32`]: PType::I32 /// [`NonNullable`]: Nullability::NonNullable -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Eq, Hash)] +#[allow(clippy::derived_hash_with_manual_eq)] // manual PartialEq adds Arc::ptr_eq fast path only pub enum DType { /// A logical null type. /// @@ -106,6 +107,42 @@ pub enum DType { Variant(Nullability), } +impl PartialEq for DType { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Null, Self::Null) => true, + (Self::Bool(a), Self::Bool(b)) => a == b, + (Self::Primitive(pa, na), Self::Primitive(pb, nb)) => pa == pb && na == nb, + (Self::Decimal(da, na), Self::Decimal(db, nb)) => da == db && na == nb, + (Self::Utf8(a), Self::Utf8(b)) => a == b, + (Self::Binary(a), Self::Binary(b)) => a == b, + (Self::List(da, na), Self::List(db, nb)) => { + na == nb && (Arc::ptr_eq(da, db) || da == db) + } + (Self::FixedSizeList(da, sa, na), Self::FixedSizeList(db, sb, nb)) => { + sa == sb && na == nb && (Arc::ptr_eq(da, db) || da == db) + } + // StructFields handles its own Arc::ptr_eq in its PartialEq impl. + (Self::Struct(a, na), Self::Struct(b, nb)) => na == nb && a == b, + (Self::Extension(a), Self::Extension(b)) => a == b, + (Self::Variant(a), Self::Variant(b)) => a == b, + // Every variant is listed in the first position so that adding a new + // variant produces a non-exhaustive match compile error. + (Self::Null, _) + | (Self::Bool(_), _) + | (Self::Primitive(..), _) + | (Self::Decimal(..), _) + | (Self::Utf8(_), _) + | (Self::Binary(_), _) + | (Self::List(..), _) + | (Self::FixedSizeList(..), _) + | (Self::Struct(..), _) + | (Self::Extension(_), _) + | (Self::Variant(_), _) => false, + } + } +} + pub use bigint::*; pub use decimal::*; pub use dtype_impl::NativeDType; From 71089dd114663c9aa6e04861ad27d932cbf96c44 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Date: Mon, 13 Apr 2026 09:47:40 -0700 Subject: [PATCH 025/250] feat: implement IsNotNull expression in vortex expression library (#6969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Closes: #6040 Add a first-class IsNotNull scalar function, replacing the previous Not(IsNull(...)) composition pattern. This simplifies the expression tree and enables direct stat_falsification for zone map pruning. Changes: New is_not_null.rs with ScalarFnVTable implementation, including stat_falsification using is_constant && null_count > 0 (with TODO for future RowCount stat) Updated all integration points: DataFusion, DuckDB, Python/Substrait to use is_not_null(...) directly Replaced the Not(IsNull(...)) fallback in erased.rs validity with IsNotNull Registered IsNotNull in ScalarFnSession and ExprBuiltins/ArrayBuiltins ### AI Assistance Disclosure This PR was developed with AI assistance (Kiro). AI was used for code review, implementing stat_falsification, writing tests, and drafting the PR description. All output was reviewed and validated by the author. API Changes New public APIs: vortex_array::expr::is_not_null(child) — creates an IsNotNull expression Expression::is_not_null() / ArrayRef::is_not_null() via ExprBuiltins/ArrayBuiltins traits Python: vortex._lib.expr.is_not_null(child) ### Testing 9 unit tests covering: return dtype, child replacement, mixed/all-valid/all-invalid evaluation, struct field access, display formatting, null sensitivity, and stat falsification pruning expression generation. --------- Signed-off-by: Xiaoxuan Li Signed-off-by: Robert Kruszewski Co-authored-by: Robert Kruszewski --- .../main/java/dev/vortex/api/Expression.java | 8 + .../dev/vortex/api/expressions/IsNotNull.java | 99 ++++++ .../dev/vortex/api/proto/Expressions.java | 5 +- .../proto/TestExpressionProtos.java | 8 + vortex-array/public-api.lock | 98 ++++++ vortex-array/src/builtins.rs | 17 + vortex-array/src/expr/exprs.rs | 15 + vortex-array/src/scalar_fn/erased.rs | 9 +- vortex-array/src/scalar_fn/fns/is_not_null.rs | 294 ++++++++++++++++++ vortex-array/src/scalar_fn/fns/mod.rs | 1 + vortex-array/src/scalar_fn/session.rs | 2 + vortex-datafusion/src/convert/exprs.rs | 3 +- vortex-duckdb/src/convert/expr.rs | 3 +- vortex-duckdb/src/convert/table_filter.rs | 4 +- vortex-layout/src/layouts/dict/reader.rs | 5 +- vortex-python/python/vortex/_lib/expr.pyi | 1 + vortex-python/python/vortex/substrait.py | 9 +- vortex-python/src/expr/mod.rs | 17 + 18 files changed, 575 insertions(+), 23 deletions(-) create mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java create mode 100644 vortex-array/src/scalar_fn/fns/is_not_null.rs diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java index 8dc4ec881b4..e2fca7be023 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java @@ -95,6 +95,14 @@ interface Visitor { */ T visitIsNull(IsNull isNull); + /** + * Visits an is not null expression (non-null check). + * + * @param isNotNull the is not null expression to visit + * @return the result of visiting the is not null expression + */ + T visitIsNotNull(IsNotNull isNotNull); + /** * For expressions that do not have a specific visitor method. */ diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java new file mode 100644 index 00000000000..05c9e03f5d9 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.api.expressions; + +import dev.vortex.api.Expression; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Represents an IS NOT NULL expression that checks whether values are non-null. + * This expression returns true for non-null values and false for null values. + */ +public final class IsNotNull implements Expression { + private final Expression child; + + private IsNotNull(Expression child) { + this.child = child; + } + + /** + * Parses an IsNotNull expression from serialized metadata and child expressions. + * This method is used during deserialization of Vortex expressions. + * + * @param metadata the serialized metadata, must be empty for IsNotNull expressions + * @param children the child expressions, must contain exactly one element + * @return a new IsNotNull expression parsed from the provided data + * @throws IllegalArgumentException if the number of children is not exactly one, + * or if metadata is not empty + */ + public static IsNotNull parse(byte[] metadata, List children) { + if (children.size() != 1) { + throw new IllegalArgumentException( + "IsNotNull expression must have exactly one child, found: " + children.size()); + } + if (metadata.length > 0) { + throw new IllegalArgumentException( + "IsNotNull expression must not have metadata, found: " + metadata.length); + } + return new IsNotNull(children.get(0)); + } + + /** + * Creates a new IsNotNull expression that checks non-nullity of the given child expression. + * + * @param child the expression to check for non-null values + * @return a new IsNotNull expression + */ + public static IsNotNull of(Expression child) { + return new IsNotNull(child); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + IsNotNull other = (IsNotNull) o; + return Objects.equals(child, other.child); + } + + @Override + public int hashCode() { + return Objects.hash(child); + } + + @Override + public String id() { + return "vortex.is_not_null"; + } + + @Override + public List children() { + return List.of(child); + } + + @Override + public Optional metadata() { + return Optional.of(new byte[] {}); + } + + @Override + public String toString() { + return "vortex.is_not_null(" + child + ")"; + } + + /** + * Returns the child expression that will be checked for non-null values. + * + * @return the child expression + */ + public Expression getChild() { + return child; + } + + @Override + public T accept(Visitor visitor) { + return visitor.visitIsNotNull(this); + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java index dc2464851f2..2e4d80691c6 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java @@ -34,7 +34,8 @@ public static ExprProtos.Expr serialize(Expression expression) { /** * Deserialize a protocol buffer representation back into an {@link Expression} object. * The method examines the expression ID and creates the appropriate concrete expression type - * based on the registered expression types (binary, get_item, root, literal, not). + * based on the registered expression types (binary, get_item, root, literal, not, is null, + * is not null). * If the expression ID is not recognized, an {@link Unknown} expression is created. * * @param expr the protocol buffer expression to deserialize @@ -58,6 +59,8 @@ public static Expression deserialize(ExprProtos.Expr expr) { return Not.parse(metadata, children); case "vortex.is_null": return IsNull.parse(metadata, children); + case "vortex.is_not_null": + return IsNotNull.parse(metadata, children); default: return new Unknown(expr.getId(), children, expr.getMetadata().toByteArray()); } diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java b/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java index f3e9fffd081..ed133bf4060 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java @@ -30,4 +30,12 @@ public void testIsNullRoundTrip() { Expression deserialized = Expressions.deserialize(proto); assertEquals(expression, deserialized); } + + @Test + public void testIsNotNullRoundTrip() { + Expression expression = IsNotNull.of(GetItem.of(Root.INSTANCE, "a.b.c")); + ExprProtos.Expr proto = Expressions.serialize(expression); + Expression deserialized = Expressions.deserialize(proto); + assertEquals(expression, deserialized); + } } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 3edeffa8457..629dcd50d4f 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -8440,6 +8440,8 @@ pub fn vortex_array::builtins::ArrayBuiltins::fill_null(&self, fill_value: impl pub fn vortex_array::builtins::ArrayBuiltins::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::builtins::ArrayBuiltins::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::builtins::ArrayBuiltins::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::builtins::ArrayBuiltins::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult @@ -8462,6 +8464,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert:: pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult @@ -8482,6 +8486,8 @@ pub fn vortex_array::builtins::ExprBuiltins::fill_null(&self, fill_value: vortex pub fn vortex_array::builtins::ExprBuiltins::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::builtins::ExprBuiltins::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::builtins::ExprBuiltins::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::builtins::ExprBuiltins::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult @@ -8502,6 +8508,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult @@ -12426,6 +12434,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult @@ -12520,6 +12530,8 @@ pub fn vortex_array::expr::immediate_scope_access<'a>(expr: &'a vortex_array::ex pub fn vortex_array::expr::immediate_scope_accesses<'a>(expr: &'a vortex_array::expr::Expression, scope: &'a vortex_array::dtype::StructFields) -> vortex_array::expr::FieldAccesses<'a> +pub fn vortex_array::expr::is_not_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression + pub fn vortex_array::expr::is_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression pub fn vortex_array::expr::is_root(expr: &vortex_array::expr::Expression) -> bool @@ -16302,6 +16314,52 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> +pub mod vortex_array::scalar_fn::fns::is_not_null + +pub struct vortex_array::scalar_fn::fns::is_not_null::IsNotNull + +impl core::clone::Clone for vortex_array::scalar_fn::fns::is_not_null::IsNotNull + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::clone(&self) -> vortex_array::scalar_fn::fns::is_not_null::IsNotNull + +impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull + +pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult>> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> + pub mod vortex_array::scalar_fn::fns::is_null pub struct vortex_array::scalar_fn::fns::is_null::IsNull @@ -18084,6 +18142,44 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> +impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull + +pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult>> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult> + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option + +pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> + impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_null::IsNull pub type vortex_array::scalar_fn::fns::is_null::IsNull::Options = vortex_array::scalar_fn::EmptyOptions @@ -22410,6 +22506,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert:: pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult + pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult diff --git a/vortex-array/src/builtins.rs b/vortex-array/src/builtins.rs index b4a3eb23984..dcbe934097e 100644 --- a/vortex-array/src/builtins.rs +++ b/vortex-array/src/builtins.rs @@ -28,6 +28,7 @@ use crate::scalar_fn::fns::binary::Binary; use crate::scalar_fn::fns::cast::Cast; use crate::scalar_fn::fns::fill_null::FillNull; use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::fns::is_not_null::IsNotNull; use crate::scalar_fn::fns::is_null::IsNull; use crate::scalar_fn::fns::list_contains::ListContains; use crate::scalar_fn::fns::mask::Mask; @@ -49,6 +50,9 @@ pub trait ExprBuiltins: Sized { /// Is null check. fn is_null(&self) -> VortexResult; + /// Is not null check. + fn is_not_null(&self) -> VortexResult; + /// Mask the expression using the given boolean mask. /// The resulting expression's validity is the intersection of the original expression's /// validity. @@ -84,6 +88,10 @@ impl ExprBuiltins for Expression { IsNull.try_new_expr(EmptyOptions, [self.clone()]) } + fn is_not_null(&self) -> VortexResult { + IsNotNull.try_new_expr(EmptyOptions, [self.clone()]) + } + fn mask(&self, mask: Expression) -> VortexResult { Mask.try_new_expr(EmptyOptions, [self.clone(), mask]) } @@ -118,6 +126,9 @@ pub trait ArrayBuiltins: Sized { /// Is null check. fn is_null(&self) -> VortexResult; + /// Is not null check. + fn is_not_null(&self) -> VortexResult; + /// Mask the array using the given boolean mask. /// The resulting array's validity is the intersection of the original array's validity /// and the mask's validity. @@ -182,6 +193,12 @@ impl ArrayBuiltins for ArrayRef { .optimize() } + fn is_not_null(&self) -> VortexResult { + IsNotNull + .try_new_array(self.len(), EmptyOptions, [self.clone()])? + .optimize() + } + fn mask(self, mask: ArrayRef) -> VortexResult { Mask.try_new_array(self.len(), EmptyOptions, [self, mask])? .optimize() diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs index cbf34cdb3c2..fb8d2f0cb77 100644 --- a/vortex-array/src/expr/exprs.rs +++ b/vortex-array/src/expr/exprs.rs @@ -29,6 +29,7 @@ use crate::scalar_fn::fns::dynamic::DynamicComparisonExpr; use crate::scalar_fn::fns::dynamic::Rhs; use crate::scalar_fn::fns::fill_null::FillNull; use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::fns::is_not_null::IsNotNull; use crate::scalar_fn::fns::is_null::IsNull; use crate::scalar_fn::fns::like::Like; use crate::scalar_fn::fns::like::LikeOptions; @@ -555,6 +556,20 @@ pub fn is_null(child: Expression) -> Expression { IsNull.new_expr(EmptyOptions, vec![child]) } +// ---- IsNotNull ---- + +/// Creates an expression that checks for non-null values. +/// +/// Returns a boolean array indicating which positions contain non-null values. +/// +/// ```rust +/// # use vortex_array::expr::{is_not_null, root}; +/// let expr = is_not_null(root()); +/// ``` +pub fn is_not_null(child: Expression) -> Expression { + IsNotNull.new_expr(EmptyOptions, vec![child]) +} + // ---- Like ---- /// Creates a SQL LIKE expression. diff --git a/vortex-array/src/scalar_fn/erased.rs b/vortex-array/src/scalar_fn/erased.rs index 154ad13bace..369627964e2 100644 --- a/vortex-array/src/scalar_fn/erased.rs +++ b/vortex-array/src/scalar_fn/erased.rs @@ -31,8 +31,7 @@ use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::SimplifyCtx; -use crate::scalar_fn::fns::is_null::IsNull; -use crate::scalar_fn::fns::not::Not; +use crate::scalar_fn::fns::is_not_null::IsNotNull; use crate::scalar_fn::options::ScalarFnOptions; use crate::scalar_fn::signature::ScalarFnSignature; use crate::scalar_fn::typed::DynScalarFn; @@ -135,11 +134,7 @@ impl ScalarFnRef { pub fn validity(&self, expr: &Expression) -> VortexResult { Ok(self.0.validity(expr)?.unwrap_or_else(|| { // TODO(ngates): make validity a mandatory method on VTable to avoid this fallback. - // TODO(ngates): add an IsNotNull expression. - Not.new_expr( - EmptyOptions, - [IsNull.new_expr(EmptyOptions, [expr.clone()])], - ) + IsNotNull.new_expr(EmptyOptions, [expr.clone()]) })) } diff --git a/vortex-array/src/scalar_fn/fns/is_not_null.rs b/vortex-array/src/scalar_fn/fns/is_not_null.rs new file mode 100644 index 00000000000..fd0dcc53fc6 --- /dev/null +++ b/vortex-array/src/scalar_fn/fns/is_not_null.rs @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Formatter; + +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +use crate::ArrayRef; +use crate::ExecutionCtx; +use crate::IntoArray; +use crate::arrays::ConstantArray; +use crate::dtype::DType; +use crate::dtype::Nullability; +use crate::expr::Expression; +use crate::expr::StatsCatalog; +use crate::expr::and; +use crate::expr::eq; +use crate::expr::gt; +use crate::expr::lit; +use crate::expr::stats::Stat; +use crate::scalar_fn::Arity; +use crate::scalar_fn::ChildName; +use crate::scalar_fn::EmptyOptions; +use crate::scalar_fn::ExecutionArgs; +use crate::scalar_fn::ScalarFnId; +use crate::scalar_fn::ScalarFnVTable; +use crate::validity::Validity; + +/// Expression that checks for non-null values. +#[derive(Clone)] +pub struct IsNotNull; + +impl ScalarFnVTable for IsNotNull { + type Options = EmptyOptions; + + fn id(&self) -> ScalarFnId { + ScalarFnId::new_ref("vortex.is_not_null") + } + + fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { + Ok(Some(vec![])) + } + + fn deserialize( + &self, + _metadata: &[u8], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyOptions) + } + + fn arity(&self, _options: &Self::Options) -> Arity { + Arity::Exact(1) + } + + fn child_name(&self, _instance: &Self::Options, child_idx: usize) -> ChildName { + match child_idx { + 0 => ChildName::from("input"), + _ => unreachable!("Invalid child index {} for IsNotNull expression", child_idx), + } + } + + fn fmt_sql( + &self, + _options: &Self::Options, + expr: &Expression, + f: &mut Formatter<'_>, + ) -> std::fmt::Result { + write!(f, "is_not_null(")?; + expr.child(0).fmt_sql(f)?; + write!(f, ")") + } + + fn return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[DType]) -> VortexResult { + Ok(DType::Bool(Nullability::NonNullable)) + } + + fn execute( + &self, + _data: &Self::Options, + args: &dyn ExecutionArgs, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + let child = args.get(0)?; + match child.validity()? { + Validity::NonNullable | Validity::AllValid => { + Ok(ConstantArray::new(true, args.row_count()).into_array()) + } + Validity::AllInvalid => Ok(ConstantArray::new(false, args.row_count()).into_array()), + Validity::Array(a) => Ok(a), + } + } + + fn is_null_sensitive(&self, _instance: &Self::Options) -> bool { + true + } + + fn is_fallible(&self, _instance: &Self::Options) -> bool { + false + } + + fn stat_falsification( + &self, + _options: &Self::Options, + expr: &Expression, + catalog: &dyn StatsCatalog, + ) -> Option { + // is_not_null is falsified when ALL values are null, i.e. null_count == len. + // Since there is no len stat in the zone map, we approximate using IsConstant: + // if the zone is constant and has any nulls, then all values must be null. + // + // TODO(#7187): Add a len stat to enable the more general falsification: + // null_count == len => is_not_null is all false. + let null_count_expr = expr.child(0).stat_expression(Stat::NullCount, catalog)?; + let is_constant_expr = expr.child(0).stat_expression(Stat::IsConstant, catalog)?; + // If the zone is constant (is_constant == true) and has nulls (null_count > 0), + // then all values must be null, so is_not_null is all false. + Some(and( + eq(is_constant_expr, lit(true)), + gt(null_count_expr, lit(0u64)), + )) + } +} + +#[cfg(test)] +mod tests { + use vortex_buffer::buffer; + use vortex_error::VortexExpect as _; + + use crate::IntoArray; + use crate::arrays::PrimitiveArray; + use crate::arrays::StructArray; + use crate::dtype::DType; + use crate::dtype::Nullability; + use crate::expr::get_item; + use crate::expr::is_not_null; + use crate::expr::root; + use crate::expr::test_harness; + use crate::scalar::Scalar; + + #[test] + fn dtype() { + let dtype = test_harness::struct_dtype(); + assert_eq!( + is_not_null(root()).return_dtype(&dtype).unwrap(), + DType::Bool(Nullability::NonNullable) + ); + } + + #[test] + fn replace_children() { + let expr = is_not_null(root()); + expr.with_children([root()]) + .vortex_expect("operation should succeed in test"); + } + + #[test] + fn evaluate_mask() { + let test_array = + PrimitiveArray::from_option_iter(vec![Some(1), None, Some(2), None, Some(3)]) + .into_array(); + let expected = [true, false, true, false, true]; + + let result = test_array.clone().apply(&is_not_null(root())).unwrap(); + + assert_eq!(result.len(), test_array.len()); + assert_eq!(result.dtype(), &DType::Bool(Nullability::NonNullable)); + + for (i, expected_value) in expected.iter().enumerate() { + assert_eq!( + result.scalar_at(i).unwrap(), + Scalar::bool(*expected_value, Nullability::NonNullable) + ); + } + } + + #[test] + fn evaluate_all_true() { + let test_array = buffer![1, 2, 3, 4, 5].into_array(); + + let result = test_array.clone().apply(&is_not_null(root())).unwrap(); + + assert_eq!(result.len(), test_array.len()); + for i in 0..result.len() { + assert_eq!( + result.scalar_at(i).unwrap(), + Scalar::bool(true, Nullability::NonNullable) + ); + } + } + + #[test] + fn evaluate_all_false() { + let test_array = + PrimitiveArray::from_option_iter(vec![None::, None, None, None, None]) + .into_array(); + + let result = test_array.clone().apply(&is_not_null(root())).unwrap(); + + assert_eq!(result.len(), test_array.len()); + for i in 0..result.len() { + assert_eq!( + result.scalar_at(i).unwrap(), + Scalar::bool(false, Nullability::NonNullable) + ); + } + } + + #[test] + fn evaluate_struct() { + let test_array = StructArray::from_fields(&[( + "a", + PrimitiveArray::from_option_iter(vec![Some(1), None, Some(2), None, Some(3)]) + .into_array(), + )]) + .unwrap() + .into_array(); + let expected = [true, false, true, false, true]; + + let result = test_array + .clone() + .apply(&is_not_null(get_item("a", root()))) + .unwrap(); + + assert_eq!(result.len(), test_array.len()); + assert_eq!(result.dtype(), &DType::Bool(Nullability::NonNullable)); + + for (i, expected_value) in expected.iter().enumerate() { + assert_eq!( + result.scalar_at(i).unwrap(), + Scalar::bool(*expected_value, Nullability::NonNullable) + ); + } + } + + #[test] + fn test_display() { + let expr = is_not_null(get_item("name", root())); + assert_eq!(expr.to_string(), "is_not_null($.name)"); + + let expr2 = is_not_null(root()); + assert_eq!(expr2.to_string(), "is_not_null($)"); + } + + #[test] + fn test_is_not_null_sensitive() { + use crate::expr::col; + assert!(is_not_null(col("a")).signature().is_null_sensitive()); + } + + #[test] + fn test_is_not_null_falsification() { + use vortex_utils::aliases::hash_map::HashMap; + use vortex_utils::aliases::hash_set::HashSet; + + use crate::dtype::Field; + use crate::dtype::FieldPath; + use crate::dtype::FieldPathSet; + use crate::expr::and; + use crate::expr::col; + use crate::expr::eq; + use crate::expr::gt; + use crate::expr::lit; + use crate::expr::pruning::checked_pruning_expr; + use crate::expr::stats::Stat; + + let expr = is_not_null(col("a")); + + let (pruning_expr, st) = checked_pruning_expr( + &expr, + &FieldPathSet::from_iter([ + FieldPath::from_iter([Field::Name("a".into()), Field::Name("null_count".into())]), + FieldPath::from_iter([Field::Name("a".into()), Field::Name("is_constant".into())]), + ]), + ) + .unwrap(); + + assert_eq!( + &pruning_expr, + &and( + eq(col("a_is_constant"), lit(true)), + gt(col("a_null_count"), lit(0u64)), + ) + ); + assert_eq!( + st.map(), + &HashMap::from_iter([( + FieldPath::from_name("a"), + HashSet::from([Stat::NullCount, Stat::IsConstant]) + )]) + ); + } +} diff --git a/vortex-array/src/scalar_fn/fns/mod.rs b/vortex-array/src/scalar_fn/fns/mod.rs index 94fc8fb0384..8fa1b66532d 100644 --- a/vortex-array/src/scalar_fn/fns/mod.rs +++ b/vortex-array/src/scalar_fn/fns/mod.rs @@ -8,6 +8,7 @@ pub mod cast; pub mod dynamic; pub mod fill_null; pub mod get_item; +pub mod is_not_null; pub mod is_null; pub mod like; pub mod list_contains; diff --git a/vortex-array/src/scalar_fn/session.rs b/vortex-array/src/scalar_fn/session.rs index eef759bf8e3..3c78f56928d 100644 --- a/vortex-array/src/scalar_fn/session.rs +++ b/vortex-array/src/scalar_fn/session.rs @@ -14,6 +14,7 @@ use crate::scalar_fn::fns::binary::Binary; use crate::scalar_fn::fns::cast::Cast; use crate::scalar_fn::fns::fill_null::FillNull; use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::fns::is_not_null::IsNotNull; use crate::scalar_fn::fns::is_null::IsNull; use crate::scalar_fn::fns::like::Like; use crate::scalar_fn::fns::list_contains::ListContains; @@ -58,6 +59,7 @@ impl Default for ScalarFnSession { this.register(Cast); this.register(FillNull); this.register(GetItem); + this.register(IsNotNull); this.register(IsNull); this.register(Like); this.register(ListContains); diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index 654b82f2cab..6a9ac0fdc52 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -26,6 +26,7 @@ use vortex::expr::Expression; use vortex::expr::and_collect; use vortex::expr::cast; use vortex::expr::get_item; +use vortex::expr::is_not_null; use vortex::expr::is_null; use vortex::expr::list_contains; use vortex::expr::lit; @@ -244,7 +245,7 @@ impl ExpressionConvertor for DefaultExpressionConvertor { if let Some(is_not_null_expr) = df.as_any().downcast_ref::() { let arg = self.convert(is_not_null_expr.arg().as_ref())?; - return Ok(not(is_null(arg))); + return Ok(is_not_null(arg)); } if let Some(in_list) = df.as_any().downcast_ref::() { diff --git a/vortex-duckdb/src/convert/expr.rs b/vortex-duckdb/src/convert/expr.rs index 471b9b41302..3cbfec883b6 100644 --- a/vortex-duckdb/src/convert/expr.rs +++ b/vortex-duckdb/src/convert/expr.rs @@ -13,6 +13,7 @@ use vortex::error::vortex_err; use vortex::expr::Expression; use vortex::expr::and_collect; use vortex::expr::col; +use vortex::expr::is_not_null; use vortex::expr::is_null; use vortex::expr::list_contains; use vortex::expr::lit; @@ -104,7 +105,7 @@ pub fn try_from_bound_expression( DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_OPERATOR_NOT => not(child), DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_OPERATOR_IS_NULL => is_null(child), DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_OPERATOR_IS_NOT_NULL => { - not(is_null(child)) + is_not_null(child) } _ => unreachable!(), } diff --git a/vortex-duckdb/src/convert/table_filter.rs b/vortex-duckdb/src/convert/table_filter.rs index 7741fd64d0a..fa9003bc09c 100644 --- a/vortex-duckdb/src/convert/table_filter.rs +++ b/vortex-duckdb/src/convert/table_filter.rs @@ -12,10 +12,10 @@ use vortex::error::vortex_bail; use vortex::expr::Expression; use vortex::expr::and_collect; use vortex::expr::get_item; +use vortex::expr::is_not_null; use vortex::expr::is_null; use vortex::expr::list_contains; use vortex::expr::lit; -use vortex::expr::not; use vortex::expr::or_collect; use vortex::scalar::Scalar; use vortex::scalar_fn::ScalarFnVTableExt; @@ -61,7 +61,7 @@ pub fn try_from_table_filter( or_collect(children).unwrap_or_else(|| lit(false)) } TableFilterClass::IsNull => is_null(col.clone()), - TableFilterClass::IsNotNull => not(is_null(col.clone())), + TableFilterClass::IsNotNull => is_not_null(col.clone()), TableFilterClass::StructExtract(name, child_filter) => { return try_from_table_filter(child_filter, &get_item(name, col.clone()), scope_dtype); } diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 486bbab14c2..46d308fcdb8 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -270,9 +270,8 @@ mod tests { use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::expr::eq; - use vortex_array::expr::is_null; + use vortex_array::expr::is_not_null; use vortex_array::expr::lit; - use vortex_array::expr::not; use vortex_array::expr::pack; use vortex_array::expr::root; use vortex_array::scalar_fn::session::ScalarFnSession; @@ -504,7 +503,7 @@ mod tests { .await .unwrap(); - let expression = not(is_null(root())); // easier to test not_is_null b/c that's the validity array + let expression = is_not_null(root()); assert_eq!(layout.encoding_id(), LayoutId::new_ref("vortex.dict")); let actual = layout .new_reader("".into(), segments, &session) diff --git a/vortex-python/python/vortex/_lib/expr.pyi b/vortex-python/python/vortex/_lib/expr.pyi index c1777095743..c69307266de 100644 --- a/vortex-python/python/vortex/_lib/expr.pyi +++ b/vortex-python/python/vortex/_lib/expr.pyi @@ -35,3 +35,4 @@ def not_(child: Expr) -> Expr: ... def and_(left: Expr, right: Expr) -> Expr: ... def cast(child: Expr, dtype: DType) -> Expr: ... def is_null(child: Expr) -> Expr: ... +def is_not_null(child: Expr) -> Expr: ... diff --git a/vortex-python/python/vortex/substrait.py b/vortex-python/python/vortex/substrait.py index 28a144f62ef..1bab4b87d2b 100644 --- a/vortex-python/python/vortex/substrait.py +++ b/vortex-python/python/vortex/substrait.py @@ -498,7 +498,7 @@ def extension_function( case "is_null": return _expr.is_null case "is_not_null": - return _is_not_null + return _expr.is_not_null case name: raise NotImplementedError(f"Function name {name} not supported") case "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml": @@ -517,13 +517,6 @@ def extension_function( raise NotImplementedError(f"Extension URI {uri} not supported") -def _is_not_null(e: _expr.Expr) -> _expr.Expr: - """ - Helper function to have a well-typed callable to return - """ - return _expr.not_(_expr.is_null(e)) - - def expression( substrait_object: Expression, functions: list[Callable[..., _expr.Expr]], diff --git a/vortex-python/src/expr/mod.rs b/vortex-python/src/expr/mod.rs index 2daaefe4f4d..17b342d7db5 100644 --- a/vortex-python/src/expr/mod.rs +++ b/vortex-python/src/expr/mod.rs @@ -35,6 +35,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { m.add_function(wrap_pyfunction!(and_, &m)?)?; m.add_function(wrap_pyfunction!(cast, &m)?)?; m.add_function(wrap_pyfunction!(is_null, &m)?)?; + m.add_function(wrap_pyfunction!(is_not_null, &m)?)?; m.add_class::()?; Ok(()) @@ -422,3 +423,19 @@ pub fn is_null(child: PyExpr) -> PyResult { inner: expr::is_null(child.into_inner()), }) } + +/// Creates an expression that checks for non-null values. +/// +/// Parameters +/// ---------- +/// child : :class:`vortex.Expr` +/// +/// Returns +/// ------- +/// :class:`vortex.Expr` +#[pyfunction] +pub fn is_not_null(child: PyExpr) -> PyResult { + Ok(PyExpr { + inner: expr::is_not_null(child.into_inner()), + }) +} From fe388a0cdec3f62eb0c1cb61bb7a4bae4903b613 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 13 Apr 2026 15:57:30 -0400 Subject: [PATCH 026/250] More robust types in the compressor (#7415) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 Makes the compressor types more robust (removes the possibility for invalid state), which additionally sets up adding tracing easier (draft at https://github.com/vortex-data/vortex/pull/7385) ## API Changes Changes some types: ```rust /// Closure type for [`DeferredEstimate::Callback`]. /// /// The compressor calls this with the same arguments it would pass to sampling. The closure must /// resolve directly to a terminal [`EstimateVerdict`]. #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &mut ArrayAndStats, CompressorContext, ) -> VortexResult + Send + Sync; /// The result of a [`Scheme`]'s compression ratio estimation. /// /// This type is returned by [`Scheme::expected_compression_ratio`] to tell the compressor how /// promising this scheme is for a given array without performing any expensive work. /// /// [`CompressionEstimate::Verdict`] means the scheme already knows the terminal answer. /// [`CompressionEstimate::Deferred`] means the compressor must do extra work before the scheme can /// produce a terminal answer. #[derive(Debug)] pub enum CompressionEstimate { /// The scheme already knows the terminal estimation verdict. Verdict(EstimateVerdict), /// The compressor must perform deferred work to resolve the terminal estimation verdict. Deferred(DeferredEstimate), } /// The terminal answer to a compression estimate request. #[derive(Debug)] pub enum EstimateVerdict { /// Do not use this scheme for this array. Skip, /// Always use this scheme, as it is definitively the best choice. /// /// Some examples include constant detection, decimal byte parts, and temporal decomposition. /// /// The compressor will select this scheme immediately without evaluating further candidates. /// Schemes that return `AlwaysUse` must be mutually exclusive per canonical type (enforced by /// [`Scheme::matches`]), otherwise the winner depends silently on registration order. /// /// [`Scheme::matches`]: crate::scheme::Scheme::matches AlwaysUse, /// The estimated compression ratio. This must be greater than `1.0` to be considered by the /// compressor, otherwise it is worse than the canonical encoding. Ratio(f64), } /// Deferred work that can resolve to a terminal [`EstimateVerdict`]. pub enum DeferredEstimate { /// The scheme cannot cheaply estimate its ratio, so the compressor should compress a small /// sample to determine effectiveness. Sample, /// A fallible estimation requiring a custom expensive computation. /// /// Use this only when the scheme needs to perform trial encoding or other costly checks to /// determine its compression ratio. The callback returns an [`EstimateVerdict`] directly, so /// it cannot request more sampling or another deferred callback. Callback(Box), } ``` This will make some changes that we want to make is the future easier as well (tracing, better decision making for what things to try, etc). ## Testing Some new tests Signed-off-by: Connor Tsui --- vortex-btrblocks/src/schemes/decimal.rs | 3 +- vortex-btrblocks/src/schemes/float.rs | 26 +- vortex-btrblocks/src/schemes/integer.rs | 80 +++-- vortex-btrblocks/src/schemes/string.rs | 14 +- vortex-btrblocks/src/schemes/temporal.rs | 3 +- vortex-compressor/public-api.lock | 32 +- .../src/builtins/constant/bool.rs | 9 +- .../src/builtins/constant/float.rs | 26 +- .../src/builtins/constant/integer.rs | 13 +- .../src/builtins/constant/string.rs | 24 +- vortex-compressor/src/builtins/dict/float.rs | 8 +- .../src/builtins/dict/integer.rs | 9 +- vortex-compressor/src/builtins/dict/string.rs | 8 +- vortex-compressor/src/compressor.rs | 332 ++++++++++++++++-- vortex-compressor/src/estimate.rs | 56 +-- vortex-compressor/src/scheme.rs | 21 +- .../src/encodings/turboquant/scheme.rs | 5 +- 17 files changed, 494 insertions(+), 175 deletions(-) diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index ce5262ba5ce..3b9def06633 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::decimal::narrowed_decimal; use vortex_array::dtype::DecimalType; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_decimal_byte_parts::DecimalByteParts; use vortex_error::VortexResult; @@ -47,7 +48,7 @@ impl Scheme for DecimalScheme { _ctx: CompressorContext, ) -> CompressionEstimate { // Decimal compression is almost always beneficial (narrowing + primitive compression). - CompressionEstimate::AlwaysUse + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } fn compress( diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 0af2b30974e..3d2557c63c7 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -21,6 +21,8 @@ use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::DeferredEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; use vortex_error::VortexResult; @@ -88,15 +90,15 @@ impl Scheme for ALPScheme { // ALP encodes floats as integers. Without integer compression afterward, the encoded ints // are the same size. if ctx.finished_cascading() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // We don't support ALP for f16. if data.array_as_primitive().ptype() == PType::F16 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -154,10 +156,10 @@ impl Scheme for ALPRDScheme { ) -> CompressionEstimate { // We don't support ALPRD for f16. if data.array_as_primitive().ptype() == PType::F16 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -225,16 +227,16 @@ impl Scheme for NullDominatedSparseScheme { // All-null arrays should be compressed as constant instead anyways. if value_count == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // If the majority (90%) of values is null, this will compress well. if stats.null_count() as f64 / len > 0.9 { - return CompressionEstimate::Ratio(len / value_count as f64); + return CompressionEstimate::Verdict(EstimateVerdict::Ratio(len / value_count as f64)); } // Otherwise we don't go this route. - CompressionEstimate::Skip + CompressionEstimate::Verdict(EstimateVerdict::Skip) } fn compress( @@ -279,7 +281,7 @@ impl Scheme for PcoScheme { _data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -326,14 +328,14 @@ impl Scheme for FloatRLEScheme { ) -> CompressionEstimate { // RLE is only useful when we cascade it with another encoding. if ctx.finished_cascading() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } if data.float_stats().average_run_length() < super::integer::RUN_LENGTH_THRESHOLD { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 100ebdfe68c..47fc52225aa 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -17,6 +17,8 @@ use vortex_array::scalar::Scalar; use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::DeferredEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; @@ -134,21 +136,21 @@ impl Scheme for FoRScheme { // FoR only subtracts the min. Without further compression (e.g. BitPacking), the output is // the same size. if ctx.finished_cascading() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let stats = data.integer_stats(); // Only apply when the min is not already zero. if stats.erased().min_is_zero() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // Difference between max and min. let for_bitwidth = match stats.erased().max_minus_min().checked_ilog2() { Some(l) => l + 1, // If max-min == 0, the we should be compressing this as a constant array. - None => return CompressionEstimate::Skip, + None => return CompressionEstimate::Verdict(EstimateVerdict::Skip), }; // If BitPacking can be applied (only non-negative values) and FoR doesn't reduce bit width @@ -162,7 +164,7 @@ impl Scheme for FoRScheme { { let bitpack_bitwidth = max_log + 1; if for_bitwidth >= bitpack_bitwidth { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } } @@ -173,7 +175,9 @@ impl Scheme for FoRScheme { .try_into() .vortex_expect("bit width must fit in u32"); - CompressionEstimate::Ratio(full_width as f64 / for_bitwidth as f64) + CompressionEstimate::Verdict(EstimateVerdict::Ratio( + full_width as f64 / for_bitwidth as f64, + )) } fn compress( @@ -265,17 +269,17 @@ impl Scheme for ZigZagScheme { // ZigZag only transforms negative values to positive. Without further compression, // the output is the same size. if ctx.finished_cascading() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let stats = data.integer_stats(); // ZigZag is only useful when there are negative values. if !stats.erased().min_is_negative() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -314,10 +318,10 @@ impl Scheme for BitPackingScheme { // BitPacking only works for non-negative values. if stats.erased().min_is_negative() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -443,12 +447,12 @@ impl Scheme for SparseScheme { // All-null arrays should be compressed as constant instead anyways. if value_count == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // If the majority (90%) of values is null, this will compress well. if stats.null_count() as f64 / len > 0.9 { - return CompressionEstimate::Ratio(len / value_count as f64); + return CompressionEstimate::Verdict(EstimateVerdict::Ratio(len / value_count as f64)); } let (_, most_frequent_count) = stats @@ -460,18 +464,20 @@ impl Scheme for SparseScheme { // If the most frequent value is the only value, we should compress as constant instead. if most_frequent_count == value_count { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } debug_assert!(value_count > most_frequent_count); // See if the most frequent value accounts for >= 90% of the set values. let freq = most_frequent_count as f64 / value_count as f64; if freq < 0.9 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // We only store the positions of the non-top values. - CompressionEstimate::Ratio(value_count as f64 / (value_count - most_frequent_count) as f64) + CompressionEstimate::Verdict(EstimateVerdict::Ratio( + value_count as f64 / (value_count - most_frequent_count) as f64, + )) } fn compress( @@ -603,10 +609,10 @@ impl Scheme for RunEndScheme { ) -> CompressionEstimate { // If the run length is below the threshold, drop it. if data.integer_stats().average_run_length() < RUN_END_THRESHOLD { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -668,14 +674,14 @@ impl Scheme for SequenceScheme { // It is pointless checking if a sample is a sequence since it will not correspond to the // entire array. if ctx.is_sample() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let stats = data.integer_stats(); // `SequenceArray` does not support nulls. if stats.null_count() > 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // If the distinct_values_count was computed, and not all values are unique, then this @@ -684,23 +690,25 @@ impl Scheme for SequenceScheme { .distinct_count() .is_some_and(|count| count as usize != data.array_len()) { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // TODO(connor): Why do we sequence encode the whole thing and then throw it away? And then // why do we divide the ratio by 2??? - CompressionEstimate::Estimate(Box::new(|_compressor, data, _ctx| { - let Some(encoded) = sequence_encode(data.array_as_primitive())? else { - // If we are unable to sequence encode this array, make sure we skip. - return Ok(CompressionEstimate::Skip); - }; - - // TODO(connor): This doesn't really make sense? - // Since two values are required to store base and multiplier the compression ratio is - // divided by 2. - Ok(CompressionEstimate::Ratio(encoded.len() as f64 / 2.0)) - })) + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, _ctx| { + let Some(encoded) = sequence_encode(data.array_as_primitive())? else { + // If we are unable to sequence encode this array, make sure we skip. + return Ok(EstimateVerdict::Skip); + }; + + // TODO(connor): This doesn't really make sense? + // Since two values are required to store base and multiplier the compression ratio is + // divided by 2. + Ok(EstimateVerdict::Ratio(encoded.len() as f64 / 2.0)) + }, + ))) } fn compress( @@ -738,10 +746,10 @@ impl Scheme for PcoScheme { // Pco does not support I8 or U8. if matches!(data.array_as_primitive().ptype(), PType::I8 | PType::U8) { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -865,14 +873,14 @@ impl Scheme for IntRLEScheme { ) -> CompressionEstimate { // RLE is only useful when we cascade it with another encoding. if ctx.finished_cascading() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } if data.integer_stats().average_run_length() < RUN_LENGTH_THRESHOLD { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 331fad9f8cb..5b21d6c1d59 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -11,6 +11,8 @@ use vortex_array::arrays::VarBinArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::DeferredEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::ChildSelection; use vortex_compressor::scheme::DescendantExclusion; use vortex_error::VortexResult; @@ -73,7 +75,7 @@ impl Scheme for FSSTScheme { _data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -161,16 +163,16 @@ impl Scheme for NullDominatedSparseScheme { // All-null arrays should be compressed as constant instead anyways. if value_count == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // If the majority (90%) of values is null, this will compress well. if stats.null_count() as f64 / len > 0.9 { - return CompressionEstimate::Ratio(len / value_count as f64); + return CompressionEstimate::Verdict(EstimateVerdict::Ratio(len / value_count as f64)); } // Otherwise we don't go this route. - CompressionEstimate::Skip + CompressionEstimate::Verdict(EstimateVerdict::Skip) } fn compress( @@ -216,7 +218,7 @@ impl Scheme for ZstdScheme { _data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( @@ -245,7 +247,7 @@ impl Scheme for ZstdBuffersScheme { _data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 306ccaf982d..47d9ae81cfb 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -15,6 +15,7 @@ use vortex_array::dtype::extension::Matcher; use vortex_array::extension::datetime::AnyTemporal; use vortex_array::extension::datetime::TemporalMetadata; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_datetime_parts::DateTimeParts; use vortex_datetime_parts::TemporalParts; use vortex_datetime_parts::split_temporal; @@ -62,7 +63,7 @@ impl Scheme for TemporalScheme { _ctx: CompressorContext, ) -> CompressionEstimate { // Temporal compression (splitting into parts) is almost always beneficial. - CompressionEstimate::AlwaysUse + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } fn compress( diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 694f3d676f0..5d9957c2ca7 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -308,21 +308,37 @@ pub mod vortex_compressor::estimate pub enum vortex_compressor::estimate::CompressionEstimate -pub vortex_compressor::estimate::CompressionEstimate::AlwaysUse +pub vortex_compressor::estimate::CompressionEstimate::Deferred(vortex_compressor::estimate::DeferredEstimate) -pub vortex_compressor::estimate::CompressionEstimate::Estimate(alloc::boxed::Box) +pub vortex_compressor::estimate::CompressionEstimate::Verdict(vortex_compressor::estimate::EstimateVerdict) -pub vortex_compressor::estimate::CompressionEstimate::Ratio(f64) +impl core::fmt::Debug for vortex_compressor::estimate::CompressionEstimate -pub vortex_compressor::estimate::CompressionEstimate::Sample +pub fn vortex_compressor::estimate::CompressionEstimate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_compressor::estimate::CompressionEstimate::Skip +pub enum vortex_compressor::estimate::DeferredEstimate -impl core::fmt::Debug for vortex_compressor::estimate::CompressionEstimate +pub vortex_compressor::estimate::DeferredEstimate::Callback(alloc::boxed::Box) -pub fn vortex_compressor::estimate::CompressionEstimate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub vortex_compressor::estimate::DeferredEstimate::Sample + +impl core::fmt::Debug for vortex_compressor::estimate::DeferredEstimate + +pub fn vortex_compressor::estimate::DeferredEstimate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +pub enum vortex_compressor::estimate::EstimateVerdict + +pub vortex_compressor::estimate::EstimateVerdict::AlwaysUse + +pub vortex_compressor::estimate::EstimateVerdict::Ratio(f64) + +pub vortex_compressor::estimate::EstimateVerdict::Skip + +impl core::fmt::Debug for vortex_compressor::estimate::EstimateVerdict + +pub fn vortex_compressor::estimate::EstimateVerdict::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &mut vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) +pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &mut vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) pub mod vortex_compressor::scheme diff --git a/vortex-compressor/src/builtins/constant/bool.rs b/vortex-compressor/src/builtins/constant/bool.rs index 62e156379e9..335bd29c2ba 100644 --- a/vortex-compressor/src/builtins/constant/bool.rs +++ b/vortex-compressor/src/builtins/constant/bool.rs @@ -12,6 +12,7 @@ use crate::builtins::BoolConstantScheme; use crate::builtins::constant::compress_constant_array_with_validity; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::Scheme; use crate::stats::ArrayAndStats; @@ -32,7 +33,7 @@ impl Scheme for BoolConstantScheme { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. if ctx.is_sample() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); @@ -41,14 +42,14 @@ impl Scheme for BoolConstantScheme { // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { debug_assert_eq!(stats.null_count() as usize, array_len); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } if stats.is_constant() { - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } - CompressionEstimate::Skip + CompressionEstimate::Verdict(EstimateVerdict::Skip) } fn compress( diff --git a/vortex-compressor/src/builtins/constant/float.rs b/vortex-compressor/src/builtins/constant/float.rs index df8ab7464b6..6cee2014334 100644 --- a/vortex-compressor/src/builtins/constant/float.rs +++ b/vortex-compressor/src/builtins/constant/float.rs @@ -14,6 +14,8 @@ use crate::builtins::FloatConstantScheme; use crate::builtins::constant::compress_constant_array_with_validity; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::Scheme; use crate::stats::ArrayAndStats; @@ -34,7 +36,7 @@ impl Scheme for FloatConstantScheme { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. if ctx.is_sample() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); @@ -43,17 +45,17 @@ impl Scheme for FloatConstantScheme { // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { if distinct_count > 1 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } else { debug_assert_eq!(distinct_count, 1); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } } // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { debug_assert_eq!(stats.null_count() as usize, array_len); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } // TODO(connor): Can we be smart here with the max and min like with integers? @@ -61,13 +63,15 @@ impl Scheme for FloatConstantScheme { // Otherwise our best bet is to actually check if the array is constant. // This is an expensive check, but in practice the distinct count is known because we often // include dictionary encoding in our set of schemes, so we rarely call this. - CompressionEstimate::Estimate(Box::new(|compressor, data, _ctx| { - if is_constant(data.array(), &mut compressor.execution_ctx())? { - Ok(CompressionEstimate::AlwaysUse) - } else { - Ok(CompressionEstimate::Skip) - } - })) + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |compressor, data, _ctx| { + if is_constant(data.array(), &mut compressor.execution_ctx())? { + Ok(EstimateVerdict::AlwaysUse) + } else { + Ok(EstimateVerdict::Skip) + } + }, + ))) } fn compress( diff --git a/vortex-compressor/src/builtins/constant/integer.rs b/vortex-compressor/src/builtins/constant/integer.rs index 0264893e5c8..1062664ca69 100644 --- a/vortex-compressor/src/builtins/constant/integer.rs +++ b/vortex-compressor/src/builtins/constant/integer.rs @@ -13,6 +13,7 @@ use crate::builtins::IntConstantScheme; use crate::builtins::constant::compress_constant_array_with_validity; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::Scheme; use crate::stats::ArrayAndStats; @@ -33,7 +34,7 @@ impl Scheme for IntConstantScheme { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. if ctx.is_sample() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); @@ -42,24 +43,24 @@ impl Scheme for IntConstantScheme { // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { if distinct_count > 1 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } else { debug_assert_eq!(distinct_count, 1); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } } // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { debug_assert_eq!(stats.null_count() as usize, array_len); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } // Otherwise, use the max and min to determine if there is a single value. match stats.erased().max_minus_min().checked_ilog2() { - Some(_) => CompressionEstimate::Skip, + Some(_) => CompressionEstimate::Verdict(EstimateVerdict::Skip), // If max-min == 0, then we know that there is only 1 value. - None => CompressionEstimate::AlwaysUse, + None => CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse), } } diff --git a/vortex-compressor/src/builtins/constant/string.rs b/vortex-compressor/src/builtins/constant/string.rs index 96e4e7ba674..694eae16872 100644 --- a/vortex-compressor/src/builtins/constant/string.rs +++ b/vortex-compressor/src/builtins/constant/string.rs @@ -14,6 +14,8 @@ use crate::builtins::StringConstantScheme; use crate::builtins::constant::compress_constant_array_with_validity; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::Scheme; use crate::stats::ArrayAndStats; @@ -34,7 +36,7 @@ impl Scheme for StringConstantScheme { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. if ctx.is_sample() { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); @@ -43,25 +45,27 @@ impl Scheme for StringConstantScheme { // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { debug_assert_eq!(stats.null_count() as usize, array_len); - return CompressionEstimate::AlwaysUse; + return CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse); } // Since the estimated distinct count is always going to be less than or equal to the actual // distinct count, if this is not equal to 1 the actual is definitely not equal to 1. if stats.estimated_distinct_count().is_some_and(|c| c > 1) { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // Otherwise our best bet is to actually check if the array is constant. // This is an expensive check, but the alternative of not compressing a constant array is // far less preferable. - CompressionEstimate::Estimate(Box::new(|compressor, data, _ctx| { - if is_constant(data.array(), &mut compressor.execution_ctx())? { - Ok(CompressionEstimate::AlwaysUse) - } else { - Ok(CompressionEstimate::Skip) - } - })) + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |compressor, data, _ctx| { + if is_constant(data.array(), &mut compressor.execution_ctx())? { + Ok(EstimateVerdict::AlwaysUse) + } else { + Ok(EstimateVerdict::Skip) + } + }, + ))) } fn compress( diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 6a188108612..18d5c216aa9 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -28,6 +28,8 @@ use crate::builtins::IntDictScheme; use crate::builtins::is_float_primitive; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; use crate::scheme::Scheme; @@ -86,7 +88,7 @@ impl Scheme for FloatDictScheme { let stats = data.float_stats(); if stats.value_count() == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let distinct_values_count = stats.distinct_count().vortex_expect( @@ -95,11 +97,11 @@ impl Scheme for FloatDictScheme { // If > 50% of the values are distinct, skip dictionary scheme. if distinct_values_count > stats.value_count() / 2 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // Let sampling determine the expected ratio. - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index d956a90990a..0693612aaee 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -26,6 +26,7 @@ use crate::builtins::IntDictScheme; use crate::builtins::is_integer_primitive; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::Scheme; use crate::scheme::SchemeExt; use crate::stats::ArrayAndStats; @@ -62,7 +63,7 @@ impl Scheme for IntDictScheme { let stats = data.integer_stats(); if stats.value_count() == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let distinct_values_count = stats.distinct_count().vortex_expect( @@ -71,7 +72,7 @@ impl Scheme for IntDictScheme { // If > 50% of the values are distinct, skip dictionary scheme. if distinct_values_count > stats.value_count() / 2 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // Ignore nulls encoding for the estimate. We only focus on values. @@ -92,7 +93,9 @@ impl Scheme for IntDictScheme { let before = stats.value_count() as usize * bit_width; - CompressionEstimate::Ratio(before as f64 / (values_size + codes_size) as f64) + CompressionEstimate::Verdict(EstimateVerdict::Ratio( + before as f64 / (values_size + codes_size) as f64, + )) } fn compress( diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index 399910a00ff..efc5511fc29 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -24,6 +24,8 @@ use crate::builtins::StringDictScheme; use crate::builtins::is_utf8_string; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateVerdict; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; use crate::scheme::Scheme; @@ -71,7 +73,7 @@ impl Scheme for StringDictScheme { let stats = data.string_stats(); if stats.value_count() == 0 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let estimated_distinct_values_count = stats.estimated_distinct_count().vortex_expect( @@ -80,11 +82,11 @@ impl Scheme for StringDictScheme { // If > 50% of the values are distinct, skip dictionary scheme. if estimated_distinct_values_count > stats.value_count() / 2 { - return CompressionEstimate::Skip; + return CompressionEstimate::Verdict(EstimateVerdict::Skip); } // Let sampling determine the expected ratio. - CompressionEstimate::Sample + CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 099cc858f92..d1d7cc28958 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -33,11 +33,12 @@ use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_panic; use crate::builtins::IntDictScheme; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; +use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateVerdict; use crate::estimate::estimate_compression_ratio_with_sampling; use crate::estimate::is_better_ratio; use crate::scheme::ChildSelection; @@ -63,6 +64,15 @@ mod root_list_children { pub const SIZES: usize = 2; } +/// The winning estimate for a scheme after all deferred work has been resolved. +#[derive(Debug, Clone, Copy, PartialEq)] +enum WinnerEstimate { + /// The scheme must be used immediately. + AlwaysUse, + /// The scheme won by numeric compression ratio. + Ratio(f64), +} + /// The main compressor type implementing cascading adaptive compression. /// /// This compressor applies adaptive compression [`Scheme`]s to arrays based on their data types and @@ -72,7 +82,7 @@ mod root_list_children { /// The compressor works by: /// 1. Canonicalizing input arrays to a standard representation. /// 2. Pre-filtering schemes by [`Scheme::matches`] and exclusion rules. -/// 3. Evaluating each matching scheme's compression ratio on a sample. +/// 3. Evaluating each matching scheme's compression estimate and resolving deferred work. /// 4. Compressing with the best scheme and verifying the result is smaller. /// /// No scheme may appear twice in a cascade chain. The compressor enforces this automatically @@ -311,13 +321,16 @@ impl CascadingCompressor { let mut data = ArrayAndStats::new(array, merged_opts); - if let Some(winner) = self.choose_best_scheme(&eligible_schemes, &mut data, ctx.clone())? { - // TODO(connor): Add a tracing warning here if compression with the chosen scheme - // failed, since there was likely more we could have done while choosing schemes. - + // TODO(connor): Add tracing support for logging the winner estimate. + if let Some((winner, _winner_estimate)) = + self.choose_best_scheme(&eligible_schemes, &mut data, ctx.clone())? + { // Sampling and estimation chose a scheme, so let's compress the whole array with it. let compressed = winner.compress(self, &mut data, ctx)?; + // TODO(connor): Add a tracing warning here if compression with the chosen scheme + // failed, since there was likely more we could have done while choosing schemes. + // Only choose the compressed array if it is smaller than the canonical one. if compressed.nbytes() < before_nbytes { // TODO(connor): Add a tracing warning here too. @@ -329,9 +342,9 @@ impl CascadingCompressor { Ok(data.into_array()) } - /// Calls [`expected_compression_ratio`] on each candidate and returns the scheme with the - /// highest ratio, or `None` if no scheme exceeds 1.0. Ties are broken by registration order - /// (earlier in the list wins). + /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme and + /// resolved winner estimate, or `None` if no scheme exceeds 1.0. Ties are broken by + /// registration order (earlier in the list wins). /// /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio fn choose_best_scheme( @@ -339,7 +352,7 @@ impl CascadingCompressor { schemes: &[&'static dyn Scheme], data: &mut ArrayAndStats, ctx: CompressorContext, - ) -> VortexResult> { + ) -> VortexResult> { let mut best: Option<(&'static dyn Scheme, f64)> = None; // TODO(connor): Might want to use an `im` data structure inside of `ctx` if the clones here @@ -347,15 +360,17 @@ impl CascadingCompressor { for &scheme in schemes { let estimate = scheme.expected_compression_ratio(data, ctx.clone()); + // TODO(connor): Rather than computing the deferred estimates eagerly, it would be + // better to look at all quick estimates and see if it makes sense to sample at all. match estimate { - CompressionEstimate::Skip => {} - CompressionEstimate::AlwaysUse => return Ok(Some(scheme)), - CompressionEstimate::Ratio(ratio) => { - if is_better_ratio(ratio, &best) { - best = Some((scheme, ratio)); + CompressionEstimate::Verdict(verdict) => { + if let Some(winner_estimate) = + Self::check_and_update_estimate_verdict(&mut best, scheme, verdict) + { + return Ok(Some((scheme, winner_estimate))); } } - CompressionEstimate::Sample => { + CompressionEstimate::Deferred(DeferredEstimate::Sample) => { let sample_ratio = estimate_compression_ratio_with_sampling( scheme, self, @@ -367,31 +382,36 @@ impl CascadingCompressor { best = Some((scheme, sample_ratio)); } } - // TODO(connor): Is there a way to deduplicate some of this code? - CompressionEstimate::Estimate(estimate_callback) => { - let estimate = estimate_callback(self, data, ctx.clone())?; - - match estimate { - CompressionEstimate::Skip => {} - CompressionEstimate::AlwaysUse => return Ok(Some(scheme)), - CompressionEstimate::Ratio(ratio) => { - if is_better_ratio(ratio, &best) { - best = Some((scheme, ratio)); - } - } - e @ (CompressionEstimate::Sample | CompressionEstimate::Estimate(_)) => { - vortex_panic!( - "an estimation function returned an invalid variant {e:?}" - ) - } + CompressionEstimate::Deferred(DeferredEstimate::Callback(estimate_callback)) => { + let verdict = estimate_callback(self, data, ctx.clone())?; + if let Some(winner_estimate) = + Self::check_and_update_estimate_verdict(&mut best, scheme, verdict) + { + return Ok(Some((scheme, winner_estimate))); } } } - - // tracing::debug!(scheme = %scheme.id(), estimate, "evaluated compression ratio"); } - Ok(best.map(|(s, _)| s)) + Ok(best.map(|(scheme, ratio)| (scheme, WinnerEstimate::Ratio(ratio)))) + } + + /// Updates `best` from a terminal estimate verdict. + fn check_and_update_estimate_verdict( + best: &mut Option<(&'static dyn Scheme, f64)>, + scheme: &'static dyn Scheme, + verdict: EstimateVerdict, + ) -> Option { + match verdict { + EstimateVerdict::Skip => None, + EstimateVerdict::AlwaysUse => Some(WinnerEstimate::AlwaysUse), + EstimateVerdict::Ratio(ratio) => { + if is_better_ratio(ratio, &*best) { + *best = Some((scheme, ratio)); + } + None + } + } } // TODO(connor): Lots of room for optimization here. @@ -502,6 +522,8 @@ impl CascadingCompressor { #[cfg(test)] mod tests { + use vortex_array::ArrayRef; + use vortex_array::Canonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::PrimitiveArray; @@ -513,12 +535,180 @@ mod tests { use crate::builtins::IntDictScheme; use crate::builtins::StringDictScheme; use crate::ctx::CompressorContext; + use crate::estimate::CompressionEstimate; + use crate::estimate::DeferredEstimate; + use crate::estimate::EstimateVerdict; use crate::scheme::SchemeExt; fn compressor() -> CascadingCompressor { CascadingCompressor::new(vec![&IntDictScheme, &FloatDictScheme, &StringDictScheme]) } + fn estimate_test_data() -> ArrayAndStats { + let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array(); + ArrayAndStats::new(array, GenerateStatsOptions::default()) + } + + fn matches_integer_primitive(canonical: &Canonical) -> bool { + matches!(canonical, Canonical::Primitive(primitive) if primitive.ptype().is_int()) + } + + #[derive(Debug)] + struct DirectRatioScheme; + + impl Scheme for DirectRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.direct_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(2.0)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct ImmediateAlwaysUseScheme; + + impl Scheme for ImmediateAlwaysUseScheme { + fn scheme_name(&self) -> &'static str { + "test.immediate_always_use" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct CallbackAlwaysUseScheme; + + impl Scheme for CallbackAlwaysUseScheme { + fn scheme_name(&self) -> &'static str { + "test.callback_always_use" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, _ctx| Ok(EstimateVerdict::AlwaysUse), + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct CallbackSkipScheme; + + impl Scheme for CallbackSkipScheme { + fn scheme_name(&self) -> &'static str { + "test.callback_skip" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, _ctx| Ok(EstimateVerdict::Skip), + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct CallbackRatioScheme; + + impl Scheme for CallbackRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.callback_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, _ctx| Ok(EstimateVerdict::Ratio(3.0)), + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + #[test] fn test_self_exclusion() { let c = compressor(); @@ -568,6 +758,76 @@ mod tests { assert!(!c.is_excluded(&IntDictScheme, &ctx)); } + #[test] + fn immediate_always_use_wins_immediately() -> VortexResult<()> { + let compressor = + CascadingCompressor::new(vec![&DirectRatioScheme, &ImmediateAlwaysUseScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &ImmediateAlwaysUseScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::AlwaysUse)) + if scheme.id() == ImmediateAlwaysUseScheme.id() + )); + Ok(()) + } + + #[test] + fn callback_always_use_wins_immediately() -> VortexResult<()> { + let compressor = + CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackAlwaysUseScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackAlwaysUseScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::AlwaysUse)) + if scheme.id() == CallbackAlwaysUseScheme.id() + )); + Ok(()) + } + + #[test] + fn callback_skip_is_ignored() -> VortexResult<()> { + let compressor = CascadingCompressor::new(vec![&CallbackSkipScheme, &DirectRatioScheme]); + let schemes: [&'static dyn Scheme; 2] = [&CallbackSkipScheme, &DirectRatioScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Ratio(2.0))) + if scheme.id() == DirectRatioScheme.id() + )); + Ok(()) + } + + #[test] + fn callback_ratio_competes_numerically() -> VortexResult<()> { + let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackRatioScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackRatioScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Ratio(3.0))) + if scheme.id() == CallbackRatioScheme.id() + )); + Ok(()) + } + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index b1e3ae3c659..957b54e57a7 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -18,35 +18,43 @@ use crate::sample::sample_count_approx_one_percent; use crate::scheme::Scheme; use crate::stats::ArrayAndStats; -/// Closure type for [`CompressionEstimate::Estimate`]. The compressor calls this with the same -/// arguments it would pass to sampling. +/// Closure type for [`DeferredEstimate::Callback`]. +/// +/// The compressor calls this with the same arguments it would pass to sampling. The closure must +/// resolve directly to a terminal [`EstimateVerdict`]. #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &mut ArrayAndStats, CompressorContext, - ) -> VortexResult + ) -> VortexResult + Send + Sync; -// TODO(connor): We should make use of the fact that some checks are cheap and some checks are -// expensive (sample or estimate variants). /// The result of a [`Scheme`]'s compression ratio estimation. /// /// This type is returned by [`Scheme::expected_compression_ratio`] to tell the compressor how /// promising this scheme is for a given array without performing any expensive work. /// -/// All expensive or fallible operations (sampling, trial encoding) are deferred to the compressor -/// via the [`Sample`](CompressionEstimate::Sample) and [`Estimate`](CompressionEstimate::Estimate) -/// variants. -/// -/// [`Sample`]: CompressionEstimate::Sample -/// [`Estimate`]: CompressionEstimate::Estimate +/// [`CompressionEstimate::Verdict`] means the scheme already knows the terminal answer. +/// [`CompressionEstimate::Deferred`] means the compressor must do extra work before the scheme can +/// produce a terminal answer. +#[derive(Debug)] pub enum CompressionEstimate { + /// The scheme already knows the terminal estimation verdict. + Verdict(EstimateVerdict), + + /// The compressor must perform deferred work to resolve the terminal estimation verdict. + Deferred(DeferredEstimate), +} + +/// The terminal answer to a compression estimate request. +#[derive(Debug)] +pub enum EstimateVerdict { /// Do not use this scheme for this array. Skip, - /// Always use this scheme, as we know it is definitively the best choice. + /// Always use this scheme, as it is definitively the best choice. /// /// Some examples include constant detection, decimal byte parts, and temporal decomposition. /// @@ -60,21 +68,20 @@ pub enum CompressionEstimate { /// The estimated compression ratio. This must be greater than `1.0` to be considered by the /// compressor, otherwise it is worse than the canonical encoding. Ratio(f64), +} +/// Deferred work that can resolve to a terminal [`EstimateVerdict`]. +pub enum DeferredEstimate { /// The scheme cannot cheaply estimate its ratio, so the compressor should compress a small /// sample to determine effectiveness. Sample, - /// A fallible estimation requiring a custom expensive computation. The compressor will call the - /// closure and handle the result. + /// A fallible estimation requiring a custom expensive computation. /// /// Use this only when the scheme needs to perform trial encoding or other costly checks to - /// determine its compression ratio. - /// - /// The estimation function must **not** return a [`Sample`](CompressionEstimate::Sample) or - /// [`Estimate`](CompressionEstimate::Estimate) variant to ensure the estimation process is - /// bounded. - Estimate(Box), + /// determine its compression ratio. The callback returns an [`EstimateVerdict`] directly, so + /// it cannot request more sampling or another deferred callback. + Callback(Box), } /// Returns `true` if `ratio` is a valid compression ratio (> 1.0, finite, not subnormal) that @@ -138,14 +145,11 @@ pub(super) fn estimate_compression_ratio_with_sampling( Ok(ratio) } -impl fmt::Debug for CompressionEstimate { +impl fmt::Debug for DeferredEstimate { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - CompressionEstimate::Skip => write!(f, "Skip"), - CompressionEstimate::AlwaysUse => write!(f, "AlwaysUse"), - CompressionEstimate::Ratio(r) => f.debug_tuple("Ratio").field(r).finish(), - CompressionEstimate::Sample => write!(f, "Sample"), - CompressionEstimate::Estimate(_) => write!(f, "Estimate(..)"), + DeferredEstimate::Sample => write!(f, "Sample"), + DeferredEstimate::Callback(_) => write!(f, "Callback(..)"), } } } diff --git a/vortex-compressor/src/scheme.rs b/vortex-compressor/src/scheme.rs index 66b35051e95..c3f1760d373 100644 --- a/vortex-compressor/src/scheme.rs +++ b/vortex-compressor/src/scheme.rs @@ -129,10 +129,10 @@ pub struct AncestorExclusion { /// /// # Implementing a scheme /// -/// [`expected_compression_ratio`] should return [`CompressionEstimate::Sample`] when a cheap -/// heuristic is not available, asking the compressor to estimate via sampling. Implementors should -/// return a more specific variant when possible (e.g. [`CompressionEstimate::AlwaysUse`] for -/// constant detection or [`CompressionEstimate::Skip`] for early rejection based on stats). +/// [`expected_compression_ratio`] should return +/// `CompressionEstimate::Deferred(DeferredEstimate::Sample)` when a cheap heuristic is not +/// available, asking the compressor to estimate via sampling. Implementors should return an +/// immediate [`CompressionEstimate::Verdict`] when possible. /// /// Schemes that need statistics that may be expensive to compute should override [`stats_options`] /// to declare what they require. The compressor merges all eligible schemes' options before @@ -184,14 +184,21 @@ pub trait Scheme: Debug + Send + Sync { /// Cheaply estimate the compression ratio for this scheme on the given array. /// - /// This method should be fast and infallible. Any expensive or fallible work should be deferred - /// to the compressor by returning [`CompressionEstimate::Sample`] or - /// [`CompressionEstimate::Estimate`]. + /// This method should be fast and infallible. Any expensive or fallible work should be + /// deferred to the compressor by returning + /// `CompressionEstimate::Deferred(DeferredEstimate::Sample)` or + /// `CompressionEstimate::Deferred(DeferredEstimate::Callback(...))`. /// /// The compressor will ask all schemes what their expected compression ratio is given the array /// and statistics. The scheme with the highest estimated ratio will then be applied to the /// entire array. /// + /// [`CompressionEstimate::Verdict`] means the scheme already knows the terminal + /// [`crate::estimate::EstimateVerdict`]. `CompressionEstimate::Deferred(DeferredEstimate::Sample)` + /// asks the compressor to sample. `CompressionEstimate::Deferred(DeferredEstimate::Callback(...))` + /// asks the compressor to run custom deferred work. Deferred callbacks must return a + /// [`crate::estimate::EstimateVerdict`] directly, never another deferred request. + /// /// Note that the compressor will also use this method when compressing samples, so some /// statistics that might hold for the samples may not hold for the entire array (e.g., /// `Constant`). Implementations should check `ctx.is_sample` to make sure that they are diff --git a/vortex-tensor/src/encodings/turboquant/scheme.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs index ba9b95dd1e0..b603f12e16e 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -29,6 +29,7 @@ use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::Scheme; use vortex_compressor::stats::ArrayAndStats; use vortex_error::VortexExpect; @@ -91,11 +92,11 @@ impl Scheme for TurboQuantScheme { .vortex_expect("invalid bit width for TurboQuant"); let dimension = vector_metadata.dimensions(); - CompressionEstimate::Ratio(estimate_compression_ratio( + CompressionEstimate::Verdict(EstimateVerdict::Ratio(estimate_compression_ratio( element_bit_width, dimension, len, - )) + ))) } fn compress( From d352612f72cd2b25553d828fda100e3ac7ea9211 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 13 Apr 2026 21:09:41 +0100 Subject: [PATCH 027/250] fix[ci]: bench no fail-fast random access (#7418) ## Summary Closes: #000 ## Testing Signed-off-by: Joe Isaacs --- .github/workflows/bench.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7be829d80bd..558a5bd51ff 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -38,6 +38,7 @@ jobs: && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) || 'ubuntu-latest' }} strategy: + fail-fast: false matrix: benchmark: - id: random-access-bench From 3fd9f260de3a6c833f34914deea4e704321f6f26 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 14 Apr 2026 00:15:20 +0100 Subject: [PATCH 028/250] Remove getrandom feature flag overrides (#7421) These have been required at some point in our codebase to get getrandom v0.2 and v0.3 to coexsit in wasm setup. However, the world has moved from getrandom v0.2 and override situation is not as complicated anymore. It seems we can wholesale remove these overrides that we kept around for way too long. fix #7382 Signed-off-by: Robert Kruszewski Signed-off-by: Robert Kruszewski --- .cargo/config.toml | 3 --- .github/workflows/ci.yml | 2 +- Cargo.lock | 6 ------ Cargo.toml | 5 ----- encodings/parquet-variant/Cargo.toml | 2 -- encodings/pco/Cargo.toml | 2 -- vortex-array/Cargo.toml | 4 ---- vortex-btrblocks/Cargo.toml | 4 ---- vortex-file/Cargo.toml | 5 ----- vortex-io/Cargo.toml | 4 ---- vortex-metrics/Cargo.toml | 4 ---- 11 files changed, 1 insertion(+), 40 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index d5b9f580321..466422a80b1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,9 +3,6 @@ rustflags = [ "-C", "force-frame-pointers=yes", ] -[target.wasm32-unknown-unknown] -rustflags = ['--cfg', 'getrandom_backend="wasm_js"', '-C', 'target-feature=+atomics'] - [alias] xtask = "run -p xtask --" vx = "run -p vortex-tui --" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb7b86d204c..e454301989f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -203,7 +203,7 @@ jobs: runner: amd64-medium target: wasm32-unknown-unknown env: - rustflags: "RUSTFLAGS='-A warnings'" + rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'" args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-sqllogictest" steps: - uses: runs-on/action@v2 diff --git a/Cargo.lock b/Cargo.lock index 0b84a2f407a..52f583dd2cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10129,7 +10129,6 @@ dependencies = [ "enum-iterator", "flatbuffers", "futures", - "getrandom 0.4.2", "goldenfile", "half", "humansize", @@ -10228,7 +10227,6 @@ name = "vortex-btrblocks" version = "0.1.0" dependencies = [ "codspeed-divan-compat", - "getrandom 0.4.2", "itertools 0.14.0", "num-traits", "pco", @@ -10553,7 +10551,6 @@ dependencies = [ "bytes", "flatbuffers", "futures", - "getrandom 0.4.2", "itertools 0.14.0", "kanal", "moka", @@ -10563,7 +10560,6 @@ dependencies = [ "pin-project-lite", "tokio", "tracing", - "uuid", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -10653,7 +10649,6 @@ dependencies = [ "bytes", "custom-labels", "futures", - "getrandom 0.4.2", "glob", "handle", "itertools 0.14.0", @@ -10770,7 +10765,6 @@ dependencies = [ name = "vortex-metrics" version = "0.1.0" dependencies = [ - "getrandom 0.4.2", "parking_lot", "sketches-ddsketch 0.4.0", ] diff --git a/Cargo.toml b/Cargo.toml index f486a90b3c7..3c67020c38a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -302,11 +302,6 @@ vortex-duckdb = { path = "./vortex-duckdb", default-features = false } vortex-test-e2e-cuda = { path = "./vortex-test/e2e-cuda", default-features = false } vortex-tui = { path = "./vortex-tui" } -[workspace.dependencies.getrandom_v03] -features = ["wasm_js"] -package = "getrandom" -version = "0.4.0" - [workspace.lints.rust] let_underscore_drop = "deny" macro_use_extern_crate = "deny" diff --git a/encodings/parquet-variant/Cargo.toml b/encodings/parquet-variant/Cargo.toml index cc554c85d10..482659a616d 100644 --- a/encodings/parquet-variant/Cargo.toml +++ b/encodings/parquet-variant/Cargo.toml @@ -36,5 +36,3 @@ vortex-session = { workspace = true } rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] diff --git a/encodings/pco/Cargo.toml b/encodings/pco/Cargo.toml index bc222bf1dcc..8d560d5f068 100644 --- a/encodings/pco/Cargo.toml +++ b/encodings/pco/Cargo.toml @@ -29,5 +29,3 @@ vortex-session = { workspace = true } rstest = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 74fa772820c..b1cb8fcdfcf 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -38,7 +38,6 @@ cudarc = { workspace = true, optional = true } enum-iterator = { workspace = true } flatbuffers = { workspace = true } futures = { workspace = true, features = ["alloc", "async-await", "std"] } -getrandom_v03 = { workspace = true } goldenfile = { workspace = true, optional = true } half = { workspace = true, features = ["num-traits"] } humansize = { workspace = true } @@ -184,6 +183,3 @@ harness = false [[bench]] name = "listview_rebuild" harness = false - -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index 35383a5f5ed..dab4f27fb51 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -14,7 +14,6 @@ rust-version = { workspace = true } version = { workspace = true } [dependencies] -getrandom_v03 = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } pco = { workspace = true, optional = true } @@ -64,6 +63,3 @@ test = false name = "compress_listview" harness = false test = false - -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] diff --git a/vortex-file/Cargo.toml b/vortex-file/Cargo.toml index e812ab228c6..77d664a12cb 100644 --- a/vortex-file/Cargo.toml +++ b/vortex-file/Cargo.toml @@ -21,7 +21,6 @@ async-trait = { workspace = true } bytes = { workspace = true } flatbuffers = { workspace = true } futures = { workspace = true, features = ["std", "async-await"] } -getrandom_v03 = { workspace = true } # Needed to pickup the "wasm_js" feature for wasm targets from the workspace configuration itertools = { workspace = true } kanal = { workspace = true } moka = { workspace = true, features = ["sync"] } @@ -31,7 +30,6 @@ parking_lot = { workspace = true } pin-project-lite = { workspace = true } tokio = { workspace = true, features = ["rt"], optional = true } tracing = { workspace = true } -uuid = { workspace = true } # Needed to pickup the "js" feature for wasm targets from the workspace configuration vortex-alp = { workspace = true } vortex-array = { workspace = true } vortex-btrblocks = { workspace = true } @@ -84,6 +82,3 @@ unstable_encodings = [ "vortex-zstd?/unstable_encodings", "vortex-btrblocks/unstable_encodings", ] - -[package.metadata.cargo-machete] -ignored = ["getrandom_v03", "uuid"] diff --git a/vortex-io/Cargo.toml b/vortex-io/Cargo.toml index a73baec63c6..dcb9377e55e 100644 --- a/vortex-io/Cargo.toml +++ b/vortex-io/Cargo.toml @@ -22,7 +22,6 @@ async-stream = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } futures = { workspace = true, features = ["std", "executor"] } -getrandom_v03 = { workspace = true } # Needed to pickup the "wasm_js" feature for wasm targets from the workspace configuration glob = { workspace = true } handle = "1.0.2" kanal = { workspace = true } @@ -67,6 +66,3 @@ tokio = ["tokio/fs", "tokio/rt-multi-thread"] [lints] workspace = true - -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] diff --git a/vortex-metrics/Cargo.toml b/vortex-metrics/Cargo.toml index fe8a783a4b5..a9d98eb448c 100644 --- a/vortex-metrics/Cargo.toml +++ b/vortex-metrics/Cargo.toml @@ -14,12 +14,8 @@ rust-version = { workspace = true } version = { workspace = true } [dependencies] -getrandom_v03 = { workspace = true } parking_lot = { workspace = true } sketches-ddsketch = { workspace = true } [lints] workspace = true - -[package.metadata.cargo-machete] -ignored = ["getrandom_v03"] From 82a608157d210480360bd79c3a2e886c354c5dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Tue, 14 Apr 2026 10:42:10 +0200 Subject: [PATCH 029/250] perf[vortex-array]: use from_trusted_len_iter in primitive casts (#7401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of our scan profiles show 10% of scan cpu time is spent in integer widening casts (nullable dictionary codes). This commit simplifies and optimizes primitive casts by hoisting a lot of hot loop branching logic. Specifically, this commit relies on values_fit_in to verify representability so that we can avoid a potential validity and error check in the hot loop. Additionally from_trusted_len_iter lets the destination BufferMut optimize the actual cast instead of using push_unchecked for each element. Results locally running the benchmark from #7400. Before: ``` $ cargo bench -p vortex-array --bench cast_primitive Finished `bench` profile [optimized + debuginfo] target(s) in 0.16s Running benches/cast_primitive.rs (target/release/deps/cast_primitive-598823f32b8f3db0) Timer precision: 41 ns cast_primitive fastest │ slowest │ median │ mean │ samples │ iters ╰─ cast_u16_to_u32 384.2 µs │ 491.6 µs │ 395.1 µs │ 397.3 µs │ 100 │ 100 ``` After: ``` cargo bench -p vortex-array --bench cast_primitive Finished `bench` profile [optimized + debuginfo] target(s) in 0.17s Running benches/cast_primitive.rs (target/release/deps/cast_primitive-598823f32b8f3db0) Timer precision: 41 ns cast_primitive fastest │ slowest │ median │ mean │ samples │ iters ╰─ cast_u16_to_u32 6.874 µs │ 543.7 µs │ 6.999 µs │ 12.7 µs │ 100 │ 100 ``` Signed-off-by: Alfonso Subiotto Marques --- vortex-array/benches/cast_primitive.rs | 3 + .../src/arrays/primitive/compute/cast.rs | 74 +++++++++---------- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/vortex-array/benches/cast_primitive.rs b/vortex-array/benches/cast_primitive.rs index 3e1b0831736..be200ef137d 100644 --- a/vortex-array/benches/cast_primitive.rs +++ b/vortex-array/benches/cast_primitive.rs @@ -9,6 +9,7 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; +use vortex_array::expr::stats::Stat; fn main() { divan::main(); @@ -28,6 +29,8 @@ fn cast_u16_to_u32(bencher: Bencher) { } })) .into_array(); + // Pre-compute min/max so values_fit_in is a cache hit during the benchmark. + arr.statistics().compute_all(&[Stat::Min, Stat::Max]).ok(); bencher.with_inputs(|| arr.clone()).bench_refs(|a| { #[expect(clippy::unwrap_used)] a.cast(DType::Primitive(PType::U32, Nullability::Nullable)) diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index dc98495ef11..13cba60bfa6 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -1,13 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use num_traits::AsPrimitive; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_err; -use vortex_mask::AllOr; -use vortex_mask::Mask; use crate::ArrayRef; use crate::ExecutionCtx; @@ -53,6 +51,14 @@ impl CastKernel for Primitive { })); } + if !values_fit_in(array, new_ptype, ctx) { + vortex_bail!( + Compute: "Cannot cast {} to {} — values exceed target range", + array.ptype(), + new_ptype, + ); + } + // Same-width integers have identical bit representations due to 2's // complement. If all values fit in the target range, reinterpret with // no allocation. @@ -60,13 +66,6 @@ impl CastKernel for Primitive { && new_ptype.is_int() && array.ptype().byte_width() == new_ptype.byte_width() { - if !values_fit_in(array, new_ptype, ctx) { - vortex_bail!( - Compute: "Cannot cast {} to {} — values exceed target range", - array.ptype(), - new_ptype, - ); - } // SAFETY: both types are integers with the same size and alignment, and // min/max confirm all valid values are representable in the target type. return Ok(Some(unsafe { @@ -79,13 +78,10 @@ impl CastKernel for Primitive { })); } - let mask = array.validity_mask(); - - // Otherwise, we need to cast the values one-by-one. + // Otherwise, cast the values element-wise. Ok(Some(match_each_native_ptype!(new_ptype, |T| { match_each_native_ptype!(array.ptype(), |F| { - PrimitiveArray::new(cast::(array.as_slice(), mask)?, new_validity) - .into_array() + PrimitiveArray::new(cast::(array.as_slice()), new_validity).into_array() }) }))) } @@ -104,30 +100,11 @@ fn values_fit_in( .is_none_or(|mm| mm.min.cast(&target_dtype).is_ok() && mm.max.cast(&target_dtype).is_ok()) } -fn cast(array: &[F], mask: Mask) -> VortexResult> { - let try_cast = |src: F| -> VortexResult { - T::from(src).ok_or_else(|| vortex_err!(Compute: "Failed to cast {} to {:?}", src, T::PTYPE)) - }; - match mask.bit_buffer() { - AllOr::None => Ok(Buffer::zeroed(array.len())), - AllOr::All => { - let mut buffer = BufferMut::with_capacity(array.len()); - for &src in array { - // SAFETY: we've pre-allocated the required capacity - unsafe { buffer.push_unchecked(try_cast(src)?) } - } - Ok(buffer.freeze()) - } - AllOr::Some(b) => { - let mut buffer = BufferMut::with_capacity(array.len()); - for (&src, valid) in array.iter().zip(b.iter()) { - let dst = if valid { try_cast(src)? } else { T::default() }; - // SAFETY: we've pre-allocated the required capacity - unsafe { buffer.push_unchecked(dst) } - } - Ok(buffer.freeze()) - } - } +/// Caller must ensure all valid values are representable via `values_fit_in`. +/// Out-of-range values at invalid positions are truncated/wrapped by `as`, +/// which is fine because they are masked out by validity. +fn cast, T: NativePType>(array: &[F]) -> Buffer { + BufferMut::from_trusted_len_iter(array.iter().map(|&src| src.as_())).freeze() } #[cfg(test)] @@ -319,6 +296,23 @@ mod test { Ok(()) } + #[test] + fn cast_u32_to_u8_with_out_of_range_nulls() -> vortex_error::VortexResult<()> { + let arr = PrimitiveArray::new( + buffer![1000u32, 10u32, 42u32], + Validity::from_iter([false, true, true]), + ); + let casted = arr + .into_array() + .cast(DType::Primitive(PType::U8, Nullability::Nullable))? + .to_primitive(); + assert_arrays_eq!( + casted, + PrimitiveArray::from_option_iter([None, Some(10u8), Some(42)]) + ); + Ok(()) + } + #[rstest] #[case(buffer![0u8, 1, 2, 3, 255].into_array())] #[case(buffer![0u16, 100, 1000, 65535].into_array())] @@ -329,7 +323,9 @@ mod test { #[case(buffer![-1000000i32, -1, 0, 1, 1000000].into_array())] #[case(buffer![-1000000000i64, -1, 0, 1, 1000000000].into_array())] #[case(buffer![0.0f32, 1.5, -2.5, 100.0, 1e6].into_array())] + #[case(buffer![f32::NAN, f32::INFINITY, f32::NEG_INFINITY, 0.0f32].into_array())] #[case(buffer![0.0f64, 1.5, -2.5, 100.0, 1e12].into_array())] + #[case(buffer![f64::NAN, f64::INFINITY, f64::NEG_INFINITY, 0.0f64].into_array())] #[case(PrimitiveArray::from_option_iter([Some(1u8), None, Some(255), Some(0), None]).into_array())] #[case(PrimitiveArray::from_option_iter([Some(1i32), None, Some(-100), Some(0), None]).into_array())] #[case(buffer![42u32].into_array())] From 0bcd8fa083adbd177719979ecfe93bd09e6b16d4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:03:37 +0100 Subject: [PATCH 030/250] Update dependency pytest to v9 [SECURITY] (#7419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [pytest](https://redirect.github.com/pytest-dev/pytest) ([changelog](https://docs.pytest.org/en/stable/changelog.html)) | `8.4.2` → `9.0.3` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/pytest/9.0.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pytest/8.4.2/9.0.3?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. ### GitHub Vulnerability Alerts #### [CVE-2025-71176](https://nvd.nist.gov/vuln/detail/CVE-2025-71176) pytest through 9.0.2 on UNIX relies on directories with the `/tmp/pytest-of-{user}` name pattern, which allows local users to cause a denial of service or possibly gain privileges. --- ### Release Notes
pytest-dev/pytest (pytest) ### [`v9.0.3`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.3) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.2...9.0.3) ### pytest 9.0.3 (2026-04-07) #### Bug fixes - [#​12444](https://redirect.github.com/pytest-dev/pytest/issues/12444): Fixed `pytest.approx` which now correctly takes into account `~collections.abc.Mapping` keys order to compare them. - [#​13634](https://redirect.github.com/pytest-dev/pytest/issues/13634): Blocking a `conftest.py` file using the `-p no:` option is now explicitly disallowed. Previously this resulted in an internal assertion failure during plugin loading. Pytest now raises a clear `UsageError` explaining that conftest files are not plugins and cannot be disabled via `-p`. - [#​13734](https://redirect.github.com/pytest-dev/pytest/issues/13734): Fixed crash when a test raises an exceptiongroup with `__tracebackhide__ = True`. - [#​14195](https://redirect.github.com/pytest-dev/pytest/issues/14195): Fixed an issue where non-string messages passed to unittest.TestCase.subTest() were not printed. - [#​14343](https://redirect.github.com/pytest-dev/pytest/issues/14343): Fixed use of insecure temporary directory (CVE-2025-71176). #### Improved documentation - [#​13388](https://redirect.github.com/pytest-dev/pytest/issues/13388): Clarified documentation for `-p` vs `PYTEST_PLUGINS` plugin loading and fixed an incorrect `-p` example. - [#​13731](https://redirect.github.com/pytest-dev/pytest/issues/13731): Clarified that capture fixtures (e.g. `capsys` and `capfd`) take precedence over the `-s` / `--capture=no` command-line options in `Accessing captured output from a test function `. - [#​14088](https://redirect.github.com/pytest-dev/pytest/issues/14088): Clarified that the default `pytest_collection` hook sets `session.items` before it calls `pytest_collection_finish`, not after. - [#​14255](https://redirect.github.com/pytest-dev/pytest/issues/14255): TOML integer log levels must be quoted: Updating reference documentation. #### Contributor-facing changes - [#​12689](https://redirect.github.com/pytest-dev/pytest/issues/12689): The test reports are now published to Codecov from GitHub Actions. The test statistics is visible [on the web interface](https://app.codecov.io/gh/pytest-dev/pytest/tests). \-- by `aleguy02` ### [`v9.0.2`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.2) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.1...9.0.2) ### pytest 9.0.2 (2025-12-06) #### Bug fixes - [#​13896](https://redirect.github.com/pytest-dev/pytest/issues/13896): The terminal progress feature added in pytest 9.0.0 has been disabled by default, except on Windows, due to compatibility issues with some terminal emulators. You may enable it again by passing `-p terminalprogress`. We may enable it by default again once compatibility improves in the future. Additionally, when the environment variable `TERM` is `dumb`, the escape codes are no longer emitted, even if the plugin is enabled. - [#​13904](https://redirect.github.com/pytest-dev/pytest/issues/13904): Fixed the TOML type of the `tmp_path_retention_count` settings in the API reference from number to string. - [#​13946](https://redirect.github.com/pytest-dev/pytest/issues/13946): The private `config.inicfg` attribute was changed in a breaking manner in pytest 9.0.0. Due to its usage in the ecosystem, it is now restored to working order using a compatibility shim. It will be deprecated in pytest 9.1 and removed in pytest 10. - [#​13965](https://redirect.github.com/pytest-dev/pytest/issues/13965): Fixed quadratic-time behavior when handling `unittest` subtests in Python 3.10. #### Improved documentation - [#​4492](https://redirect.github.com/pytest-dev/pytest/issues/4492): The API Reference now contains cross-reference-able documentation of `pytest's command-line flags `. ### [`v9.0.1`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.1) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/9.0.0...9.0.1) ### pytest 9.0.1 (2025-11-12) #### Bug fixes - [#​13895](https://redirect.github.com/pytest-dev/pytest/issues/13895): Restore support for skipping tests via `raise unittest.SkipTest`. - [#​13896](https://redirect.github.com/pytest-dev/pytest/issues/13896): The terminal progress plugin added in pytest 9.0 is now automatically disabled when iTerm2 is detected, it generated desktop notifications instead of the desired functionality. - [#​13904](https://redirect.github.com/pytest-dev/pytest/issues/13904): Fixed the TOML type of the verbosity settings in the API reference from number to string. - [#​13910](https://redirect.github.com/pytest-dev/pytest/issues/13910): Fixed UserWarning: Do not expect file\_or\_dir on some earlier Python 3.12 and 3.13 point versions. #### Packaging updates and notes for downstreams - [#​13933](https://redirect.github.com/pytest-dev/pytest/issues/13933): The tox configuration has been adjusted to make sure the desired version string can be passed into its `package_env` through the `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST` environment variable as a part of the release process -- by `webknjaz`. #### Contributor-facing changes - [#​13891](https://redirect.github.com/pytest-dev/pytest/issues/13891), [#​13942](https://redirect.github.com/pytest-dev/pytest/issues/13942): The CI/CD part of the release automation is now capable of creating GitHub Releases without having a Git checkout on disk -- by `bluetech` and `webknjaz`. - [#​13933](https://redirect.github.com/pytest-dev/pytest/issues/13933): The tox configuration has been adjusted to make sure the desired version string can be passed into its `package_env` through the `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST` environment variable as a part of the release process -- by `webknjaz`. ### [`v9.0.0`](https://redirect.github.com/pytest-dev/pytest/releases/tag/9.0.0) [Compare Source](https://redirect.github.com/pytest-dev/pytest/compare/8.4.2...9.0.0) ### pytest 9.0.0 (2025-11-05) #### New features - [#​1367](https://redirect.github.com/pytest-dev/pytest/issues/1367): **Support for subtests** has been added. `subtests ` are an alternative to parametrization, useful in situations where the parametrization values are not all known at collection time. Example: ```python def contains_docstring(p: Path) -> bool: """Return True if the given Python file contains a top-level docstring.""" ... def test_py_files_contain_docstring(subtests: pytest.Subtests) -> None: for path in Path.cwd().glob("*.py"): with subtests.test(path=str(path)): assert contains_docstring(path) ``` Each assert failure or error is caught by the context manager and reported individually, giving a clear picture of all files that are missing a docstring. In addition, `unittest.TestCase.subTest` is now also supported. This feature was originally implemented as a separate plugin in [pytest-subtests](https://redirect.github.com/pytest-dev/pytest-subtests), but since then has been merged into the core. > \[!NOTE] > This feature is experimental and will likely evolve in future releases. By that we mean that we might change how subtests are reported on failure, but the functionality and how to use it are stable. - [#​13743](https://redirect.github.com/pytest-dev/pytest/issues/13743): Added support for **native TOML configuration files**. While pytest, since version 6, supports configuration in `pyproject.toml` files under `[tool.pytest.ini_options]`, it does so in an "INI compatibility mode", where all configuration values are treated as strings or list of strings. Now, pytest supports the native TOML data model. In `pyproject.toml`, the native TOML configuration is under the `[tool.pytest]` table. ```toml # pyproject.toml [tool.pytest] minversion = "9.0" addopts = ["-ra", "-q"] testpaths = [ "tests", "integration", ] ``` The `[tool.pytest.ini_options]` table remains supported, but both tables cannot be used at the same time. If you prefer to use a separate configuration file, or don't use `pyproject.toml`, you can use `pytest.toml` or `.pytest.toml`: ```toml # pytest.toml or .pytest.toml [pytest] minversion = "9.0" addopts = ["-ra", "-q"] testpaths = [ "tests", "integration", ] ``` The documentation now (sometimes) shows configuration snippets in both TOML and INI formats, in a tabbed interface. See `config file formats` for full details. - [#​13823](https://redirect.github.com/pytest-dev/pytest/issues/13823): Added a **"strict mode"** enabled by the `strict` configuration option. When set to `true`, the `strict` option currently enables - `strict_config` - `strict_markers` - `strict_parametrization_ids` - `strict_xfail` The individual strictness options can be explicitly set to override the global `strict` setting. The previously-deprecated `--strict` command-line flag now enables strict mode. If pytest adds new strictness options in the future, they will also be enabled in strict mode. Therefore, you should only enable strict mode if you use a pinned/locked version of pytest, or if you want to proactively adopt new strictness options as they are added. See `strict mode` for more details. - [#​13737](https://redirect.github.com/pytest-dev/pytest/issues/13737): Added the `strict_parametrization_ids` configuration option. When set, pytest emits an error if it detects non-unique parameter set IDs, rather than automatically making the IDs unique by adding 0, 1, ... to them. This can be particularly useful for catching unintended duplicates. - [#​13072](https://redirect.github.com/pytest-dev/pytest/issues/13072): Added support for displaying test session **progress in the terminal tab** using the [OSC 9;4;](https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC) ANSI sequence. When pytest runs in a supported terminal emulator like ConEmu, Gnome Terminal, Ptyxis, Windows Terminal, Kitty or Ghostty, you'll see the progress in the terminal tab or window, allowing you to monitor pytest's progress at a glance. This feature is automatically enabled when running in a TTY. It is implemented as an internal plugin. If needed, it can be disabled as follows: - On a user level, using `-p no:terminalprogress` on the command line or via an environment variable `PYTEST_ADDOPTS='-p no:terminalprogress'`. - On a project configuration level, using `addopts = "-p no:terminalprogress"`. - [#​478](https://redirect.github.com/pytest-dev/pytest/issues/478): Support PEP420 (implicit namespace packages) as --pyargs target when `consider_namespace_packages` is true in the config. Previously, this option only impacted package imports, now it also impacts tests discovery. - [#​13678](https://redirect.github.com/pytest-dev/pytest/issues/13678): Added a new `faulthandler_exit_on_timeout` configuration option set to "false" by default to let faulthandler interrupt the pytest process after a timeout in case of deadlock. Previously, a faulthandler timeout would only dump the traceback of all threads to stderr, but would not interrupt the pytest process. \-- by `ogrisel`. - [#​13829](https://redirect.github.com/pytest-dev/pytest/issues/13829): Added support for configuration option aliases via the `aliases` parameter in `Parser.addini() `. Plugins can now register alternative names for configuration options, allowing for more flexibility in configuration naming and supporting backward compatibility when renaming options. The canonical name always takes precedence if both the canonical name and an alias are specified in the configuration file. #### Improvements in existing functionality - [#​13330](https://redirect.github.com/pytest-dev/pytest/issues/13330): Having pytest configuration spread over more than one file (for example having both a `pytest.ini` file and `pyproject.toml` with a `[tool.pytest.ini_options]` table) will now print a warning to make it clearer to the user that only one of them is actually used. \-- by `sgaist` - [#​13574](https://redirect.github.com/pytest-dev/pytest/issues/13574): The single argument `--version` no longer loads the entire plugin infrastructure, making it faster and more reliable when displaying only the pytest version. Passing `--version` twice (e.g., `pytest --version --version`) retains the original behavior, showing both the pytest version and plugin information. > \[!NOTE] > Since `--version` is now processed early, it only takes effect when passed directly via the command line. It will not work if set through other mechanisms, such as `PYTEST_ADDOPTS` or `addopts`. - [#​13823](https://redirect.github.com/pytest-dev/pytest/issues/13823): Added `strict_xfail` as an alias to the `xfail_strict` option, `strict_config` as an alias to the `--strict-config` flag, and `strict_markers` as an alias to the `--strict-markers` flag. This makes all strictness options consistently have configuration options with the prefix `strict_`. - [#​13700](https://redirect.github.com/pytest-dev/pytest/issues/13700): --junitxml no longer prints the generated xml file summary at the end of the pytest session when --quiet is given. - [#​13732](https://redirect.github.com/pytest-dev/pytest/issues/13732): Previously, when filtering warnings, pytest would fail if the filter referenced a class that could not be imported. Now, this only outputs a message indicating the problem. - [#​13859](https://redirect.github.com/pytest-dev/pytest/issues/13859): Clarify the error message for pytest.raises() when a regex match fails. - [#​13861](https://redirect.github.com/pytest-dev/pytest/issues/13861): Better sentence structure in a test's expected error message. Previously, the error message would be "expected exception must be \, but got \". Now, it is "Expected \, but got \". #### Removals and backward incompatible breaking changes - [#​12083](https://redirect.github.com/pytest-dev/pytest/issues/12083): Fixed a bug where an invocation such as pytest a/ a/b would cause only tests from a/b to run, and not other tests under a/. The fix entails a few breaking changes to how such overlapping arguments and duplicates are handled: 1. pytest a/b a/ or pytest a/ a/b are equivalent to pytest a; if an argument overlaps another arguments, only the prefix remains. 2. pytest x.py x.py is equivalent to pytest x.py; previously such an invocation was taken as an explicit request to run the tests from the file twice. If you rely on these behaviors, consider using `--keep-duplicates `, which retains its existing behavior (including the bug). - [#​13719](https://redirect.github.com/pytest-dev/pytest/issues/13719): Support for Python 3.9 is dropped following its end of life. - [#​13766](https://redirect.github.com/pytest-dev/pytest/issues/13766): Previously, pytest would assume it was running in a CI/CD environment if either of the environment variables $CI or $BUILD\_NUMBER was defined; now, CI mode is only activated if at least one of those variables is defined and set to a *non-empty* value. - [#​13779](https://redirect.github.com/pytest-dev/pytest/issues/13779): **PytestRemovedIn9Warning deprecation warnings are now errors by default.** Following our plan to remove deprecated features with as little disruption as possible, all warnings of type `PytestRemovedIn9Warning` now generate errors instead of warning messages by default. **The affected features will be effectively removed in pytest 9.1**, so please consult the `deprecations` section in the docs for directions on how to update existing code. In the pytest `9.0.X` series, it is possible to change the errors back into warnings as a stopgap measure by adding this to your `pytest.ini` file: ```ini [pytest] filterwarnings = ignore::pytest.PytestRemovedIn9Warning ``` But this will stop working when pytest `9.1` is released. **If you have concerns** about the removal of a specific feature, please add a comment to `13779`. #### Deprecations (removal in next major release) - [#​13807](https://redirect.github.com/pytest-dev/pytest/issues/13807): `monkeypatch.syspath_prepend() ` now issues a deprecation warning when the prepended path contains legacy namespace packages (those using `pkg_resources.declare_namespace()`). Users should migrate to native namespace packages (`420`). See `monkeypatch-fixup-namespace-packages` for details. #### Bug fixes - [#​13445](https://redirect.github.com/pytest-dev/pytest/issues/13445): Made the type annotations of `pytest.skip` and friends more spec-complaint to have them work across more type checkers. - [#​13537](https://redirect.github.com/pytest-dev/pytest/issues/13537): Fixed a bug in which `ExceptionGroup` with only `Skipped` exceptions in teardown was not handled correctly and showed as error. - [#​13598](https://redirect.github.com/pytest-dev/pytest/issues/13598): Fixed possible collection confusion on Windows when short paths and symlinks are involved. - [#​13716](https://redirect.github.com/pytest-dev/pytest/issues/13716): Fixed a bug where a nonsensical invocation like `pytest x.py[a]` (a file cannot be parametrized) was silently treated as `pytest x.py`. This is now a usage error. - [#​13722](https://redirect.github.com/pytest-dev/pytest/issues/13722): Fixed a misleading assertion failure message when using `pytest.approx` on mappings with differing lengths. - [#​13773](https://redirect.github.com/pytest-dev/pytest/issues/13773): Fixed the static fixture closure calculation to properly consider transitive dependencies requested by overridden fixtures. - [#​13816](https://redirect.github.com/pytest-dev/pytest/issues/13816): Fixed `pytest.approx` which now returns a clearer error message when comparing mappings with different keys. - [#​13849](https://redirect.github.com/pytest-dev/pytest/issues/13849): Hidden `.pytest.ini` files are now picked up as the config file even if empty. This was an inconsistency with non-hidden `pytest.ini`. - [#​13865](https://redirect.github.com/pytest-dev/pytest/issues/13865): Fixed --show-capture with --tb=line. - [#​13522](https://redirect.github.com/pytest-dev/pytest/issues/13522): Fixed `pytester` in subprocess mode ignored all :attr\`pytester.plugins \\` except the first. Fixed `pytester` in subprocess mode silently ignored non-str `pytester.plugins `. Now it errors instead. If you are affected by this, specify the plugin by name, or switch the affected tests to use `pytester.runpytest_inprocess ` explicitly instead. #### Packaging updates and notes for downstreams - [#​13791](https://redirect.github.com/pytest-dev/pytest/issues/13791): Minimum requirements on `iniconfig` and `packaging` were bumped to `1.0.1` and `22.0.0`, respectively. #### Contributor-facing changes - [#​12244](https://redirect.github.com/pytest-dev/pytest/issues/12244): Fixed self-test failures when TERM=dumb. - [#​12474](https://redirect.github.com/pytest-dev/pytest/issues/12474): Added scheduled GitHub Action Workflow to run Sphinx linkchecks in repo documentation. - [#​13621](https://redirect.github.com/pytest-dev/pytest/issues/13621): pytest's own testsuite now handles the `lsof` command hanging (e.g. due to unreachable network filesystems), with the affected selftests being skipped after 10 seconds. - [#​13638](https://redirect.github.com/pytest-dev/pytest/issues/13638): Fixed deprecated `gh pr new` command in `scripts/prepare-release-pr.py`. The script now uses `gh pr create` which is compatible with GitHub CLI v2.0+. - [#​13695](https://redirect.github.com/pytest-dev/pytest/issues/13695): Flush stdout and stderr in Pytester.run to avoid truncated outputs in test\_faulthandler.py::test\_timeout on CI -- by `ogrisel`. - [#​13771](https://redirect.github.com/pytest-dev/pytest/issues/13771): Skip test\_do\_not\_collect\_symlink\_siblings on Windows environments without symlink support to avoid false negatives. - [#​13841](https://redirect.github.com/pytest-dev/pytest/issues/13841): `tox>=4` is now required when contributing to pytest. - [#​13625](https://redirect.github.com/pytest-dev/pytest/issues/13625): Added missing docstrings to `pytest_addoption()`, `pytest_configure()`, and `cacheshow()` functions in `cacheprovider.py`. #### Miscellaneous internal changes - [#​13830](https://redirect.github.com/pytest-dev/pytest/issues/13830): Configuration overrides (`-o`/`--override-ini`) are now processed during startup rather than during `config.getini() `.
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - "" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index c5ec5b0d865..a81fafa3462 100644 --- a/uv.lock +++ b/uv.lock @@ -1142,7 +1142,7 @@ wheels = [ [[package]] name = "pytest" -version = "8.4.2" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1151,9 +1151,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] From d222c870195a0d73e0bc8a841380966dcb0a49c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Tue, 14 Apr 2026 14:38:54 +0200 Subject: [PATCH 031/250] chore[vortex-layout]: allow specifying a set of stats on CompressingStrategy (#7422) ## Summary This allows users to e.g. disable UncompressedSizeInBytes computation, which may be expensive for deeply nested types. ## Testing Signed-off-by: Alfonso Subiotto Marques --- vortex-layout/public-api.lock | 2 ++ vortex-layout/src/layouts/compressed.rs | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 6cc0db21db5..fd983e4e94c 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -172,6 +172,8 @@ pub fn vortex_layout::layouts::compressed::CompressingStrategy::new Self +pub fn vortex_layout::layouts::compressed::CompressingStrategy::with_stats(self, stats: &[vortex_array::expr::stats::Stat]) -> Self + impl core::clone::Clone for vortex_layout::layouts::compressed::CompressingStrategy pub fn vortex_layout::layouts::compressed::CompressingStrategy::clone(&self) -> vortex_layout::layouts::compressed::CompressingStrategy diff --git a/vortex-layout/src/layouts/compressed.rs b/vortex-layout/src/layouts/compressed.rs index 24dc2a02874..60f87b8efe0 100644 --- a/vortex-layout/src/layouts/compressed.rs +++ b/vortex-layout/src/layouts/compressed.rs @@ -54,6 +54,7 @@ impl CompressorPlugin for BtrBlocksCompressor { pub struct CompressingStrategy { child: Arc, compressor: Arc, + stats: Arc<[Stat]>, concurrency: usize, } @@ -63,6 +64,7 @@ impl CompressingStrategy { Self { child: Arc::new(child), compressor: Arc::new(compressor), + stats: Stat::all().collect(), concurrency: std::thread::available_parallelism() .map(|v| v.get()) .unwrap_or(1), @@ -73,6 +75,13 @@ impl CompressingStrategy { self.concurrency = concurrency; self } + + /// Override the set of statistics computed on each chunk before compression. + /// Defaults to `Stat::all()`. + pub fn with_stats(mut self, stats: &[Stat]) -> Self { + self.stats = stats.into(); + self + } } #[async_trait] @@ -87,17 +96,17 @@ impl LayoutStrategy for CompressingStrategy { ) -> VortexResult { let dtype = stream.dtype().clone(); let compressor = Arc::clone(&self.compressor); + let stats = Arc::clone(&self.stats); let handle = session.handle(); let stream = stream .map(move |chunk| { let compressor = Arc::clone(&compressor); + let stats = Arc::clone(&stats); handle.spawn_cpu(move || { let (sequence_id, chunk) = chunk?; // Compute the stats for the chunk prior to compression - chunk - .statistics() - .compute_all(&Stat::all().collect::>())?; + chunk.statistics().compute_all(&stats)?; Ok((sequence_id, compressor.compress_chunk(&chunk)?)) }) }) From d674aa2418c4d31625a974a5ddd6761c5e5d34e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Tue, 14 Apr 2026 15:11:01 +0200 Subject: [PATCH 032/250] fix[vortex-array]: avoid unnecessary checks in check_listview_constant (#7423) ## Summary ListView sizes are checked on each element iteration. This can cause unecessary calls to arrays_value_equal if most but not all sizes are equal. This commit pulls up some fast checks we can do before executing the expensive arrays_value_equal loop. Concretely, only execute the loop if all sizes are equal *and* non-zero *and* offsets are not all equal, since the inverse is trivially constant. [Profile link here, we're seeing that this check is expensive](https://pprof.me/39a57fd47bacf99f6669aa6ba5beed31/?profile_filters=p%3Ahide_libc%3AHide%2520libc%2Cp%3Ahide_tokio_frames%3AHide%2520Tokio%2520Frames%2Cp%3Ahide_rust_futures%3AHide%2520Rust%2520Futures%2520Infrastructure%2Cp%3Ahide_rust_panic_backtrace%3AHide%2520Rust%2520Panic%2520Backtrace%2520Infrastructure) Signed-off-by: Alfonso Subiotto Marques --- .../src/aggregate_fn/fns/is_constant/list.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/list.rs b/vortex-array/src/aggregate_fn/fns/is_constant/list.rs index 533ffa6edf8..c33e325d463 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/list.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/list.rs @@ -4,6 +4,7 @@ use vortex_error::VortexResult; use super::arrays_value_equal; +use super::is_constant; use crate::ExecutionCtx; use crate::arrays::ListViewArray; use crate::arrays::listview::ListViewArrayExt; @@ -20,16 +21,24 @@ pub(super) fn check_listview_constant( return Ok(true); } + if !is_constant(l.sizes(), ctx)? { + // If sizes aren't all equal, can't be constant. + return Ok(false); + } + let first_size = l.size_at(0); - let first_elements = l.list_elements_at(0)?; + if first_size == 0 { + return Ok(true); + } + + if is_constant(l.offsets(), ctx)? { + // If all offsets are identical, every list references the same slice. + return Ok(true); + } + // Check each list individually, this can be expensive. + let first_elements = l.list_elements_at(0)?; for i in 1..l.len() { - if l.size_at(i) != first_size { - return Ok(false); - } - if first_size == 0 { - continue; - } let current_elements = l.list_elements_at(i)?; if !arrays_value_equal(&first_elements, ¤t_elements, ctx)? { return Ok(false); From d87dfedeb6129ca6c806be409000bf4181cf4a91 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 14 Apr 2026 14:22:34 +0100 Subject: [PATCH 033/250] Lower days before stale to 14 from 30 (#7425) The pr will get pinged earlier and the author can choose to close it or remove the label if they want another 14 days Signed-off-by: Robert Kruszewski --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 672be333f07..d24b6a1ec6b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/stale@v10 with: # After 30 days of no activity, a PR will be marked as stale - days-before-pr-stale: 30 + days-before-pr-stale: 14 # PR has 7 more days to become active, otherwise it will be closed. days-before-pr-close: 7 stale-pr-label: "stale" From 1a565602e4e7593b210ec49a427f97626bf44f3a Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Tue, 14 Apr 2026 11:05:21 -0400 Subject: [PATCH 034/250] remove check for ArrayPlugin encoding ID match (#7427) ## Summary When deserializing an array encoded using the experimental patched encoding `ArrayPlugin`, reads can fail b/c of this assertion. `ArrayPlugin`s can/should be able to support multiple codecs The alternative to this is to change ArrayPlugin to return a set of ArrayId's that it covers, rather than a single ID. --------- Signed-off-by: Andrew Duffy --- encodings/fastlanes/src/bitpacking/plugin.rs | 4 ++++ vortex-array/public-api.lock | 12 ++++++++++++ vortex-array/src/array/plugin.rs | 11 +++++++++++ vortex-array/src/serde.rs | 6 +++--- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index fc67714fedd..b712d45b45b 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -82,6 +82,10 @@ impl ArrayPlugin for BitPackedPatchedPlugin { Ok(patched.into_array()) } + + fn is_supported_encoding(&self, id: &ArrayId) -> bool { + id == &BitPacked::ID || id == &Patched.id() + } } #[cfg(test)] diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 629dcd50d4f..1f812c3e7a0 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -19412,6 +19412,8 @@ pub fn vortex_array::vtable::ArrayPlugin::deserialize(&self, dtype: &vortex_arra pub fn vortex_array::vtable::ArrayPlugin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::vtable::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn vortex_array::vtable::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> impl vortex_array::ArrayPlugin for V @@ -19420,6 +19422,8 @@ pub fn V::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, met pub fn V::id(&self) -> arcref::ArcRef +pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result>, vortex_error::VortexError> pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug @@ -23056,6 +23060,10 @@ pub fn vortex_array::ArrayPlugin::id(&self) -> vortex_array::ArrayId pub fn vortex_array::ArrayPlugin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + +pub fn vortex_array::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn vortex_array::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> pub fn vortex_array::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> @@ -23072,6 +23080,10 @@ pub fn V::id(&self) -> arcref::ArcRef pub fn V::id(&self) -> arcref::ArcRef +pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + +pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result>, vortex_error::VortexError> pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result>, vortex_error::VortexError> diff --git a/vortex-array/src/array/plugin.rs b/vortex-array/src/array/plugin.rs index e1ea3657d70..f9411584f44 100644 --- a/vortex-array/src/array/plugin.rs +++ b/vortex-array/src/array/plugin.rs @@ -27,6 +27,9 @@ pub type ArrayPluginRef = Arc; /// [`deserialize`]: ArrayPlugin::deserialize pub trait ArrayPlugin: 'static + Send + Sync { /// Returns the ID for this array encoding. + /// + /// During serde, this is the key the registry uses to find + /// this plugin instance and call the appropriate method on it. fn id(&self) -> ArrayId; /// Serialize the array metadata. @@ -49,6 +52,14 @@ pub trait ArrayPlugin: 'static + Send + Sync { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult; + + /// Can this plugin emit an array with the given encoding. + /// + /// By default, this is just the [ID][Self::id] of the plugin, but + /// can be overridden if this plugin instance supports reading/writing multiple arrays. + fn is_supported_encoding(&self, id: &ArrayId) -> bool { + self.id() == *id + } } impl std::fmt::Debug for dyn ArrayPlugin { diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index 68a00569caf..edbc31bdd0d 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -367,9 +367,9 @@ impl SerializedArray { decoded.dtype(), dtype, ); - assert_eq!( - decoded.encoding_id(), - encoding_id, + + assert!( + plugin.is_supported_encoding(&decoded.encoding_id()), "Array decoded from {} has incorrect encoding {}", encoding_id, decoded.encoding_id(), From 3f56b09aa3094bf0cb0e2d64f3adf5fed1406587 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 14 Apr 2026 16:47:43 +0100 Subject: [PATCH 035/250] chore: benchmark with execution context input (#7429) Create execution context outside of benchmark hot loop --------- Signed-off-by: Joe Isaacs --- .../fastlanes/benches/canonicalize_bench.rs | 21 ++-- .../fsst/benches/chunked_dict_fsst_builder.rs | 16 +-- encodings/fsst/benches/fsst_like.rs | 16 +-- vortex-array/benches/chunk_array_builder.rs | 108 ++++++++++-------- vortex-array/benches/chunked_dict_builder.rs | 20 ++-- vortex-array/benches/dict_mask.rs | 7 +- vortex-array/benches/expr/case_when_bench.rs | 63 +++++----- vortex/benches/single_encoding_throughput.rs | 22 ++-- 8 files changed, 142 insertions(+), 131 deletions(-) diff --git a/encodings/fastlanes/benches/canonicalize_bench.rs b/encodings/fastlanes/benches/canonicalize_bench.rs index e6942067fca..666bf1420a9 100644 --- a/encodings/fastlanes/benches/canonicalize_bench.rs +++ b/encodings/fastlanes/benches/canonicalize_bench.rs @@ -78,11 +78,11 @@ fn canonical_into_non_nullable( chunked.dtype().nullability(), chunk_len * chunk_count, ); - (chunked, primitive_builder) + (chunked, primitive_builder, SESSION.create_execution_ctx()) }) - .bench_refs(|(chunked, primitive_builder)| { + .bench_refs(|(chunked, primitive_builder, ctx)| { chunked - .append_to_builder(primitive_builder, &mut SESSION.create_execution_ctx()) + .append_to_builder(primitive_builder, ctx) .vortex_expect("append failed"); primitive_builder.finish() }); @@ -114,8 +114,13 @@ fn into_canonical_nullable( .collect::>(); bencher - .with_inputs(|| ChunkedArray::from_iter(chunks.clone()).into_array()) - .bench_values(|chunked| chunked.execute::(&mut SESSION.create_execution_ctx())); + .with_inputs(|| { + ( + ChunkedArray::from_iter(chunks.clone()).into_array(), + SESSION.create_execution_ctx(), + ) + }) + .bench_values(|(chunked, mut ctx)| chunked.execute::(&mut ctx)); } #[cfg(not(codspeed))] @@ -140,11 +145,11 @@ fn canonical_into_nullable( chunked.dtype().nullability(), chunk_len * chunk_count, ); - (chunked, primitive_builder) + (chunked, primitive_builder, SESSION.create_execution_ctx()) }) - .bench_refs(|(chunked, primitive_builder)| { + .bench_refs(|(chunked, primitive_builder, ctx)| { chunked - .append_to_builder(primitive_builder, &mut SESSION.create_execution_ctx()) + .append_to_builder(primitive_builder, ctx) .vortex_expect("append failed"); primitive_builder.finish() }); diff --git a/encodings/fsst/benches/chunked_dict_fsst_builder.rs b/encodings/fsst/benches/chunked_dict_fsst_builder.rs index 50e2488d360..fbc959cfd41 100644 --- a/encodings/fsst/benches/chunked_dict_fsst_builder.rs +++ b/encodings/fsst/benches/chunked_dict_fsst_builder.rs @@ -49,13 +49,15 @@ fn chunked_dict_fsst_canonical_into( ) { let chunk = make_dict_fsst_chunks::(len, unique_values, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(args = BENCH_ARGS)] diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index 57489ba158e..2771b558ccc 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -109,13 +109,15 @@ fn bench_like(bencher: Bencher, fsst: &FSSTArray, pattern: &str) { let len = fsst.len(); let arr = fsst.clone().into_array(); let pattern = ConstantArray::new(pattern, len).into_array(); - bencher.bench_local(|| { - Like.try_new_array(len, LikeOptions::default(), [arr.clone(), pattern.clone()]) - .unwrap() - .into_array() - .execute::(&mut SESSION.create_execution_ctx()) - .unwrap() - }); + bencher + .with_inputs(|| SESSION.create_execution_ctx()) + .bench_refs(|ctx| { + Like.try_new_array(len, LikeOptions::default(), [arr.clone(), pattern.clone()]) + .unwrap() + .into_array() + .execute::(ctx) + .unwrap() + }); } #[divan::bench(args = [ diff --git a/vortex-array/benches/chunk_array_builder.rs b/vortex-array/benches/chunk_array_builder.rs index b334b65e9e6..7d1e360c7ce 100644 --- a/vortex-array/benches/chunk_array_builder.rs +++ b/vortex-array/benches/chunk_array_builder.rs @@ -39,26 +39,30 @@ static SESSION: LazyLock = fn chunked_bool_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) { let chunk = make_bool_chunks(len, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(args = BENCH_ARGS)] fn chunked_opt_bool_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) { let chunk = make_opt_bool_chunks(len, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(args = BENCH_ARGS)] @@ -74,16 +78,18 @@ fn chunked_opt_bool_into_canonical(bencher: Bencher, (len, chunk_count): (usize, fn chunked_varbinview_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) { let chunks = make_string_chunks(false, len, chunk_count); - bencher.with_inputs(|| &chunks).bench_refs(|chunk| { - let mut builder = VarBinViewBuilder::with_capacity( - DType::Utf8(chunk.dtype().nullability()), - len * chunk_count, - ); - chunk - .append_to_builder(&mut builder, &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunks, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = VarBinViewBuilder::with_capacity( + DType::Utf8(chunk.dtype().nullability()), + len * chunk_count, + ); + chunk + .append_to_builder(&mut builder, ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(args = BENCH_ARGS)] @@ -99,16 +105,18 @@ fn chunked_varbinview_into_canonical(bencher: Bencher, (len, chunk_count): (usiz fn chunked_varbinview_opt_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) { let chunks = make_string_chunks(true, len, chunk_count); - bencher.with_inputs(|| &chunks).bench_refs(|chunk| { - let mut builder = VarBinViewBuilder::with_capacity( - DType::Utf8(chunk.dtype().nullability()), - len * chunk_count, - ); - chunk - .append_to_builder(&mut builder, &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunks, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = VarBinViewBuilder::with_capacity( + DType::Utf8(chunk.dtype().nullability()), + len * chunk_count, + ); + chunk + .append_to_builder(&mut builder, ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(args = BENCH_ARGS)] @@ -124,13 +132,15 @@ fn chunked_varbinview_opt_into_canonical(bencher: Bencher, (len, chunk_count): ( fn chunked_constant_i32_append_to_builder(bencher: Bencher, (len, chunk_count): (usize, usize)) { let chunk = make_constant_i32_chunks(len, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } const CONSTANT_UTF8_BENCH_ARGS: &[(&str, usize, usize)] = &[ @@ -146,13 +156,15 @@ fn chunked_constant_utf8_append_to_builder( ) { let chunk = make_constant_utf8_chunks(value, len, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } fn make_constant_utf8_chunks(value: &str, len: usize, chunk_count: usize) -> ArrayRef { diff --git a/vortex-array/benches/chunked_dict_builder.rs b/vortex-array/benches/chunked_dict_builder.rs index b42514fcf19..85c8737356f 100644 --- a/vortex-array/benches/chunked_dict_builder.rs +++ b/vortex-array/benches/chunked_dict_builder.rs @@ -40,13 +40,15 @@ fn chunked_dict_primitive_canonical_into( { let chunk = gen_dict_primitive_chunks::(len, unique_values, chunk_count); - bencher.with_inputs(|| &chunk).bench_refs(|chunk| { - let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); - chunk - .append_to_builder(builder.as_mut(), &mut SESSION.create_execution_ctx()) - .vortex_expect("append failed"); - builder.finish() - }) + bencher + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| { + let mut builder = builder_with_capacity(chunk.dtype(), len * chunk_count); + chunk + .append_to_builder(builder.as_mut(), ctx) + .vortex_expect("append failed"); + builder.finish() + }) } #[divan::bench(types = [u32, u64, f32, f64], args = BENCH_ARGS)] @@ -59,6 +61,6 @@ fn chunked_dict_primitive_into_canonical( let chunk = gen_dict_primitive_chunks::(len, unique_values, chunk_count); bencher - .with_inputs(|| chunk.clone()) - .bench_values(|chunk| chunk.execute::(&mut SESSION.create_execution_ctx())) + .with_inputs(|| (chunk.clone(), SESSION.create_execution_ctx())) + .bench_values(|(chunk, mut ctx)| chunk.execute::(&mut ctx)) } diff --git a/vortex-array/benches/dict_mask.rs b/vortex-array/benches/dict_mask.rs index 6f60691cc97..1951e610a51 100644 --- a/vortex-array/benches/dict_mask.rs +++ b/vortex-array/benches/dict_mask.rs @@ -62,14 +62,13 @@ fn bench_dict_mask(bencher: Bencher, (fraction_valid, fraction_masked): (f64, f6 let filter_mask = filter_mask(len, fraction_masked, &mut rng); let session = VortexSession::empty(); bencher - .with_inputs(|| (&array, &filter_mask)) - .bench_refs(|(array, filter_mask)| { - let mut ctx = session.create_execution_ctx(); + .with_inputs(|| (&array, &filter_mask, session.create_execution_ctx())) + .bench_refs(|(array, filter_mask, ctx)| { array .clone() .mask(filter_mask.clone().into_array()) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } diff --git a/vortex-array/benches/expr/case_when_bench.rs b/vortex-array/benches/expr/case_when_bench.rs index 7ab1d3d4bfc..f37daa2b304 100644 --- a/vortex-array/benches/expr/case_when_bench.rs +++ b/vortex-array/benches/expr/case_when_bench.rs @@ -70,14 +70,13 @@ fn case_when_simple(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -98,14 +97,13 @@ fn case_when_nary_3_conditions(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -127,14 +125,13 @@ fn case_when_nary_10_conditions(bencher: Bencher, size: usize) { let expr = nested_case_when(pairs, Some(lit(0i32))); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -151,14 +148,13 @@ fn case_when_nary_equality_lookup(bencher: Bencher, size: usize) { let expr = nested_case_when(pairs, Some(lit(-1i32))); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -172,14 +168,13 @@ fn case_when_without_else(bencher: Bencher, size: usize) { let expr = case_when_no_else(gt(get_item("value", root()), lit(500i32)), lit(100i32)); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -197,14 +192,13 @@ fn case_when_all_true(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -231,14 +225,13 @@ fn case_when_nary_early_dominant(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -256,14 +249,13 @@ fn case_when_all_false(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } @@ -284,14 +276,13 @@ fn case_when_fragmented(bencher: Bencher, size: usize) { ); bencher - .with_inputs(|| (&expr, &array)) - .bench_refs(|(expr, array)| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (&expr, &array, SESSION.create_execution_ctx())) + .bench_refs(|(expr, array, ctx)| { array .clone() .apply(expr) .unwrap() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 9b7e069d963..e82f5fdd87a 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -152,9 +152,9 @@ fn bench_delta_compress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &uint_array) - .bench_refs(|a| { - let (_bases, _deltas) = delta_compress(a, &mut SESSION.create_execution_ctx()).unwrap(); + .with_inputs(|| (&uint_array, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| { + let (_bases, _deltas) = delta_compress(a, ctx).unwrap(); DeltaData::try_new(0).unwrap() }); } @@ -512,16 +512,15 @@ mod turboquant_benches { let normalized_ext = setup_normalized_vector_ext($dim); let config = turboquant_config($bits); with_byte_counter(bencher, (NUM_VECTORS * $dim * 4) as u64) - .with_inputs(|| normalized_ext.clone()) - .bench_refs(|a| { - let mut ctx = SESSION.create_execution_ctx(); + .with_inputs(|| (normalized_ext.clone(), SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| { let normalized = a .as_ref() .as_opt::() .expect("normalized benchmark input should be an Extension array"); // SAFETY: Benchmark inputs are normalized once up front so the timed // region measures only TurboQuant encoding. - unsafe { turboquant_encode_unchecked(normalized, &config, &mut ctx) } + unsafe { turboquant_encode_unchecked(normalized, &config, ctx) } .unwrap() }); } @@ -539,12 +538,11 @@ mod turboquant_benches { } .unwrap(); with_byte_counter(bencher, (NUM_VECTORS * $dim * 4) as u64) - .with_inputs(|| &compressed) - .bench_refs(|a| { - let mut ctx = SESSION.create_execution_ctx(); - a.clone() + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| { + (*a).clone() .into_array() - .execute::(&mut ctx) + .execute::(ctx) .unwrap() }); } From 3e6834eb68f3edca9922ee25e504ebb9587a080c Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Tue, 14 Apr 2026 11:52:42 -0400 Subject: [PATCH 036/250] fix ALP patch plugin (#7430) ## Summary Closes: #000 ## Testing Signed-off-by: Andrew Duffy --- encodings/alp/src/alp/plugin.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index f57f3d27bf5..020acb7e1ac 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -76,6 +76,10 @@ impl ArrayPlugin for ALPPatchedPlugin { Ok(patched.into_array()) } + + fn is_supported_encoding(&self, id: &ArrayId) -> bool { + id == &Patched.id() || id == &ALP.id() + } } #[cfg(test)] From 5330d32926e48fea938d16ee5b4308ebc37373b0 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 14 Apr 2026 18:00:17 +0100 Subject: [PATCH 037/250] add unstable encodings and patches to benchmarks (#7426) --- .github/scripts/run-sql-bench.sh | 3 +++ .github/workflows/bench-pr.yml | 4 +++- .github/workflows/bench.yml | 4 +++- .github/workflows/sql-benchmarks.yml | 2 +- benchmarks/compress-bench/Cargo.toml | 1 + benchmarks/datafusion-bench/Cargo.toml | 1 + benchmarks/duckdb-bench/Cargo.toml | 1 + benchmarks/lance-bench/Cargo.toml | 3 +++ benchmarks/random-access-bench/Cargo.toml | 1 + vortex-bench/Cargo.toml | 3 +++ 10 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-sql-bench.sh b/.github/scripts/run-sql-bench.sh index 96c2e2f19dc..bdbcae3f7a0 100755 --- a/.github/scripts/run-sql-bench.sh +++ b/.github/scripts/run-sql-bench.sh @@ -22,6 +22,9 @@ set -Eeu -o pipefail +export VORTEX_EXPERIMENTAL_PATCHED_ARRAY=1 +export FLAT_LAYOUT_INLINE_ARRAY_NODE=1 + subcommand="$1" targets="$2" shift 2 diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index f392f1a6e45..d3de66765dc 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -60,7 +60,7 @@ jobs: env: RUSTFLAGS: "-C target-cpu=native" run: | - cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} + cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false @@ -76,6 +76,8 @@ jobs: shell: bash env: RUST_BACKTRACE: full + VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 558a5bd51ff..abc80c1d93e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -74,7 +74,7 @@ jobs: env: RUSTFLAGS: "-C target-cpu=native" run: | - cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} + cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - name: Setup Polar Signals uses: polarsignals/gh-actions-ps-profiling@v0.8.1 @@ -89,6 +89,8 @@ jobs: shell: bash env: RUST_BACKTRACE: full + VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 4c114f00bda..c12a5fe7bea 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -144,7 +144,7 @@ jobs: if [ "${{ matrix.build_lance }}" = "true" ]; then packages="$packages --bin lance-bench" fi - cargo build $packages --profile release_debug + cargo build $packages --profile release_debug --features unstable_encodings - name: Generate data shell: bash diff --git a/benchmarks/compress-bench/Cargo.toml b/benchmarks/compress-bench/Cargo.toml index 4c72a20f712..baff1daad74 100644 --- a/benchmarks/compress-bench/Cargo.toml +++ b/benchmarks/compress-bench/Cargo.toml @@ -34,6 +34,7 @@ vortex-bench = { workspace = true } [features] lance = ["dep:lance-bench"] +unstable_encodings = ["vortex/unstable_encodings"] [lints] workspace = true diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index eea6e70e8aa..d0015dd2995 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -46,6 +46,7 @@ custom-labels = { workspace = true } [features] cuda = ["dep:vortex-cuda"] +unstable_encodings = ["vortex/unstable_encodings"] [lints] workspace = true diff --git a/benchmarks/duckdb-bench/Cargo.toml b/benchmarks/duckdb-bench/Cargo.toml index ad9ece90fc3..b70375cf963 100644 --- a/benchmarks/duckdb-bench/Cargo.toml +++ b/benchmarks/duckdb-bench/Cargo.toml @@ -27,6 +27,7 @@ vortex-duckdb = { workspace = true } [features] cuda = ["dep:vortex-cuda"] +unstable_encodings = ["vortex/unstable_encodings"] [lints] workspace = true diff --git a/benchmarks/lance-bench/Cargo.toml b/benchmarks/lance-bench/Cargo.toml index acfe39ef531..e5c6044e4f5 100644 --- a/benchmarks/lance-bench/Cargo.toml +++ b/benchmarks/lance-bench/Cargo.toml @@ -29,5 +29,8 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } vortex-bench = { workspace = true } +[features] +unstable_encodings = ["vortex-bench/unstable_encodings"] + [lints] workspace = true diff --git a/benchmarks/random-access-bench/Cargo.toml b/benchmarks/random-access-bench/Cargo.toml index c22736b65d4..cde8fb6f81a 100644 --- a/benchmarks/random-access-bench/Cargo.toml +++ b/benchmarks/random-access-bench/Cargo.toml @@ -26,6 +26,7 @@ vortex-bench = { workspace = true } [features] lance = ["dep:lance-bench"] +unstable_encodings = ["vortex-bench/unstable_encodings"] [lints] workspace = true diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index 62d302f12e1..8b2bb6efe25 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -63,3 +63,6 @@ vortex = { workspace = true, features = [ "tokio", "zstd", ] } + +[features] +unstable_encodings = ["vortex/unstable_encodings"] From 97b18fbd63f612539f7a254884ddd84fc76aa7f2 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 14 Apr 2026 18:26:49 +0100 Subject: [PATCH 038/250] Add benchmarks for take on a FilterArray (#7420) Before we make changes to this code we should make sure we know what we are improving --------- Signed-off-by: Robert Kruszewski --- vortex-array/Cargo.toml | 4 + vortex-array/benches/take_filter.rs | 140 ++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 vortex-array/benches/take_filter.rs diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index b1cb8fcdfcf..e324c70aadb 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -176,6 +176,10 @@ harness = false name = "take_fsl" harness = false +[[bench]] +name = "take_filter" +harness = false + [[bench]] name = "filter_bool" harness = false diff --git a/vortex-array/benches/take_filter.rs b/vortex-array/benches/take_filter.rs new file mode 100644 index 00000000000..cfe837d37bf --- /dev/null +++ b/vortex-array/benches/take_filter.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Benchmarks for taking from a lazy [`FilterArray`]. +//! +//! Parameterized over: +//! - Number of indices to take +//! - Number of rows retained by the filter +//! - Filter mask layout (single contiguous slice vs random positions) +//! - Take index layout (sequential vs random ranks) + +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] + +use divan::Bencher; +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand::seq::SliceRandom; +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::RecursiveCanonical; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::PrimitiveArray; +use vortex_buffer::Buffer; +use vortex_mask::Mask; + +fn main() { + divan::main(); +} + +const ARRAY_LEN: usize = 100_000; +const FILTERED_LENS: &[usize] = &[10_000, 50_000, 90_000]; +const NUM_INDICES: &[usize] = &[1_000, 10_000]; +const MASK_SEED: u64 = 42; +const INDEX_SEED: u64 = 43; + +fn primitive_array() -> ArrayRef { + PrimitiveArray::from_iter(0..ARRAY_LEN as u32).into_array() +} + +fn slice_mask(filtered_len: usize) -> Mask { + let start = (ARRAY_LEN - filtered_len) / 2; + Mask::from_slices(ARRAY_LEN, vec![(start, start + filtered_len)]) +} + +fn random_mask(filtered_len: usize) -> Mask { + let mut indices: Vec = (0..ARRAY_LEN).collect(); + indices.shuffle(&mut StdRng::seed_from_u64(MASK_SEED)); + indices.truncate(filtered_len); + indices.sort_unstable(); + Mask::from_indices(ARRAY_LEN, indices) +} + +fn sequential_indices(num_indices: usize) -> ArrayRef { + Buffer::from_iter(0..num_indices as u64).into_array() +} + +fn random_indices(num_indices: usize, filtered_len: usize) -> ArrayRef { + let mut rng = StdRng::seed_from_u64(INDEX_SEED); + Buffer::from_iter((0..num_indices).map(|_| rng.random_range(0..filtered_len as u64))) + .into_array() +} + +#[divan::bench(args = NUM_INDICES, consts = FILTERED_LENS)] +fn take_filter_slice_mask_sequential_indices( + bencher: Bencher, + num_indices: usize, +) { + let array = primitive_array().filter(slice_mask(FILTERED_LEN)).unwrap(); + let indices = sequential_indices(num_indices); + + bencher + .with_inputs(|| (&array, &indices, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(array, indices, ctx)| { + array + .take(indices.clone()) + .unwrap() + .execute::(ctx) + .unwrap() + }); +} + +#[divan::bench(args = NUM_INDICES, consts = FILTERED_LENS)] +fn take_filter_slice_mask_random_indices( + bencher: Bencher, + num_indices: usize, +) { + let array = primitive_array().filter(slice_mask(FILTERED_LEN)).unwrap(); + let indices = random_indices(num_indices, FILTERED_LEN); + + bencher + .with_inputs(|| (&array, &indices, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(array, indices, ctx)| { + array + .take(indices.clone()) + .unwrap() + .execute::(ctx) + .unwrap() + }); +} + +#[divan::bench(args = NUM_INDICES, consts = FILTERED_LENS)] +fn take_filter_random_mask_sequential_indices( + bencher: Bencher, + num_indices: usize, +) { + let array = primitive_array().filter(random_mask(FILTERED_LEN)).unwrap(); + let indices = sequential_indices(num_indices); + + bencher + .with_inputs(|| (&array, &indices, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(array, indices, ctx)| { + array + .take(indices.clone()) + .unwrap() + .execute::(ctx) + .unwrap() + }); +} + +#[divan::bench(args = NUM_INDICES, consts = FILTERED_LENS)] +fn take_filter_random_mask_random_indices( + bencher: Bencher, + num_indices: usize, +) { + let array = primitive_array().filter(random_mask(FILTERED_LEN)).unwrap(); + let indices = random_indices(num_indices, FILTERED_LEN); + + bencher + .with_inputs(|| (&array, &indices, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(array, indices, ctx)| { + array + .take(indices.clone()) + .unwrap() + .execute::(ctx) + .unwrap() + }); +} From 6d07a23ff18d95615dd4e040fe5501b1e0883b9d Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 14 Apr 2026 13:38:17 -0400 Subject: [PATCH 039/250] Optimize inner product, update boilerplate, remove bad benchmarks (#7428) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Optimizes inner product with manual partial sum decomposition (because the compiler can't optimize this because float addition is not associative). Also removes the old benchmarks as they no longer really time the correct thing anymore. The real benchmarks will be finished here: https://github.com/vortex-data/vortex/pull/7399. This change also adds the `vortex-tensor/src/vector_search.rs` file to support that soon. ## Testing N/A Signed-off-by: Connor Tsui --- vortex-tensor/Cargo.toml | 5 - vortex-tensor/benches/similarity_search.rs | 108 ------- .../benches/similarity_search_common/mod.rs | 256 --------------- vortex-tensor/public-api.lock | 8 + vortex-tensor/src/lib.rs | 2 + vortex-tensor/src/scalar_fns/inner_product.rs | 58 +++- vortex-tensor/src/vector_search.rs | 305 ++++++++++++++++++ vortex/benches/single_encoding_throughput.rs | 1 + 8 files changed, 362 insertions(+), 381 deletions(-) delete mode 100644 vortex-tensor/benches/similarity_search.rs delete mode 100644 vortex-tensor/benches/similarity_search_common/mod.rs create mode 100644 vortex-tensor/src/vector_search.rs diff --git a/vortex-tensor/Cargo.toml b/vortex-tensor/Cargo.toml index b14e82287bb..2f92ce5a107 100644 --- a/vortex-tensor/Cargo.toml +++ b/vortex-tensor/Cargo.toml @@ -37,8 +37,3 @@ rand = { workspace = true } rand_distr = { workspace = true } rstest = { workspace = true } vortex-btrblocks = { path = "../vortex-btrblocks" } - -[[bench]] -name = "similarity_search" -harness = false -test = false diff --git a/vortex-tensor/benches/similarity_search.rs b/vortex-tensor/benches/similarity_search.rs deleted file mode 100644 index 1112eda11b2..00000000000 --- a/vortex-tensor/benches/similarity_search.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! End-to-end similarity-search execution benchmark. -//! -//! For each of three compression strategies (uncompressed, default BtrBlocks, TurboQuant), this -//! bench: -//! -//! 1. Generates a deterministic random `Vector` batch. -//! 2. Applies the compression strategy *outside* the timed region. -//! 3. Builds the lazy -//! `Binary(Gt, [CosineSimilarity(data, query), threshold])` -//! tree *outside* the timed region. -//! 4. Times *only* `tree.execute::(&mut ctx)`. -//! -//! Run with: `cargo bench -p vortex-tensor --bench similarity_search` - -use divan::Bencher; -use mimalloc::MiMalloc; -use vortex_array::VortexSessionExecute; -use vortex_array::arrays::BoolArray; -use vortex_error::VortexExpect; - -#[path = "similarity_search_common/mod.rs"] -mod common; - -use common::Variant; -use common::build_similarity_search_tree; -use common::extract_row_as_query; -use common::generate_random_vectors; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -/// Number of vectors in the benchmark dataset. -const NUM_ROWS: usize = 100_000; - -/// Dimensionality of each vector. Must be `>= vortex_tensor::encodings::turboquant::MIN_DIMENSION` -/// (128) for the TurboQuant variant to work. -const DIM: u32 = 768; - -/// Deterministic PRNG seed for the generated dataset. -const SEED: u64 = 0xC0FFEE; - -/// Cosine similarity threshold for the "greater than" filter. Random f32 vectors from N(0, 1) at -/// this dimension have near-zero pairwise similarity, so picking a row of the dataset as the -/// query guarantees at least that row matches. -const THRESHOLD: f32 = 0.8; - -fn main() { - divan::main(); -} - -/// Runs one end-to-end execution of the similarity-search tree for the given variant. All dataset -/// generation and tree construction happens in the bench setup closure so only the execution of -/// the lazy tree is timed. -fn bench_variant(bencher: Bencher<'_, '_>, variant: Variant) { - bencher - .with_inputs(|| { - let mut ctx = common::SESSION.create_execution_ctx(); - - // Use row 0 of the uncompressed data as the query so we always have at least one - // match. Keeping the query extraction separate from the compressed-data build keeps - // the query identical across all three variants. - let raw = generate_random_vectors(NUM_ROWS, DIM, SEED); - let query = extract_row_as_query(&raw, 0, DIM); - let data = match variant { - Variant::Uncompressed => raw, - Variant::DefaultCompression => { - common::compress_default(raw).vortex_expect("default compression succeeds") - } - Variant::TurboQuant => common::compress_turboquant(raw, &mut ctx) - .vortex_expect("turboquant compression succeeds"), - }; - - // println!( - // "\n\n{}: {}\n\n", - // variant, - // data.display_tree_encodings_only() - // ); - - let tree = build_similarity_search_tree(data, &query, THRESHOLD) - .vortex_expect("tree construction succeeds"); - - (tree, ctx) - }) - .bench_values(|(tree, mut ctx)| { - // Hot path: only the .execute() call is timed. The result is a BoolArray of length - // NUM_ROWS with true at positions where cosine_similarity > THRESHOLD. - tree.execute::(&mut ctx) - .vortex_expect("similarity search tree executes to a BoolArray") - }); -} - -#[divan::bench] -fn execute_uncompressed(bencher: Bencher) { - bench_variant(bencher, Variant::Uncompressed); -} - -#[divan::bench] -fn execute_default_compression(bencher: Bencher) { - bench_variant(bencher, Variant::DefaultCompression); -} - -#[divan::bench] -fn execute_turboquant(bencher: Bencher) { - bench_variant(bencher, Variant::TurboQuant); -} diff --git a/vortex-tensor/benches/similarity_search_common/mod.rs b/vortex-tensor/benches/similarity_search_common/mod.rs deleted file mode 100644 index c22cb5a9f08..00000000000 --- a/vortex-tensor/benches/similarity_search_common/mod.rs +++ /dev/null @@ -1,256 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Shared helpers for the similarity-search benchmark and example. -//! -//! This module is included from both `vortex-tensor/benches/similarity_search.rs` and -//! `vortex-tensor/examples/similarity_search.rs` via an explicit `#[path = ...]` so both targets -//! use the exact same array-tree builder. -//! -//! The three main entry points are: -//! -//! - [`generate_random_vectors`] to build a deterministic random [`Vector`] extension array. -//! - [`build_variant`] to take a raw vector array and apply the requested compression strategy -//! (uncompressed, default BtrBlocks, or TurboQuant). -//! - [`build_similarity_search_tree`] to wire a cosine-similarity + threshold expression on top of -//! a prepared data array and a single-row query vector. -//! -//! [`Vector`]: vortex_tensor::vector::Vector - -#![allow(dead_code)] - -use std::fmt; -use std::sync::LazyLock; - -use rand::SeedableRng; -use rand::rngs::StdRng; -use rand_distr::Distribution; -use rand_distr::Normal; -use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; -use vortex_array::VortexSessionExecute; -use vortex_array::arrays::ConstantArray; -use vortex_array::arrays::Extension; -use vortex_array::arrays::ExtensionArray; -use vortex_array::arrays::FixedSizeListArray; -use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::extension::ExtensionArrayExt; -use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -use vortex_array::builtins::ArrayBuiltins; -use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; -use vortex_array::dtype::extension::ExtDType; -use vortex_array::extension::EmptyMetadata; -use vortex_array::scalar::Scalar; -use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; -use vortex_array::validity::Validity; -use vortex_btrblocks::BtrBlocksCompressor; -use vortex_buffer::BufferMut; -use vortex_error::VortexExpect; -use vortex_error::VortexResult; -use vortex_error::vortex_panic; -use vortex_session::VortexSession; -use vortex_tensor::encodings::turboquant::TurboQuantConfig; -use vortex_tensor::encodings::turboquant::turboquant_encode_unchecked; -use vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity; -use vortex_tensor::scalar_fns::l2_denorm::L2Denorm; -use vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm; -use vortex_tensor::vector::Vector; - -/// A shared [`VortexSession`] pre-loaded with the builtin [`ArraySession`] so both bench and -/// example can create execution contexts cheaply. -pub static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - -/// The three compression strategies the benchmark and example exercise. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Variant { - /// Raw `Vector` with no compression applied. - Uncompressed, - /// `BtrBlocksCompressor::default()` walks into the extension array and compresses the - /// underlying FSL storage child with the default scheme set (no TurboQuant). - DefaultCompression, - /// TurboQuant: normalize, quantize to `FSL(Dict)`, wrap in SORF + `L2Denorm`. - TurboQuant, -} - -impl fmt::Display for Variant { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Uncompressed => f.write_str("Uncompressed"), - Self::DefaultCompression => f.write_str("DefaultCompression"), - Self::TurboQuant => f.write_str("TurboQuant"), - } - } -} - -/// Generate `num_rows` random f32 vectors of dimension `dim`, wrapped in a [`Vector`] extension -/// array. The values are drawn from a standard normal distribution seeded by `seed` so results -/// are reproducible across runs. -/// -/// [`Vector`]: vortex_tensor::vector::Vector -pub fn generate_random_vectors(num_rows: usize, dim: u32, seed: u64) -> ArrayRef { - let mut rng = StdRng::seed_from_u64(seed); - // `Normal::new(0, 1)` is infallible for these parameters. `rand_distr::NormalError` does - // not implement `Into`, so we cannot use `vortex_expect` here; fall back to - // `vortex_panic!` on the (impossible) error path instead. - let normal = - Normal::new(0.0f32, 1.0).unwrap_or_else(|_| vortex_panic!("Normal(0, 1) is well-defined")); - - let dim_usize = dim as usize; - let mut buf = BufferMut::::with_capacity(num_rows * dim_usize); - for _ in 0..(num_rows * dim_usize) { - buf.push(normal.sample(&mut rng)); - } - - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = - FixedSizeListArray::try_new(elements.into_array(), dim, Validity::NonNullable, num_rows) - .vortex_expect("FSL with valid shape and matching children length"); - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) - .vortex_expect("Vector extension dtype is valid for an f32 FSL") - .erased(); - ExtensionArray::new(ext_dtype, fsl.into_array()).into_array() -} - -/// Pull the `row`-th vector out of a `Vector` extension array as a plain `Vec`. -/// -/// Used to extract a single query vector from a batch of generated data. The input must already -/// be fully materialized (no lazy scalar-fn wrappers); pass a raw array from -/// [`generate_random_vectors`], not a compressed variant. -pub fn extract_row_as_query(vectors: &ArrayRef, row: usize, dim: u32) -> Vec { - let ext = vectors - .as_opt::() - .vortex_expect("data must be a Vector extension array"); - - let mut ctx = SESSION.create_execution_ctx(); - let fsl: FixedSizeListArray = ext - .storage_array() - .clone() - .execute(&mut ctx) - .vortex_expect("storage array executes to an FSL"); - let elements: PrimitiveArray = fsl - .elements() - .clone() - .execute(&mut ctx) - .vortex_expect("FSL elements execute to a PrimitiveArray"); - - let slice = elements.as_slice::(); - let dim_usize = dim as usize; - let start = row * dim_usize; - slice[start..start + dim_usize].to_vec() -} - -/// Build a `Vector` extension array whose storage is a [`ConstantArray`] broadcasting a -/// single query vector across `num_rows` rows. This is how we hand a single query vector to -/// `CosineSimilarity` on the `rhs` side -- `ScalarFnArray` requires both children to have the -/// same length, so we broadcast the query instead of hand-rolling a 1-row input. -fn build_constant_query_vector(query: &[f32], num_rows: usize) -> VortexResult { - let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - - let children: Vec = query - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - - let storage = ConstantArray::new(storage_scalar, num_rows).into_array(); - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, storage).into_array()) -} - -/// Compresses a raw `Vector` array with the default BtrBlocks pipeline. -/// -/// [`BtrBlocksCompressor`] walks into the extension array and recursively compresses the -/// underlying FSL storage child. TurboQuant is *not* exercised by this path -- it is not -/// registered in the default scheme set -- so this measures "generic" lossless compression -/// applied to float vectors. -pub fn compress_default(data: ArrayRef) -> VortexResult { - BtrBlocksCompressor::default().compress(&data) -} - -/// Compresses a raw `Vector` array with the TurboQuant pipeline by hand, producing the -/// same tree shape that -/// [`vortex_tensor::encodings::turboquant::TurboQuantScheme`] would: -/// -/// ```text -/// L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms) -/// ``` -/// -/// Calling the encode helpers directly (instead of going through -/// `BtrBlocksCompressorBuilder::with_turboquant()`) lets this example avoid depending on the -/// `unstable_encodings` feature flag. -/// -/// See `vortex-tensor/src/encodings/turboquant/tests/mod.rs::normalize_and_encode` for the same -/// canonical recipe. -pub fn compress_turboquant(data: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - let l2_denorm = normalize_as_l2_denorm(data, ctx)?; - let normalized = l2_denorm.child_at(0).clone(); - let norms = l2_denorm.child_at(1).clone(); - let num_rows = l2_denorm.len(); - - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - - let config = TurboQuantConfig::default(); - // SAFETY: `normalize_as_l2_denorm` guarantees every row is unit-norm (or zero), which is the - // invariant `turboquant_encode_unchecked` expects. - let tq = unsafe { turboquant_encode_unchecked(normalized_ext, &config, ctx) }?; - - Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) -} - -/// Dispatch helper that builds the data array for the requested [`Variant`], starting from a -/// single random-vector generation. Always returns an `ArrayRef` whose logical dtype is -/// `Vector`. -pub fn build_variant( - variant: Variant, - num_rows: usize, - dim: u32, - seed: u64, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let raw = generate_random_vectors(num_rows, dim, seed); - match variant { - Variant::Uncompressed => Ok(raw), - Variant::DefaultCompression => compress_default(raw), - Variant::TurboQuant => compress_turboquant(raw, ctx), - } -} - -/// Build the lazy similarity-search array tree for a prepared data array and a single query -/// vector. The returned tree is a boolean array of length `data.len()` where position `i` is -/// `true` iff `cosine_similarity(data[i], query) > threshold`. -/// -/// The tree shape is: -/// -/// ```text -/// Binary(Gt, [ -/// CosineSimilarity([data, ConstantArray(query_vec, n)]), -/// ConstantArray(threshold, n), -/// ]) -/// ``` -/// -/// This function does no execution; it is safe to call inside a benchmark setup closure. -pub fn build_similarity_search_tree( - data: ArrayRef, - query: &[f32], - threshold: f32, -) -> VortexResult { - let num_rows = data.len(); - let query_vec = build_constant_query_vector(query, num_rows)?; - - let cosine = CosineSimilarity::try_new_array(data, query_vec, num_rows)?.into_array(); - - let threshold_scalar = Scalar::primitive(threshold, Nullability::NonNullable); - let threshold_array = ConstantArray::new(threshold_scalar, num_rows).into_array(); - - cosine.binary(threshold_array, Operator::Gt) -} diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index bec8df1cb29..4f6c2dddbfb 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -550,4 +550,12 @@ impl core::marker::Copy for vortex_tensor::vector::VectorMatcherMetadata impl core::marker::StructuralPartialEq for vortex_tensor::vector::VectorMatcherMetadata +pub mod vortex_tensor::vector_search + +pub fn vortex_tensor::vector_search::build_constant_query_vector>(query: &[T], num_rows: usize) -> vortex_error::VortexResult + +pub fn vortex_tensor::vector_search::build_similarity_search_tree>(data: vortex_array::array::erased::ArrayRef, query: &[T], threshold: T) -> vortex_error::VortexResult + +pub fn vortex_tensor::vector_search::compress_turboquant(data: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_tensor::initialize(session: &vortex_session::VortexSession) diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index 3d3563aa8e4..b3cf6c21695 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -25,6 +25,8 @@ pub mod vector; pub mod encodings; +pub mod vector_search; + mod utils; /// Initialize the Vortex tensor library with a Vortex session. diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index c883cf00982..0d6b2bf76aa 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -527,18 +527,9 @@ impl InnerProduct { let values: &[f32] = values_prim.as_slice::(); debug_assert_eq!(codes.len(), len * padded_dim); - // Direct codebook lookup in the hot loop. See the function doc comment for why this - // beats an explicit product table here. - let mut out = BufferMut::::with_capacity(len); - for row in 0..len { - let row_codes = &codes[row * padded_dim..(row + 1) * padded_dim]; - let mut acc = 0.0f32; - for j in 0..padded_dim { - acc += q[j] * values[row_codes[j] as usize]; - } - // SAFETY: we reserved `len` slots above and push exactly once per row. - unsafe { out.push_unchecked(acc) }; - } + // The hot loop is extracted into [`execute_dict_constant_inner_product`] with + // unchecked indexing so the compiler can vectorize the inner gather-accumulate. + let out = execute_dict_constant_inner_product(q, values, codes, len, padded_dim); // SAFETY: the buffer length equals `len`, which matches the validity length. let result = unsafe { PrimitiveArray::new_unchecked(out.freeze(), validity) }.into_array(); @@ -556,6 +547,49 @@ fn inner_product_row(a: &[T], b: &[T]) -> T { .fold(T::zero(), |acc, v| acc + v) } +/// Compute inner products between a constant query vector and dictionary-encoded rows. +/// +/// For each row, computes `sum(q[j] * values[codes[row * dim + j]])` using the codebook +/// `values` directly instead of decoding the dictionary into dense vectors. +/// +/// The inner loop uses four independent accumulators so the CPU can pipeline FP additions +/// instead of waiting for each `fadd` to retire before starting the next. +fn execute_dict_constant_inner_product( + q: &[f32], + values: &[f32], + codes: &[u8], + num_rows: usize, + dim: usize, +) -> BufferMut { + let mut out = BufferMut::::with_capacity(num_rows); + + const PARTIAL_SUMS: usize = 8; + + for row_codes in codes.chunks_exact(dim) { + let mut acc = [0.0f32; PARTIAL_SUMS]; + + let code_chunks = row_codes.chunks_exact(PARTIAL_SUMS); + let q_chunks = q.chunks_exact(PARTIAL_SUMS); + let code_rem = code_chunks.remainder(); + let q_rem = q_chunks.remainder(); + + for (cc, qd) in code_chunks.zip(q_chunks) { + for i in 0..PARTIAL_SUMS { + acc[i] = qd[i].mul_add(values[cc[i] as usize], acc[i]); + } + } + + for (&code, &q_val) in code_rem.iter().zip(q_rem.iter()) { + acc[0] = q_val.mul_add(values[code as usize], acc[0]); + } + + // SAFETY: we reserved `num_rows` slots and push exactly once per row. + unsafe { out.push_unchecked(acc.iter().sum::()) }; + } + + out +} + #[cfg(test)] mod tests { use std::sync::LazyLock; diff --git a/vortex-tensor/src/vector_search.rs b/vortex-tensor/src/vector_search.rs new file mode 100644 index 00000000000..81a379683db --- /dev/null +++ b/vortex-tensor/src/vector_search.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Reusable helpers for building brute-force vector similarity search expressions over +//! [`Vector`] extension arrays. +//! +//! This module exposes three small building blocks that together make it straightforward to +//! stand up a cosine-similarity-plus-threshold scan on top of a prepared data array: +//! +//! - [`compress_turboquant`] applies the canonical TurboQuant encoding pipeline +//! (`L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms)`) to a raw +//! `Vector` array without requiring the caller to plumb the +//! `unstable_encodings` feature flag on the `vortex` facade. +//! - [`build_constant_query_vector`] wraps a single query vector into a +//! [`Vector`] extension array whose storage is a [`ConstantArray`] broadcast +//! across `num_rows` rows. This is the shape expected by +//! [`CosineSimilarity::try_new_array`] for the RHS of a database-vs-query scan. +//! - [`build_similarity_search_tree`] wires everything together into a lazy +//! `Binary(Gt, [CosineSimilarity(data, query), threshold])` expression. +//! +//! Executing the tree from [`build_similarity_search_tree`] into a +//! [`BoolArray`](vortex_array::arrays::BoolArray) yields one boolean per row indicating whether +//! that row's cosine similarity to the query exceeds `threshold`. +//! +//! # Example +//! +//! ```ignore +//! use vortex_array::{ArrayRef, VortexSessionExecute}; +//! use vortex_array::arrays::BoolArray; +//! use vortex_session::VortexSession; +//! use vortex_tensor::vector_search::{build_similarity_search_tree, compress_turboquant}; +//! +//! fn run(session: &VortexSession, data: ArrayRef, query: &[f32]) -> anyhow::Result<()> { +//! let mut ctx = session.create_execution_ctx(); +//! let data = compress_turboquant(data, &mut ctx)?; +//! let tree = build_similarity_search_tree(data, query, 0.8)?; +//! let _matches: BoolArray = tree.execute(&mut ctx)?; +//! Ok(()) +//! } +//! ``` +//! +//! [`Vector`]: crate::vector::Vector +//! [`CosineSimilarity::try_new_array`]: crate::scalar_fns::cosine_similarity::CosineSimilarity::try_new_array + +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::Extension; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::extension::EmptyMetadata; +use vortex_array::scalar::PValue; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; + +use crate::encodings::turboquant::TurboQuantConfig; +use crate::encodings::turboquant::turboquant_encode_unchecked; +use crate::scalar_fns::cosine_similarity::CosineSimilarity; +use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use crate::vector::Vector; + +/// Apply the canonical TurboQuant encoding pipeline to a `Vector` array. +/// +/// The returned array has the shape +/// `L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms)` — exactly what +/// [`crate::encodings::turboquant::TurboQuantScheme`] produces when invoked through +/// `BtrBlocksCompressorBuilder::with_turboquant()`, but without requiring callers to enable +/// the `unstable_encodings` feature on the `vortex` facade. +/// +/// The input `data` must be a [`Vector`] extension array whose element type is `f32` and whose +/// dimensionality is at least +/// [`turboquant::MIN_DIMENSION`](crate::encodings::turboquant::MIN_DIMENSION). The TurboQuant +/// configuration used is [`TurboQuantConfig::default()`] (8-bit codes, 3 SORF rounds, seed 42). +/// +/// # Errors +/// +/// Returns an error if `data` is not a [`Vector`] extension array, if normalization fails, or +/// if the underlying TurboQuant encoder rejects the input shape. +pub fn compress_turboquant(data: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + let l2_denorm = normalize_as_l2_denorm(data, ctx)?; + let normalized = l2_denorm.child_at(0).clone(); + let norms = l2_denorm.child_at(1).clone(); + let num_rows = l2_denorm.len(); + + let Some(normalized_ext) = normalized.as_opt::() else { + vortex_bail!("normalize_as_l2_denorm must produce an Extension array child"); + }; + + let config = TurboQuantConfig::default(); + // SAFETY: `normalize_as_l2_denorm` guarantees every row is unit-norm (or zero), which is + // the invariant `turboquant_encode_unchecked` expects. + let tq = unsafe { turboquant_encode_unchecked(normalized_ext, &config, ctx) }?; + + Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) +} + +/// Build a [`Vector`] extension array whose storage is a [`ConstantArray`] broadcasting a single +/// query vector across `num_rows` rows. +/// +/// The element type is inferred from `T` (e.g. `f32` or `f64`). This is the shape expected for +/// the RHS of a database-vs-query [`CosineSimilarity`] scan: the `ScalarFnArray` contract +/// requires both children to have the same length, so rather than hand-rolling a 1-row input we +/// broadcast the query across the whole database. +/// +/// # Errors +/// +/// Returns an error if the [`Vector`] extension dtype rejects the constructed storage dtype. +pub fn build_constant_query_vector>( + query: &[T], + num_rows: usize, +) -> VortexResult { + let element_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable); + + let children: Vec = query + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + + let storage = ConstantArray::new(storage_scalar, num_rows).into_array(); + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, storage).into_array()) +} + +/// Build the lazy similarity-search expression tree for a prepared database array and a +/// single query vector. +/// +/// The returned array is a lazy boolean expression of length `data.len()` whose position `i` +/// is `true` iff `cosine_similarity(data[i], query) > threshold`. Executing it into a +/// [`BoolArray`](vortex_array::arrays::BoolArray) runs the full scan. +/// +/// The tree shape is: +/// +/// ```text +/// Binary(Gt, [ +/// CosineSimilarity([data, ConstantArray(query_vec, n)]), +/// ConstantArray(threshold, n), +/// ]) +/// ``` +/// +/// The element type is inferred from `T` and must match the element type of `data`'s +/// [`Vector`] extension dtype. +/// +/// This function performs no execution; it is safe to call inside a benchmark setup closure. +/// +/// # Errors +/// +/// Returns an error if `query` has a length incompatible with `data`'s vector dimension, or +/// if any of the intermediate array constructors fails. +pub fn build_similarity_search_tree>( + data: ArrayRef, + query: &[T], + threshold: T, +) -> VortexResult { + let num_rows = data.len(); + let query_vec = build_constant_query_vector(query, num_rows)?; + + let cosine = CosineSimilarity::try_new_array(data, query_vec, num_rows)?.into_array(); + + let threshold_scalar = Scalar::primitive(threshold, Nullability::NonNullable); + let threshold_array = ConstantArray::new(threshold_scalar, num_rows).into_array(); + + cosine.binary(threshold_array, Operator::Gt) +} + +#[cfg(test)] +mod tests { + use vortex_array::ArrayRef; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::Extension; + use vortex_array::arrays::ExtensionArray; + use vortex_array::arrays::FixedSizeListArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::bool::BoolArrayExt; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::extension::EmptyMetadata; + use vortex_array::session::ArraySession; + use vortex_array::validity::Validity; + use vortex_buffer::BufferMut; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + + use super::build_constant_query_vector; + use super::build_similarity_search_tree; + use super::compress_turboquant; + use crate::vector::Vector; + + /// Build a `Vector` extension array from a flat f32 slice. Each contiguous + /// group of `DIM` values becomes one row. + fn vector_array(dim: u32, values: &[f32]) -> VortexResult { + let dim_usize = dim as usize; + assert_eq!(values.len() % dim_usize, 0); + let num_rows = values.len() / dim_usize; + + let mut buf = BufferMut::::with_capacity(values.len()); + for &v in values { + buf.push(v); + } + let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim, + Validity::NonNullable, + num_rows, + )?; + + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + } + + fn test_session() -> VortexSession { + VortexSession::empty().with::() + } + + #[test] + fn constant_query_vector_has_vector_extension_dtype() -> VortexResult<()> { + let query = vec![1.0f32, 0.0, 0.0, 0.0]; + let rhs = build_constant_query_vector(&query, 5)?; + + assert_eq!(rhs.len(), 5); + assert!(rhs.as_opt::().is_some()); + Ok(()) + } + + #[test] + fn similarity_search_tree_executes_to_bool_array() -> VortexResult<()> { + // 4 rows of 3-dim vectors; the first and last match the query [1, 0, 0]. + let data = vector_array( + 3, + &[ + 1.0, 0.0, 0.0, // + 0.0, 1.0, 0.0, // + 0.0, 0.0, 1.0, // + 1.0, 0.0, 0.0, // + ], + )?; + let query = [1.0f32, 0.0, 0.0]; + + let tree = build_similarity_search_tree(data, &query, 0.5)?; + let mut ctx = test_session().create_execution_ctx(); + let result: BoolArray = tree.execute(&mut ctx)?; + + let bits = result.to_bit_buffer(); + assert_eq!(bits.len(), 4); + assert!(bits.value(0)); + assert!(!bits.value(1)); + assert!(!bits.value(2)); + assert!(bits.value(3)); + Ok(()) + } + + #[test] + fn turboquant_roundtrip_preserves_ranking() -> VortexResult<()> { + // Build 6 rows of 128-dim vectors where row 0 is highly correlated with the query. + // TurboQuant should preserve the "row 0 is the best match" ordering. + const DIM: u32 = 128; + const NUM_ROWS: usize = 6; + + let mut values = Vec::::with_capacity(NUM_ROWS * DIM as usize); + let query: Vec = (0..DIM as usize) + .map(|i| ((i as f32) * 0.017).sin()) + .collect(); + + // Row 0: identical to query (cosine=1.0) + values.extend_from_slice(&query); + // Row 1: query + noise + for (i, q) in query.iter().enumerate() { + values.push(q + 0.05 * ((i as f32) * 0.03).cos()); + } + // Rows 2..6: unrelated patterns + for row in 2..NUM_ROWS { + for i in 0..DIM as usize { + values.push(((row as f32 * 1.3 + i as f32) * 0.07).sin()); + } + } + + let data = vector_array(DIM, &values)?; + let mut ctx = test_session().create_execution_ctx(); + let compressed = compress_turboquant(data, &mut ctx)?; + assert_eq!(compressed.len(), NUM_ROWS); + + // Build a tree with a low threshold so row 0 (cosine=1.0 exact) matches. + let tree = build_similarity_search_tree(compressed, &query, 0.95)?; + let result: BoolArray = tree.execute(&mut ctx)?; + let bits = result.to_bit_buffer(); + assert_eq!(bits.len(), NUM_ROWS); + assert!( + bits.value(0), + "row 0 (identical to query) must match at threshold 0.95 even after TurboQuant" + ); + Ok(()) + } +} diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index e82f5fdd87a..4cedf23c07d 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -430,6 +430,7 @@ fn bench_zstd_decompress_string(bencher: Bencher) { .bench_refs(|a| a.to_canonical()); } +// TODO(connor): Remove this. // TurboQuant vector quantization benchmarks. #[cfg(feature = "unstable_encodings")] mod turboquant_benches { From 646e6be11e0c8388f7e00e504a2a9a97d7446c5c Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 14 Apr 2026 13:56:39 -0400 Subject: [PATCH 040/250] ScalarFns as Arrays (#7361) Provide a utility for serializing scalar functions as arrays --------- Signed-off-by: Nicholas Gates --- vortex-array/public-api.lock | 112 ++++++++++-------- vortex-array/src/arrays/scalar_fn/array.rs | 2 +- vortex-array/src/arrays/scalar_fn/mod.rs | 2 +- vortex-array/src/arrays/scalar_fn/plugin.rs | 92 ++++++++++++++ .../src/arrays/scalar_fn/vtable/mod.rs | 12 +- 5 files changed, 163 insertions(+), 57 deletions(-) create mode 100644 vortex-array/src/arrays/scalar_fn/plugin.rs diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 1f812c3e7a0..14f61b213d2 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3878,6 +3878,34 @@ pub type vortex_array::arrays::primitive::PrimitiveArray = vortex_array::Array + +pub vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts::children: alloc::vec::Vec + +pub vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts::options: ::Options + +pub struct vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin(_) + +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::new(vtable: V) -> Self + +impl vortex_array::ArrayPlugin for vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + +pub trait vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable: vortex_array::scalar_fn::ScalarFnVTable + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable::serialize(&self, view: &vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, Self>, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + pub struct vortex_array::arrays::scalar_fn::AnyScalarFn impl core::fmt::Debug for vortex_array::arrays::scalar_fn::AnyScalarFn @@ -3922,34 +3950,6 @@ pub type vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::Target = vor pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::deref(&self) -> &Self::Target -pub struct vortex_array::arrays::scalar_fn::ScalarFnData - -impl vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::build(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef - -impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnData - -impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::fmt::Display for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::ArrayEq for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool - -impl vortex_array::ArrayHash for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) - pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3966,7 +3966,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -4002,7 +4002,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6400,7 +6400,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6436,7 +6436,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -19426,6 +19426,14 @@ pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result>, vortex_error::VortexError> +impl vortex_array::ArrayPlugin for vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + core::fmt::Display + vortex_array::ArrayHash + vortex_array::ArrayEq @@ -20228,7 +20236,7 @@ pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_arr impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -20264,7 +20272,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -21200,7 +21208,7 @@ pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_arr impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21236,7 +21244,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -22888,10 +22896,6 @@ impl vortex_array::ArrayEq for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool -impl vortex_array::ArrayEq for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_eq(&self, other: &Self, _precision: vortex_array::Precision) -> bool - impl vortex_array::ArrayEq for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool @@ -23000,10 +23004,6 @@ impl vortex_array::ArrayHash for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) -impl vortex_array::ArrayHash for vortex_array::arrays::scalar_fn::ScalarFnData - -pub fn vortex_array::arrays::scalar_fn::ScalarFnData::array_hash(&self, state: &mut H, _precision: vortex_array::Precision) - impl vortex_array::ArrayHash for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) @@ -23088,6 +23088,22 @@ pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_sess pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result>, vortex_error::VortexError> +impl vortex_array::ArrayPlugin for vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin + +impl vortex_array::ArrayPlugin for vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + pub trait vortex_array::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + core::fmt::Display + vortex_array::ArrayHash + vortex_array::ArrayEq @@ -23890,7 +23906,7 @@ pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_arr impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -23926,7 +23942,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -25110,7 +25126,7 @@ pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_arr impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -25146,7 +25162,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 6e568f7aa08..d7d0118d02e 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -92,7 +92,7 @@ impl Array { let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); let dtype = scalar_fn.return_dtype(&arg_dtypes)?; let data = ScalarFnData::build(scalar_fn.clone(), children.clone(), len)?; - let vtable = ScalarFnVTable { scalar_fn }; + let vtable = ScalarFnVTable { id: scalar_fn.id() }; Ok(unsafe { Array::from_parts_unchecked( ArrayParts::new(vtable, dtype, len, data) diff --git a/vortex-array/src/arrays/scalar_fn/mod.rs b/vortex-array/src/arrays/scalar_fn/mod.rs index 9d2e9e66ed8..6c508dcad96 100644 --- a/vortex-array/src/arrays/scalar_fn/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/mod.rs @@ -2,10 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub mod plugin; mod rules; mod vtable; pub use array::ScalarFnArrayExt; -pub use array::ScalarFnData; pub use vtable::ScalarFnFactoryExt; pub use vtable::*; diff --git a/vortex-array/src/arrays/scalar_fn/plugin.rs b/vortex-array/src/arrays/scalar_fn/plugin.rs new file mode 100644 index 00000000000..1fb656323d9 --- /dev/null +++ b/vortex-array/src/arrays/scalar_fn/plugin.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +use crate::ArrayId; +use crate::ArrayPlugin; +use crate::ArrayRef; +use crate::IntoArray; +use crate::arrays::ScalarFnArray; +use crate::arrays::scalar_fn::ExactScalarFn; +use crate::arrays::scalar_fn::ScalarFnArrayView; +use crate::buffer::BufferHandle; +use crate::dtype::DType; +use crate::scalar_fn::ScalarFn; +use crate::scalar_fn::ScalarFnVTable; +use crate::serde::ArrayChildren; + +/// An adapter for enabling a scalar function to be serialized as an array. +pub struct ScalarFnArrayPlugin(V); + +impl ScalarFnArrayPlugin { + /// Create a new plugin for the given scalar function vtable. + pub fn new(vtable: V) -> Self { + Self(vtable) + } +} + +pub trait ScalarFnArrayVTable: ScalarFnVTable { + /// Serialize metadata for storing the scalar function as an array. + /// + /// Notably, this metadata needs enough information to reconstruct the child DTypes, as well + /// as the scalar function's own options. + fn serialize( + &self, + view: &ScalarFnArrayView, + session: &VortexSession, + ) -> VortexResult>>; + + /// Deserialize a scalar function array from its serialized components. + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult>; +} + +/// The parts used to construct a ScalarFnArray. +pub struct ScalarFnArrayParts { + pub options: V::Options, + pub children: Vec, +} + +impl ArrayPlugin for ScalarFnArrayPlugin { + fn id(&self) -> ArrayId { + self.0.id() + } + + fn serialize( + &self, + array: &ArrayRef, + session: &VortexSession, + ) -> VortexResult>> { + // We serialize the scalar function options, along with any scalar function array data. + let scalar_fn = array.as_::>(); + ::serialize(&self.0, &scalar_fn, session) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult { + let parts = ::deserialize( + &self.0, dtype, len, metadata, children, session, + )?; + Ok(ScalarFnArray::try_new( + ScalarFn::new(self.0.clone(), parts.options).erased(), + parts.children, + len, + )? + .into_array()) + } +} diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 996fbee1046..c663ddfc64e 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -41,7 +41,6 @@ use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; use crate::scalar_fn::ExecutionArgs; use crate::scalar_fn::ScalarFnId; -use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::VecExecutionArgs; use crate::serde::ArrayChildren; @@ -51,7 +50,7 @@ pub type ScalarFnArray = Array; #[derive(Clone, Debug)] pub struct ScalarFnVTable { - pub(super) scalar_fn: ScalarFnRef, + pub(super) id: ScalarFnId, } impl ArrayHash for ScalarFnData { @@ -72,7 +71,7 @@ impl VTable for ScalarFnVTable { type ValidityVTable = Self; fn id(&self) -> ArrayId { - self.scalar_fn.id() + self.id.clone() } fn validate( @@ -83,7 +82,7 @@ impl VTable for ScalarFnVTable { slots: &[Option], ) -> VortexResult<()> { vortex_ensure!( - data.scalar_fn == self.scalar_fn, + data.scalar_fn.id() == self.id, "ScalarFnArray data scalar_fn does not match vtable" ); vortex_ensure!( @@ -97,7 +96,7 @@ impl VTable for ScalarFnVTable { .map(|c| c.dtype().clone()) .collect_vec(); vortex_ensure!( - self.scalar_fn.return_dtype(&child_dtypes)? == *dtype, + data.scalar_fn.return_dtype(&child_dtypes)? == *dtype, "ScalarFnArray dtype does not match scalar function return dtype" ); Ok(()) @@ -128,7 +127,6 @@ impl VTable for ScalarFnVTable { _dtype: &DType, _len: usize, _metadata: &[u8], - _buffers: &[BufferHandle], _children: &dyn ArrayChildren, _session: &VortexSession, @@ -189,7 +187,7 @@ pub trait ScalarFnFactoryExt: scalar_fn::ScalarFnVTable { let data = ScalarFnData { scalar_fn: scalar_fn.clone(), }; - let vtable = ScalarFnVTable { scalar_fn }; + let vtable = ScalarFnVTable { id: scalar_fn.id() }; Ok(unsafe { Array::from_parts_unchecked( ArrayParts::new(vtable, dtype, len, data) From 59c92abf10035539080c1fa6019d50ea8c588b90 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:26:53 -0400 Subject: [PATCH 041/250] Fix lockfiles (#7433) ## Summary Closes: #000 ## Testing Signed-off-by: Connor Tsui --- vortex-array/public-api.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 14f61b213d2..d79cc795aa1 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3898,6 +3898,8 @@ pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserial pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> pub trait vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable: vortex_array::scalar_fn::ScalarFnVTable @@ -19432,6 +19434,8 @@ pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::deserial pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug @@ -23100,6 +23104,10 @@ pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + +pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool + pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> From 8b587bb56853b370e5058d9d3fea61e325967a9a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 14 Apr 2026 20:08:45 +0100 Subject: [PATCH 042/250] perf: intern identifiers (array, layout, etc). (#7412) We use id for scalar_fns, array and layouts. This we used to match rules among other things. This PR replace Arc ID with interned str (u32) id. --------- Signed-off-by: Nicholas Gates Signed-off-by: Joe Isaacs --- Cargo.lock | 41 ++-- Cargo.toml | 1 + encodings/alp/public-api.lock | 6 - encodings/alp/src/alp/array.rs | 8 +- encodings/alp/src/alp/plugin.rs | 16 +- encodings/alp/src/alp_rd/array.rs | 6 +- encodings/alp/src/lib.rs | 3 +- encodings/bytebool/public-api.lock | 2 - encodings/bytebool/src/array.rs | 6 +- encodings/datetime-parts/public-api.lock | 2 - encodings/datetime-parts/src/array.rs | 6 +- encodings/datetime-parts/src/lib.rs | 3 +- encodings/decimal-byte-parts/public-api.lock | 2 - .../src/decimal_byte_parts/mod.rs | 6 +- encodings/decimal-byte-parts/src/lib.rs | 3 +- encodings/fastlanes/public-api.lock | 8 - encodings/fastlanes/src/bitpacking/plugin.rs | 17 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 6 +- encodings/fastlanes/src/delta/vtable/mod.rs | 6 +- encodings/fastlanes/src/for/vtable/mod.rs | 6 +- encodings/fastlanes/src/lib.rs | 7 +- encodings/fastlanes/src/rle/vtable/mod.rs | 6 +- encodings/fsst/public-api.lock | 2 - encodings/fsst/src/array.rs | 6 +- encodings/parquet-variant/src/vtable.rs | 8 +- encodings/pco/public-api.lock | 2 - encodings/pco/src/array.rs | 6 +- encodings/runend/public-api.lock | 2 - encodings/runend/src/array.rs | 6 +- encodings/runend/src/lib.rs | 7 +- encodings/sequence/public-api.lock | 2 - encodings/sequence/src/array.rs | 6 +- encodings/sequence/src/lib.rs | 5 +- encodings/sparse/public-api.lock | 2 - encodings/sparse/src/lib.rs | 6 +- encodings/zigzag/public-api.lock | 2 - encodings/zigzag/src/array.rs | 6 +- encodings/zstd/public-api.lock | 2 - encodings/zstd/src/array.rs | 6 +- encodings/zstd/src/zstd_buffers.rs | 8 +- vortex-array/public-api.lock | 178 ++---------------- vortex-array/src/aggregate_fn/accumulator.rs | 2 +- .../src/aggregate_fn/fns/count/mod.rs | 2 +- .../src/aggregate_fn/fns/first/mod.rs | 2 +- .../src/aggregate_fn/fns/is_constant/mod.rs | 2 +- .../src/aggregate_fn/fns/is_sorted/mod.rs | 2 +- vortex-array/src/aggregate_fn/fns/last/mod.rs | 2 +- .../src/aggregate_fn/fns/min_max/mod.rs | 2 +- .../src/aggregate_fn/fns/nan_count/mod.rs | 2 +- vortex-array/src/aggregate_fn/fns/sum/mod.rs | 2 +- vortex-array/src/aggregate_fn/foreign.rs | 2 +- vortex-array/src/aggregate_fn/mod.rs | 4 +- vortex-array/src/aggregate_fn/proto.rs | 9 +- vortex-array/src/aggregate_fn/session.rs | 17 +- vortex-array/src/array/foreign.rs | 2 +- vortex-array/src/array/mod.rs | 4 +- vortex-array/src/array/plugin.rs | 7 +- vortex-array/src/arrays/bool/vtable/mod.rs | 9 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 9 +- .../src/arrays/constant/vtable/mod.rs | 8 +- vortex-array/src/arrays/decimal/vtable/mod.rs | 9 +- vortex-array/src/arrays/dict/compute/rules.rs | 5 +- vortex-array/src/arrays/dict/vtable/mod.rs | 8 +- .../src/arrays/extension/compute/rules.rs | 4 +- .../src/arrays/extension/vtable/mod.rs | 8 +- vortex-array/src/arrays/filter/vtable.rs | 9 +- .../src/arrays/fixed_size_list/vtable/mod.rs | 9 +- vortex-array/src/arrays/list/vtable/mod.rs | 9 +- .../src/arrays/listview/vtable/mod.rs | 9 +- vortex-array/src/arrays/masked/vtable/mod.rs | 8 +- vortex-array/src/arrays/null/mod.rs | 8 +- vortex-array/src/arrays/patched/vtable/mod.rs | 4 +- .../src/arrays/primitive/vtable/mod.rs | 8 +- .../src/arrays/scalar_fn/vtable/mod.rs | 4 +- vortex-array/src/arrays/shared/vtable.rs | 9 +- vortex-array/src/arrays/slice/vtable.rs | 9 +- vortex-array/src/arrays/struct_/vtable/mod.rs | 10 +- vortex-array/src/arrays/varbin/vtable/mod.rs | 9 +- .../src/arrays/varbinview/vtable/mod.rs | 8 +- vortex-array/src/arrays/variant/vtable/mod.rs | 8 +- vortex-array/src/dtype/extension/foreign.rs | 2 +- vortex-array/src/dtype/extension/mod.rs | 3 +- vortex-array/src/dtype/serde/flatbuffers.rs | 11 +- vortex-array/src/dtype/serde/proto.rs | 2 +- vortex-array/src/dtype/serde/serde.rs | 2 +- vortex-array/src/expr/proto.rs | 4 +- vortex-array/src/extension/datetime/date.rs | 2 +- vortex-array/src/extension/datetime/time.rs | 2 +- .../src/extension/datetime/timestamp.rs | 2 +- .../src/extension/tests/divisible_int.rs | 2 +- vortex-array/src/extension/uuid/vtable.rs | 2 +- vortex-array/src/normalize.rs | 11 +- vortex-array/src/scalar/arrow.rs | 2 +- vortex-array/src/scalar/tests/casting.rs | 6 +- .../src/scalar/typed_view/extension/tests.rs | 6 +- vortex-array/src/scalar_fn/fns/between/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/binary/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/case_when.rs | 2 +- vortex-array/src/scalar_fn/fns/cast/mod.rs | 2 +- .../src/scalar_fn/fns/fill_null/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/get_item.rs | 2 +- vortex-array/src/scalar_fn/fns/is_not_null.rs | 2 +- vortex-array/src/scalar_fn/fns/like/mod.rs | 2 +- .../src/scalar_fn/fns/list_contains/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/mask/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/not/mod.rs | 2 +- vortex-array/src/scalar_fn/fns/root.rs | 2 +- vortex-array/src/scalar_fn/fns/select.rs | 2 +- vortex-array/src/scalar_fn/fns/zip/mod.rs | 2 +- vortex-array/src/scalar_fn/foreign.rs | 2 +- vortex-array/src/scalar_fn/mod.rs | 4 +- vortex-array/src/serde.rs | 4 +- .../src/dynamic_dispatch/plan_builder.rs | 39 ++-- vortex-cuda/src/executor.rs | 3 +- vortex-cuda/src/layout.rs | 5 +- vortex-cuda/src/lib.rs | 31 +-- vortex-cuda/src/session.rs | 8 +- vortex-duckdb/src/convert/dtype.rs | 2 +- vortex-file/src/footer/mod.rs | 4 +- vortex-ipc/src/messages/decoder.rs | 2 +- vortex-layout/public-api.lock | 4 +- vortex-layout/src/encoding.rs | 4 +- vortex-layout/src/flatbuffers.rs | 4 +- vortex-layout/src/layout.rs | 5 +- vortex-layout/src/layouts/chunked/mod.rs | 2 +- vortex-layout/src/layouts/dict/mod.rs | 2 +- vortex-layout/src/layouts/dict/reader.rs | 4 +- vortex-layout/src/layouts/flat/mod.rs | 2 +- vortex-layout/src/layouts/foreign/mod.rs | 4 +- vortex-layout/src/layouts/row_idx/expr.rs | 2 +- vortex-layout/src/layouts/struct_/mod.rs | 2 +- vortex-layout/src/layouts/zoned/mod.rs | 2 +- vortex-python/src/arrays/py/array.rs | 2 +- vortex-python/src/arrays/py/mod.rs | 22 +-- vortex-python/src/arrays/py/vtable.rs | 2 +- vortex-python/src/serde/context.rs | 4 +- vortex-session/Cargo.toml | 1 + vortex-session/public-api.lock | 74 +++++++- vortex-session/src/registry.rs | 132 ++++++++++++- vortex-tensor/src/fixed_shape/vtable.rs | 2 +- vortex-tensor/src/scalar_fns/l2_denorm.rs | 2 +- .../src/scalar_fns/sorf_transform/vtable.rs | 2 +- vortex-tensor/src/vector/vtable.rs | 2 +- .../fixtures/arrays/synthetic/arrays/bool.rs | 3 +- .../arrays/synthetic/arrays/chunked.rs | 3 +- .../arrays/synthetic/arrays/datetime.rs | 3 +- .../arrays/synthetic/arrays/decimal.rs | 3 +- .../synthetic/arrays/fixed_size_list.rs | 3 +- .../fixtures/arrays/synthetic/arrays/list.rs | 3 +- .../arrays/synthetic/arrays/listview.rs | 3 +- .../fixtures/arrays/synthetic/arrays/null.rs | 3 +- .../arrays/synthetic/arrays/primitive.rs | 3 +- .../arrays/synthetic/arrays/struct_nested.rs | 3 +- .../arrays/synthetic/arrays/varbin.rs | 3 +- .../arrays/synthetic/arrays/varbinview.rs | 3 +- .../arrays/synthetic/encodings/alp.rs | 3 +- .../arrays/synthetic/encodings/alprd.rs | 3 +- .../arrays/synthetic/encodings/bitpacked.rs | 3 +- .../arrays/synthetic/encodings/bytebool.rs | 3 +- .../arrays/synthetic/encodings/constant.rs | 3 +- .../synthetic/encodings/datetimeparts.rs | 3 +- .../synthetic/encodings/decimal_byte_parts.rs | 3 +- .../arrays/synthetic/encodings/delta.rs | 3 +- .../arrays/synthetic/encodings/dict.rs | 3 +- .../arrays/synthetic/encodings/for_.rs | 3 +- .../arrays/synthetic/encodings/fsst.rs | 3 +- .../arrays/synthetic/encodings/pco.rs | 3 +- .../arrays/synthetic/encodings/rle.rs | 3 +- .../arrays/synthetic/encodings/runend.rs | 3 +- .../arrays/synthetic/encodings/sequence.rs | 3 +- .../arrays/synthetic/encodings/sparse.rs | 3 +- .../arrays/synthetic/encodings/zigzag.rs | 3 +- .../arrays/synthetic/encodings/zstd.rs | 3 +- 173 files changed, 625 insertions(+), 611 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52f583dd2cc..c925ab4d18c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,7 +961,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -3534,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4898,7 +4898,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4979,7 +4979,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5603,6 +5603,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -6287,7 +6297,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6958,9 +6968,9 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.8.5", + "rand 0.9.2", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -7167,7 +7177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -7199,7 +7209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8154,7 +8164,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8212,7 +8222,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9291,7 +9301,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9310,7 +9320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10903,6 +10913,7 @@ version = "0.1.0" dependencies = [ "arcref", "dashmap", + "lasso", "parking_lot", "vortex-error", "vortex-utils", @@ -11282,7 +11293,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3c67020c38a..f4cfcecbf38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,6 +163,7 @@ itertools = "0.14.0" jetscii = "0.5.3" jiff = "0.2.0" kanal = "0.1.1" +lasso = { version = "0.7", features = ["multi-threaded"] } lending-iterator = "0.1.7" libfuzzer-sys = "0.4" libloading = "0.8" diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index 0f3259d6d6f..ff4535970be 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -6,10 +6,6 @@ pub struct vortex_alp::ALP impl vortex_alp::ALP -pub const vortex_alp::ALP::ID: vortex_array::array::ArrayId - -impl vortex_alp::ALP - pub fn vortex_alp::ALP::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray @@ -148,8 +144,6 @@ pub struct vortex_alp::ALPRD impl vortex_alp::ALPRD -pub const vortex_alp::ALPRD::ID: vortex_array::array::ArrayId - pub unsafe fn vortex_alp::ALPRD::new_unchecked(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option) -> vortex_alp::ALPRDArray pub fn vortex_alp::ALPRD::try_new(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option) -> vortex_error::VortexResult diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 2c3d2d70923..be0db0f938e 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -39,6 +39,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ALPFloat; use crate::alp::Exponents; @@ -72,7 +73,8 @@ impl VTable for ALP { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.alp"); + *ID } fn validate( @@ -237,10 +239,6 @@ impl Display for ALPData { #[derive(Clone, Debug)] pub struct ALP; -impl ALP { - pub const ID: ArrayId = ArrayId::new_ref("vortex.alp"); -} - #[derive(Clone, prost::Message)] pub struct ALPMetadata { #[prost(uint32, tag = "1")] diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 020acb7e1ac..3b2c36ebd02 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -6,9 +6,11 @@ //! //! This enables zero-cost backward compatibility with previously written datasets. +use vortex_array::Array; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; @@ -32,7 +34,8 @@ impl ArrayPlugin for ALPPatchedPlugin { fn id(&self) -> ArrayId { // We reuse the existing `ALP` ID so that we can take over its // deserialization pathway. - ALP::ID + // TODO(joe): dedup method name + ArrayVTable::id(&ALP) } fn serialize( @@ -53,10 +56,10 @@ impl ArrayPlugin for ALPPatchedPlugin { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let alp_array = ALP - .deserialize(dtype, len, metadata, buffers, children, session)? - .try_downcast::() - .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?; + let alp_array = Array::::try_from_parts(ArrayVTable::deserialize( + &ALP, dtype, len, metadata, buffers, children, session, + )?) + .map_err(|_| vortex_err!("ALP plugin should only deserialize vortex.alp"))?; // Check if there are interior patches to externalize. let Some(patches) = alp_array.patches() else { @@ -78,7 +81,8 @@ impl ArrayPlugin for ALPPatchedPlugin { } fn is_supported_encoding(&self, id: &ArrayId) -> bool { - id == &Patched.id() || id == &ALP.id() + // TODO(joe): dedup method name + id == ArrayVTable::id(&Patched) || id == ArrayVTable::id(&ALP) } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index c4d6d3b7f02..be5f5f9a918 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -44,6 +44,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::alp_rd::kernel::PARENT_KERNELS; use crate::alp_rd::rules::RULES; @@ -92,7 +93,8 @@ impl VTable for ALPRD { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.alprd"); + *ID } fn validate( @@ -357,8 +359,6 @@ pub struct ALPRDDataParts { pub struct ALPRD; impl ALPRD { - pub const ID: ArrayId = ArrayId::new_ref("vortex.alprd"); - pub fn try_new( dtype: DType, left_parts: ArrayRef, diff --git a/encodings/alp/src/lib.rs b/encodings/alp/src/lib.rs index 35a7534d7aa..05ce75763b2 100644 --- a/encodings/alp/src/lib.rs +++ b/encodings/alp/src/lib.rs @@ -18,6 +18,7 @@ pub use alp::*; pub use alp_rd::*; +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::nan_count::NanCount; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; @@ -41,7 +42,7 @@ pub fn initialize(session: &VortexSession) { // Register the ALP-specific NaN count aggregate kernel. session.aggregate_fns().register_aggregate_kernel( - ALP::ID, + ALP.id(), Some(NanCount.id()), &compute::nan_count::ALPNanCountKernel, ); diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 63d9c1b9518..514f8742a28 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_bytebool::ByteBool impl vortex_bytebool::ByteBool -pub const vortex_bytebool::ByteBool::ID: vortex_array::array::ArrayId - pub fn vortex_bytebool::ByteBool::from_option_vec(data: alloc::vec::Vec>) -> vortex_bytebool::ByteBoolArray pub fn vortex_bytebool::ByteBool::from_vec>(data: alloc::vec::Vec, validity: V) -> vortex_bytebool::ByteBoolArray diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index c1a2ebf7fce..28205d973be 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -37,6 +37,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_mask::Mask; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::kernel::PARENT_KERNELS; @@ -62,7 +63,8 @@ impl VTable for ByteBool { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.bytebool"); + *ID } fn validate( @@ -200,8 +202,6 @@ impl> ByteBoolArrayExt for T {} pub struct ByteBool; impl ByteBool { - pub const ID: ArrayId = ArrayId::new_ref("vortex.bytebool"); - pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { let dtype = DType::Bool(validity.nullability()); let slots = ByteBoolData::make_slots(&validity, buffer.len()); diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 3cdc22cc7b7..555681ac824 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_datetime_parts::DateTimeParts impl vortex_datetime_parts::DateTimeParts -pub const vortex_datetime_parts::DateTimeParts::ID: vortex_array::array::ArrayId - pub fn vortex_datetime_parts::DateTimeParts::try_from_temporal(temporal: vortex_array::arrays::datetime::TemporalArray) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimeParts::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index f66bd2b742a..67edc3dfc20 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -35,6 +35,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::TemporalParts; use crate::canonical::decode_to_temporal; @@ -92,7 +93,8 @@ impl VTable for DateTimeParts { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.datetimeparts"); + *ID } fn validate( @@ -250,8 +252,6 @@ impl> DateTimePartsArrayExt for T {} pub struct DateTimeParts; impl DateTimeParts { - pub const ID: ArrayId = ArrayId::new_ref("vortex.datetimeparts"); - /// Construct a new [`DateTimePartsArray`] from its components. pub fn try_new( dtype: DType, diff --git a/encodings/datetime-parts/src/lib.rs b/encodings/datetime-parts/src/lib.rs index 1babf4abaab..e061d69bccd 100644 --- a/encodings/datetime-parts/src/lib.rs +++ b/encodings/datetime-parts/src/lib.rs @@ -11,6 +11,7 @@ mod compute; mod ops; mod timestamp; +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; @@ -22,7 +23,7 @@ pub fn initialize(session: &VortexSession) { session.arrays().register(DateTimeParts); session.aggregate_fns().register_aggregate_kernel( - DateTimeParts::ID, + DateTimeParts.id(), Some(IsConstant.id()), &compute::is_constant::DateTimePartsIsConstantKernel, ); diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 397885e1b45..57a83581aa2 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_decimal_byte_parts::DecimalByteParts impl vortex_decimal_byte_parts::DecimalByteParts -pub const vortex_decimal_byte_parts::DecimalByteParts::ID: vortex_array::array::ArrayId - pub fn vortex_decimal_byte_parts::DecimalByteParts::try_new(msp: vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType) -> vortex_error::VortexResult impl core::clone::Clone for vortex_decimal_byte_parts::DecimalByteParts diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index ea90bf575e0..390fd15f0fe 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -43,6 +43,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::decimal_byte_parts::compute::kernel::PARENT_KERNELS; use crate::decimal_byte_parts::rules::PARENT_RULES; @@ -75,7 +76,8 @@ impl VTable for DecimalByteParts { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.decimal_byte_parts"); + *ID } fn validate( @@ -253,8 +255,6 @@ impl DecimalBytePartsData { pub struct DecimalByteParts; impl DecimalByteParts { - pub const ID: ArrayId = ArrayId::new_ref("vortex.decimal_byte_parts"); - /// Construct a new [`DecimalBytePartsArray`] from an MSP array and decimal dtype. pub fn try_new( msp: ArrayRef, diff --git a/encodings/decimal-byte-parts/src/lib.rs b/encodings/decimal-byte-parts/src/lib.rs index 58c2d1dc69a..86566027afa 100644 --- a/encodings/decimal-byte-parts/src/lib.rs +++ b/encodings/decimal-byte-parts/src/lib.rs @@ -13,6 +13,7 @@ use decimal_byte_parts::compute::is_constant::DecimalBytePartsIsConstantKernel; /// an i128 decimal could be converted into a [i64, u64] with further narrowing applied to either /// value. pub use decimal_byte_parts::*; +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; @@ -24,7 +25,7 @@ pub fn initialize(session: &VortexSession) { session.arrays().register(DecimalByteParts); session.aggregate_fns().register_aggregate_kernel( - DecimalByteParts::ID, + DecimalByteParts.id(), Some(IsConstant.id()), &DecimalBytePartsIsConstantKernel, ); diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 7746cd91168..cf7776cd872 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -120,8 +120,6 @@ pub struct vortex_fastlanes::BitPacked impl vortex_fastlanes::BitPacked -pub const vortex_fastlanes::BitPacked::ID: vortex_array::array::ArrayId - pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedDataParts @@ -280,8 +278,6 @@ pub struct vortex_fastlanes::Delta impl vortex_fastlanes::Delta -pub const vortex_fastlanes::Delta::ID: vortex_array::array::ArrayId - pub fn vortex_fastlanes::Delta::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult @@ -368,8 +364,6 @@ pub struct vortex_fastlanes::FoR impl vortex_fastlanes::FoR -pub const vortex_fastlanes::FoR::ID: vortex_array::array::ArrayId - pub fn vortex_fastlanes::FoR::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult pub fn vortex_fastlanes::FoR::try_new(encoded: vortex_array::array::erased::ArrayRef, reference: vortex_array::scalar::Scalar) -> vortex_error::VortexResult @@ -474,8 +468,6 @@ pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE -pub const vortex_fastlanes::RLE::ID: vortex_array::array::ArrayId - pub fn vortex_fastlanes::RLE::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index b712d45b45b..fec101ff895 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -6,9 +6,11 @@ //! //! This enables zero-cost backward compatibility with previously written datasets. +use vortex_array::Array; use vortex_array::ArrayId; use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; @@ -31,7 +33,8 @@ impl ArrayPlugin for BitPackedPatchedPlugin { fn id(&self) -> ArrayId { // We reuse the existing `BitPacked` ID so that we can take over its // deserialization pathway. - BitPacked::ID + // TODO(joe): dedup method name + ArrayVTable::id(&BitPacked) } fn serialize( @@ -52,12 +55,10 @@ impl ArrayPlugin for BitPackedPatchedPlugin { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let bitpacked = BitPacked - .deserialize(dtype, len, metadata, buffers, children, session)? - .try_downcast::() - .map_err(|_| { - vortex_err!("BitPacked plugin should only deserialize fastlanes.bitpacked") - })?; + let bitpacked = Array::::try_from_parts(ArrayVTable::deserialize( + &BitPacked, dtype, len, metadata, buffers, children, session, + )?) + .map_err(|_| vortex_err!("BitPacked plugin should only deserialize fastlanes.bitpacked"))?; // Create a new BitPackedArray without the interior patches installed. let Some(patches) = bitpacked.patches() else { @@ -84,7 +85,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin { } fn is_supported_encoding(&self, id: &ArrayId) -> bool { - id == &BitPacked::ID || id == &Patched.id() + id == ArrayVTable::id(&BitPacked) || id == ArrayVTable::id(&Patched) } } diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 93c5d116fdc..9c1875468a1 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -36,6 +36,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::BitPackedArrayExt; use crate::BitPackedData; @@ -91,7 +92,8 @@ impl VTable for BitPacked { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("fastlanes.bitpacked"); + *ID } fn validate( @@ -309,8 +311,6 @@ impl VTable for BitPacked { pub struct BitPacked; impl BitPacked { - pub const ID: ArrayId = ArrayId::new_ref("fastlanes.bitpacked"); - pub fn try_new( packed: BufferHandle, ptype: PType, diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index e74243b4050..bb5add65aa0 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -30,6 +30,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::DeltaData; use crate::delta::array::BASES_SLOT; @@ -75,7 +76,8 @@ impl VTable for Delta { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("fastlanes.delta"); + *ID } fn validate( @@ -180,8 +182,6 @@ impl VTable for Delta { pub struct Delta; impl Delta { - pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta"); - pub fn try_new( bases: ArrayRef, deltas: ArrayRef, diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 3f3207a8e24..85f7cf2429b 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -30,6 +30,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::FoRData; use crate::r#for::array::FoRArrayExt; @@ -66,7 +67,8 @@ impl VTable for FoR { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("fastlanes.for"); + *ID } fn validate( @@ -162,8 +164,6 @@ impl VTable for FoR { pub struct FoR; impl FoR { - pub const ID: ArrayId = ArrayId::new_ref("fastlanes.for"); - /// Construct a new FoR array from an encoded array and a reference scalar. pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 763c55484ea..217d05d73b8 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -24,6 +24,7 @@ pub(crate) const FL_CHUNK_SIZE: usize = 1024; use bitpacking::compute::is_constant::BitPackedIsConstantKernel; use r#for::compute::is_constant::FoRIsConstantKernel; use r#for::compute::is_sorted::FoRIsSortedKernel; +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; @@ -47,17 +48,17 @@ pub fn initialize(session: &VortexSession) { // Register the encoding-specific aggregate kernels. session.aggregate_fns().register_aggregate_kernel( - BitPacked::ID, + BitPacked.id(), Some(IsConstant.id()), &BitPackedIsConstantKernel, ); session.aggregate_fns().register_aggregate_kernel( - FoR::ID, + FoR.id(), Some(IsConstant.id()), &FoRIsConstantKernel, ); session.aggregate_fns().register_aggregate_kernel( - FoR::ID, + FoR.id(), Some(IsSorted.id()), &FoRIsSortedKernel, ); diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 9587603aacc..6d37aad85fd 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -28,6 +28,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::RLEData; use crate::rle::array::INDICES_SLOT; @@ -80,7 +81,8 @@ impl VTable for RLE { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("fastlanes.rle"); + *ID } fn validate( @@ -209,8 +211,6 @@ impl VTable for RLE { pub struct RLE; impl RLE { - pub const ID: ArrayId = ArrayId::new_ref("fastlanes.rle"); - pub fn try_new( values: ArrayRef, indices: ArrayRef, diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 341a2fd8222..024701dad3b 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_fsst::FSST impl vortex_fsst::FSST -pub const vortex_fsst::FSST::ID: vortex_array::array::ArrayId - pub fn vortex_fsst::FSST::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fsst::FSST diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 5526276aeb4..d3a51a254e9 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -49,6 +49,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::canonical::canonicalize_fsst; use crate::canonical::fsst_decode_views; @@ -101,7 +102,8 @@ impl VTable for FSST { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.fsst"); + *ID } fn validate( @@ -363,8 +365,6 @@ impl Debug for FSSTData { pub struct FSST; impl FSST { - pub const ID: ArrayId = ArrayId::new_ref("vortex.fsst"); - /// Build an FSST array from a set of `symbols` and `codes`. /// /// The `codes` VarBinArray is decomposed: its bytes are stored in [`FSSTData`], while diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index efc501cdac6..7dc3b3c90b2 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -30,6 +30,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_proto::dtype as pb; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::array::METADATA_SLOT; use crate::array::ParquetVariantArrayExt; @@ -45,10 +46,6 @@ use crate::kernel::PARENT_KERNELS; #[derive(Debug, Clone)] pub struct ParquetVariant; -impl ParquetVariant { - pub const ID: ArrayId = ArrayId::new_ref("vortex.parquet.variant"); -} - #[derive(Clone, prost::Message)] struct ParquetVariantMetadataProto { /// Whether the un-shredded `value` child is present. @@ -71,7 +68,8 @@ impl VTable for ParquetVariant { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.parquet.variant"); + *ID } fn validate( diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 07307c38196..1020dfa0eb1 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_pco::Pco impl vortex_pco::Pco -pub const vortex_pco::Pco::ID: vortex_array::array::ArrayId - pub fn vortex_pco::Pco::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_pco::Pco diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 8f83dbd8fa8..7c8e74c825c 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -55,6 +55,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::PcoChunkInfo; use crate::PcoMetadata; @@ -129,7 +130,8 @@ impl VTable for Pco { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.pco"); + *ID } fn validate( @@ -273,8 +275,6 @@ pub(crate) fn vortex_err_from_pco(err: PcoError) -> VortexError { pub struct Pco; impl Pco { - pub const ID: ArrayId = ArrayId::new_ref("vortex.pco"); - pub(crate) fn try_new( dtype: DType, data: PcoData, diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index c746d4f54a4..163db53f69e 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -20,8 +20,6 @@ pub struct vortex_runend::RunEnd impl vortex_runend::RunEnd -pub const vortex_runend::RunEnd::ID: vortex_array::array::ArrayId - pub fn vortex_runend::RunEnd::encode(array: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult pub fn vortex_runend::RunEnd::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_runend::RunEndArray diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index f714d09fc70..5844f2c9cf4 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -39,6 +39,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::compress::runend_decode_primitive; use crate::compress::runend_decode_varbinview; @@ -79,7 +80,8 @@ impl VTable for RunEnd { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.runend"); + *ID } fn validate( @@ -241,8 +243,6 @@ impl> RunEndArrayExt for T {} pub struct RunEnd; impl RunEnd { - pub const ID: ArrayId = ArrayId::new_ref("vortex.runend"); - /// Build a new [`RunEndArray`] without validation. /// /// # Safety diff --git a/encodings/runend/src/lib.rs b/encodings/runend/src/lib.rs index cff09767d38..8770dbbf58e 100644 --- a/encodings/runend/src/lib.rs +++ b/encodings/runend/src/lib.rs @@ -26,6 +26,7 @@ pub mod _benchmarking { use super::*; } +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; @@ -40,17 +41,17 @@ pub fn initialize(session: &VortexSession) { // Register the RunEnd-specific aggregate kernels. session.aggregate_fns().register_aggregate_kernel( - RunEnd::ID, + RunEnd.id(), Some(MinMax.id()), &compute::min_max::RunEndMinMaxKernel, ); session.aggregate_fns().register_aggregate_kernel( - RunEnd::ID, + RunEnd.id(), Some(IsConstant.id()), &compute::is_constant::RunEndIsConstantKernel, ); session.aggregate_fns().register_aggregate_kernel( - RunEnd::ID, + RunEnd.id(), Some(IsSorted.id()), &compute::is_sorted::RunEndIsSortedKernel, ); diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 6bb8f69d1d2..455a2fd9809 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_sequence::Sequence impl vortex_sequence::Sequence -pub const vortex_sequence::Sequence::ID: vortex_array::array::ArrayId - pub fn vortex_sequence::Sequence::try_new(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, ptype: vortex_array::dtype::ptype::PType, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult pub fn vortex_sequence::Sequence::try_new_typed>(base: T, multiplier: T, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index f941d4cf1a6..413862a6c30 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -45,6 +45,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::compress::sequence_decompress; use crate::kernel::PARENT_KERNELS; @@ -231,7 +232,8 @@ impl VTable for Sequence { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.sequence"); + *ID } fn validate( @@ -369,8 +371,6 @@ impl ValidityVTable for Sequence { pub struct Sequence; impl Sequence { - pub const ID: ArrayId = ArrayId::new_ref("vortex.sequence"); - fn stats(multiplier: PValue) -> StatsSet { // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, // and strictly sorted iff multiplier > 0. diff --git a/encodings/sequence/src/lib.rs b/encodings/sequence/src/lib.rs index aec425e0bae..bd6ab2f508c 100644 --- a/encodings/sequence/src/lib.rs +++ b/encodings/sequence/src/lib.rs @@ -16,6 +16,7 @@ pub use array::SequenceArray; pub use array::SequenceData; pub use array::SequenceDataParts; pub use compress::sequence_encode; +use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::fns::min_max::MinMax; @@ -29,12 +30,12 @@ pub fn initialize(session: &VortexSession) { // Register the Sequence-specific aggregate kernels. session.aggregate_fns().register_aggregate_kernel( - Sequence::ID, + Sequence.id(), Some(MinMax.id()), &compute::min_max::SequenceMinMaxKernel, ); session.aggregate_fns().register_aggregate_kernel( - Sequence::ID, + Sequence.id(), Some(IsSorted.id()), &compute::is_sorted::SequenceIsSortedKernel, ); diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 30a2e9a726c..9bcfb086753 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_sparse::Sparse impl vortex_sparse::Sparse -pub const vortex_sparse::Sparse::ID: vortex_array::array::ArrayId - pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayRef, fill_value: core::option::Option) -> vortex_error::VortexResult pub fn vortex_sparse::Sparse::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 87fd640fd1c..845d918a7a3 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -47,6 +47,7 @@ use vortex_error::vortex_panic; use vortex_mask::AllOr; use vortex_mask::Mask; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::canonical::execute_sparse; use crate::rules::RULES; @@ -88,7 +89,8 @@ impl VTable for Sparse { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.sparse"); + *ID } fn validate( @@ -227,8 +229,6 @@ impl Display for SparseData { pub struct Sparse; impl Sparse { - pub const ID: ArrayId = ArrayId::new_ref("vortex.sparse"); - /// Construct a new [`SparseArray`] from indices, values, length, and fill value. pub fn try_new( indices: ArrayRef, diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index d94894a7ca0..299344043ad 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_zigzag::ZigZag impl vortex_zigzag::ZigZag -pub const vortex_zigzag::ZigZag::ID: vortex_array::array::ArrayId - pub fn vortex_zigzag::ZigZag::try_new(encoded: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult impl core::clone::Clone for vortex_zigzag::ZigZag diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 851b40b438b..bc10192ce78 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -33,6 +33,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use zigzag::ZigZag as ExternalZigZag; use crate::compute::ZigZagEncoded; @@ -50,7 +51,8 @@ impl VTable for ZigZag { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.zigzag"); + *ID } fn validate( @@ -195,8 +197,6 @@ impl> ZigZagArrayExt for T {} pub struct ZigZag; impl ZigZag { - pub const ID: ArrayId = ArrayId::new_ref("vortex.zigzag"); - /// Construct a new [`ZigZagArray`] from an encoded unsigned integer array. pub fn try_new(encoded: ArrayRef) -> VortexResult { let dtype = ZigZagData::dtype_from_encoded_dtype(encoded.dtype())?; diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 44365606591..7a4e13bb8cd 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -4,8 +4,6 @@ pub struct vortex_zstd::Zstd impl vortex_zstd::Zstd -pub const vortex_zstd::Zstd::ID: vortex_array::array::ArrayId - pub fn vortex_zstd::Zstd::decompress(array: &vortex_zstd::ZstdArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index a11d48defb6..789a0ae91ad 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -55,6 +55,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_mask::AllOr; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ZstdFrameMetadata; use crate::ZstdMetadata; @@ -134,7 +135,8 @@ impl VTable for Zstd { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.zstd"); + *ID } fn validate( @@ -253,8 +255,6 @@ impl VTable for Zstd { pub struct Zstd; impl Zstd { - pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd"); - pub fn try_new(dtype: DType, data: ZstdData, validity: Validity) -> VortexResult { let len = data.len(); data.validate(&dtype, len, &validity)?; diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index e1712b01c2e..ffeb70210ae 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -34,6 +34,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ZstdBuffersMetadata; @@ -44,8 +45,6 @@ pub type ZstdBuffersArray = Array; pub struct ZstdBuffers; impl ZstdBuffers { - pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd_buffers"); - pub fn try_new( dtype: DType, len: usize, @@ -333,7 +332,7 @@ fn compute_output_layout( } fn array_id_from_string(s: &str) -> ArrayId { - ArrayId::new_arc(Arc::from(s)) + ArrayId::new(s) } impl ArrayHash for ZstdBuffersData { @@ -369,7 +368,8 @@ impl VTable for ZstdBuffers { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.zstd_buffers"); + *ID } fn validate( diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index d79cc795aa1..8f26230791a 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -562,7 +562,7 @@ impl vortex_array::aggregate_fn::session::AggregateFnSession pub fn vortex_array::aggregate_fn::session::AggregateFnSession::register(&self, vtable: V) -pub fn vortex_array::aggregate_fn::session::AggregateFnSession::register_aggregate_kernel(&self, array_id: vortex_array::ArrayId, agg_fn_id: core::option::Option, kernel: &'static dyn vortex_array::aggregate_fn::kernels::DynAggregateKernel) +pub fn vortex_array::aggregate_fn::session::AggregateFnSession::register_aggregate_kernel(&self, array_id: impl core::convert::Into, agg_fn_id: core::option::Option>, kernel: &'static dyn vortex_array::aggregate_fn::kernels::DynAggregateKernel) pub fn vortex_array::aggregate_fn::session::AggregateFnSession::registry(&self) -> &vortex_array::aggregate_fn::session::AggregateFnRegistry @@ -744,7 +744,7 @@ impl vortex_array::aggregate_f pub fn V::deserialize(&self, metadata: &[u8], session: &vortex_session::VortexSession) -> core::result::Result -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id pub trait vortex_array::aggregate_fn::AggregateFnVTable: 'static + core::marker::Sized + core::clone::Clone + core::marker::Send + core::marker::Sync @@ -1116,7 +1116,7 @@ pub fn vortex_array::aggregate_fn::GroupedAccumulator::flush(&mut self) -> vo pub type vortex_array::aggregate_fn::AccumulatorRef = alloc::boxed::Box -pub type vortex_array::aggregate_fn::AggregateFnId = arcref::ArcRef +pub type vortex_array::aggregate_fn::AggregateFnId = vortex_session::registry::Id pub type vortex_array::aggregate_fn::AggregateFnPluginRef = alloc::sync::Arc @@ -1128,10 +1128,6 @@ pub mod vortex_array::arrays::bool pub struct vortex_array::arrays::bool::Bool -impl vortex_array::arrays::Bool - -pub const vortex_array::arrays::Bool::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Bool pub fn vortex_array::arrays::Bool::clone(&self) -> vortex_array::arrays::Bool @@ -1306,10 +1302,6 @@ pub mod vortex_array::arrays::chunked pub struct vortex_array::arrays::chunked::Chunked -impl vortex_array::arrays::Chunked - -pub const vortex_array::arrays::Chunked::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Chunked pub fn vortex_array::arrays::Chunked::clone(&self) -> vortex_array::arrays::Chunked @@ -1472,10 +1464,6 @@ pub struct vortex_array::arrays::constant::Constant impl vortex_array::arrays::Constant -pub const vortex_array::arrays::Constant::ID: vortex_array::ArrayId - -impl vortex_array::arrays::Constant - pub const vortex_array::arrays::Constant::TAKE_RULES: vortex_array::optimizer::rules::ParentRuleSet impl core::clone::Clone for vortex_array::arrays::Constant @@ -1658,10 +1646,6 @@ pub mod vortex_array::arrays::decimal pub struct vortex_array::arrays::decimal::Decimal -impl vortex_array::arrays::Decimal - -pub const vortex_array::arrays::Decimal::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::clone(&self) -> vortex_array::arrays::Decimal @@ -1878,10 +1862,6 @@ pub mod vortex_array::arrays::dict::vtable pub struct vortex_array::arrays::dict::vtable::Dict -impl vortex_array::arrays::dict::Dict - -pub const vortex_array::arrays::dict::Dict::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::dict::Dict pub fn vortex_array::arrays::dict::Dict::clone(&self) -> vortex_array::arrays::dict::Dict @@ -1974,10 +1954,6 @@ pub type vortex_array::arrays::dict::vtable::DictArray = vortex_array::Array vortex_array::arrays::dict::Dict @@ -2312,10 +2288,6 @@ pub mod vortex_array::arrays::extension pub struct vortex_array::arrays::extension::Extension -impl vortex_array::arrays::Extension - -pub const vortex_array::arrays::Extension::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Extension pub fn vortex_array::arrays::Extension::clone(&self) -> vortex_array::arrays::Extension @@ -2442,10 +2414,6 @@ pub mod vortex_array::arrays::filter pub struct vortex_array::arrays::filter::Filter -impl vortex_array::arrays::Filter - -pub const vortex_array::arrays::Filter::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Filter pub fn vortex_array::arrays::Filter::clone(&self) -> vortex_array::arrays::Filter @@ -2634,10 +2602,6 @@ pub mod vortex_array::arrays::fixed_size_list pub struct vortex_array::arrays::fixed_size_list::FixedSizeList -impl vortex_array::arrays::FixedSizeList - -pub const vortex_array::arrays::FixedSizeList::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::clone(&self) -> vortex_array::arrays::FixedSizeList @@ -2782,10 +2746,6 @@ pub mod vortex_array::arrays::list pub struct vortex_array::arrays::list::List -impl vortex_array::arrays::List - -pub const vortex_array::arrays::List::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::List pub fn vortex_array::arrays::List::clone(&self) -> vortex_array::arrays::List @@ -2966,10 +2926,6 @@ pub vortex_array::arrays::listview::ListViewRebuildMode::TrimElements pub struct vortex_array::arrays::listview::ListView -impl vortex_array::arrays::ListView - -pub const vortex_array::arrays::ListView::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::clone(&self) -> vortex_array::arrays::ListView @@ -3150,10 +3106,6 @@ pub mod vortex_array::arrays::masked pub struct vortex_array::arrays::masked::Masked -impl vortex_array::arrays::Masked - -pub const vortex_array::arrays::Masked::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::clone(&self) -> vortex_array::arrays::Masked @@ -3278,10 +3230,6 @@ pub struct vortex_array::arrays::null::Null impl vortex_array::arrays::null::Null -pub const vortex_array::arrays::null::Null::ID: vortex_array::ArrayId - -impl vortex_array::arrays::null::Null - pub const vortex_array::arrays::null::Null::TAKE_RULES: vortex_array::optimizer::rules::ParentRuleSet impl core::clone::Clone for vortex_array::arrays::null::Null @@ -3656,10 +3604,6 @@ pub fn vortex_array::arrays::primitive::NativeValue::partial_cmp(&self, other pub struct vortex_array::arrays::primitive::Primitive -impl vortex_array::arrays::Primitive - -pub const vortex_array::arrays::Primitive::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::clone(&self) -> vortex_array::arrays::Primitive @@ -4064,10 +4008,6 @@ pub mod vortex_array::arrays::shared pub struct vortex_array::arrays::shared::Shared -impl vortex_array::arrays::Shared - -pub const vortex_array::arrays::Shared::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Shared pub fn vortex_array::arrays::Shared::clone(&self) -> vortex_array::arrays::Shared @@ -4180,10 +4120,6 @@ pub mod vortex_array::arrays::slice pub struct vortex_array::arrays::slice::Slice -impl vortex_array::arrays::slice::Slice - -pub const vortex_array::arrays::slice::Slice::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::slice::Slice pub fn vortex_array::arrays::slice::Slice::clone(&self) -> vortex_array::arrays::slice::Slice @@ -4410,10 +4346,6 @@ pub mod vortex_array::arrays::struct_ pub struct vortex_array::arrays::struct_::Struct -impl vortex_array::arrays::Struct - -pub const vortex_array::arrays::Struct::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::clone(&self) -> vortex_array::arrays::Struct @@ -4572,10 +4504,6 @@ pub struct vortex_array::arrays::varbin::VarBin impl vortex_array::arrays::VarBin -pub const vortex_array::arrays::VarBin::ID: vortex_array::ArrayId - -impl vortex_array::arrays::VarBin - pub fn vortex_array::arrays::VarBin::_slice(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, range: core::ops::range::Range) -> vortex_error::VortexResult impl core::clone::Clone for vortex_array::arrays::VarBin @@ -4944,10 +4872,6 @@ impl core::marker::Copy for vortex_array::arrays::varbinview::Ref pub struct vortex_array::arrays::varbinview::VarBinView -impl vortex_array::arrays::VarBinView - -pub const vortex_array::arrays::VarBinView::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::clone(&self) -> vortex_array::arrays::VarBinView @@ -5134,10 +5058,6 @@ pub mod vortex_array::arrays::variant pub struct vortex_array::arrays::variant::Variant -impl vortex_array::arrays::Variant - -pub const vortex_array::arrays::Variant::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Variant pub fn vortex_array::arrays::Variant::clone(&self) -> vortex_array::arrays::Variant @@ -5206,10 +5126,6 @@ pub type vortex_array::arrays::variant::VariantArray = vortex_array::Array vortex_array::arrays::Bool @@ -5298,10 +5214,6 @@ pub fn vortex_array::arrays::Bool::mask(array: vortex_array::ArrayView<'_, vorte pub struct vortex_array::arrays::Chunked -impl vortex_array::arrays::Chunked - -pub const vortex_array::arrays::Chunked::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Chunked pub fn vortex_array::arrays::Chunked::clone(&self) -> vortex_array::arrays::Chunked @@ -5390,10 +5302,6 @@ pub struct vortex_array::arrays::Constant impl vortex_array::arrays::Constant -pub const vortex_array::arrays::Constant::ID: vortex_array::ArrayId - -impl vortex_array::arrays::Constant - pub const vortex_array::arrays::Constant::TAKE_RULES: vortex_array::optimizer::rules::ParentRuleSet impl core::clone::Clone for vortex_array::arrays::Constant @@ -5482,10 +5390,6 @@ pub fn vortex_array::arrays::Constant::invert(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::Decimal -impl vortex_array::arrays::Decimal - -pub const vortex_array::arrays::Decimal::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Decimal pub fn vortex_array::arrays::Decimal::clone(&self) -> vortex_array::arrays::Decimal @@ -5574,10 +5478,6 @@ pub fn vortex_array::arrays::Decimal::mask(array: vortex_array::ArrayView<'_, vo pub struct vortex_array::arrays::Dict -impl vortex_array::arrays::dict::Dict - -pub const vortex_array::arrays::dict::Dict::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::dict::Dict pub fn vortex_array::arrays::dict::Dict::clone(&self) -> vortex_array::arrays::dict::Dict @@ -5668,10 +5568,6 @@ pub fn vortex_array::arrays::dict::Dict::mask(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::Extension -impl vortex_array::arrays::Extension - -pub const vortex_array::arrays::Extension::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Extension pub fn vortex_array::arrays::Extension::clone(&self) -> vortex_array::arrays::Extension @@ -5754,10 +5650,6 @@ pub fn vortex_array::arrays::Extension::mask(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::Filter -impl vortex_array::arrays::Filter - -pub const vortex_array::arrays::Filter::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Filter pub fn vortex_array::arrays::Filter::clone(&self) -> vortex_array::arrays::Filter @@ -5816,10 +5708,6 @@ pub fn vortex_array::arrays::Filter::validity(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::FixedSizeList -impl vortex_array::arrays::FixedSizeList - -pub const vortex_array::arrays::FixedSizeList::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::FixedSizeList pub fn vortex_array::arrays::FixedSizeList::clone(&self) -> vortex_array::arrays::FixedSizeList @@ -5894,10 +5782,6 @@ pub fn vortex_array::arrays::FixedSizeList::mask(array: vortex_array::ArrayView< pub struct vortex_array::arrays::List -impl vortex_array::arrays::List - -pub const vortex_array::arrays::List::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::List pub fn vortex_array::arrays::List::clone(&self) -> vortex_array::arrays::List @@ -5976,10 +5860,6 @@ pub fn vortex_array::arrays::List::mask(array: vortex_array::ArrayView<'_, vorte pub struct vortex_array::arrays::ListView -impl vortex_array::arrays::ListView - -pub const vortex_array::arrays::ListView::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::clone(&self) -> vortex_array::arrays::ListView @@ -6054,10 +5934,6 @@ pub fn vortex_array::arrays::ListView::mask(array: vortex_array::ArrayView<'_, v pub struct vortex_array::arrays::Masked -impl vortex_array::arrays::Masked - -pub const vortex_array::arrays::Masked::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Masked pub fn vortex_array::arrays::Masked::clone(&self) -> vortex_array::arrays::Masked @@ -6134,10 +6010,6 @@ pub struct vortex_array::arrays::Null impl vortex_array::arrays::null::Null -pub const vortex_array::arrays::null::Null::ID: vortex_array::ArrayId - -impl vortex_array::arrays::null::Null - pub const vortex_array::arrays::null::Null::TAKE_RULES: vortex_array::optimizer::rules::ParentRuleSet impl core::clone::Clone for vortex_array::arrays::null::Null @@ -6296,10 +6168,6 @@ pub fn vortex_array::arrays::patched::Patched::compare(lhs: vortex_array::ArrayV pub struct vortex_array::arrays::Primitive -impl vortex_array::arrays::Primitive - -pub const vortex_array::arrays::Primitive::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::clone(&self) -> vortex_array::arrays::Primitive @@ -6446,10 +6314,6 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_a pub struct vortex_array::arrays::Shared -impl vortex_array::arrays::Shared - -pub const vortex_array::arrays::Shared::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Shared pub fn vortex_array::arrays::Shared::clone(&self) -> vortex_array::arrays::Shared @@ -6508,10 +6372,6 @@ pub fn vortex_array::arrays::Shared::validity(array: vortex_array::ArrayView<'_, pub struct vortex_array::arrays::Slice -impl vortex_array::arrays::slice::Slice - -pub const vortex_array::arrays::slice::Slice::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::slice::Slice pub fn vortex_array::arrays::slice::Slice::clone(&self) -> vortex_array::arrays::slice::Slice @@ -6574,10 +6434,6 @@ pub fn vortex_array::arrays::slice::Slice::slice(array: vortex_array::ArrayView< pub struct vortex_array::arrays::Struct -impl vortex_array::arrays::Struct - -pub const vortex_array::arrays::Struct::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Struct pub fn vortex_array::arrays::Struct::clone(&self) -> vortex_array::arrays::Struct @@ -6658,10 +6514,6 @@ pub struct vortex_array::arrays::VarBin impl vortex_array::arrays::VarBin -pub const vortex_array::arrays::VarBin::ID: vortex_array::ArrayId - -impl vortex_array::arrays::VarBin - pub fn vortex_array::arrays::VarBin::_slice(array: vortex_array::ArrayView<'_, vortex_array::arrays::VarBin>, range: core::ops::range::Range) -> vortex_error::VortexResult impl core::clone::Clone for vortex_array::arrays::VarBin @@ -6746,10 +6598,6 @@ pub fn vortex_array::arrays::VarBin::mask(array: vortex_array::ArrayView<'_, vor pub struct vortex_array::arrays::VarBinView -impl vortex_array::arrays::VarBinView - -pub const vortex_array::arrays::VarBinView::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::VarBinView pub fn vortex_array::arrays::VarBinView::clone(&self) -> vortex_array::arrays::VarBinView @@ -6828,10 +6676,6 @@ pub fn vortex_array::arrays::VarBinView::zip(if_true: vortex_array::ArrayView<'_ pub struct vortex_array::arrays::Variant -impl vortex_array::arrays::Variant - -pub const vortex_array::arrays::Variant::ID: vortex_array::ArrayId - impl core::clone::Clone for vortex_array::arrays::Variant pub fn vortex_array::arrays::Variant::clone(&self) -> vortex_array::arrays::Variant @@ -8832,7 +8676,7 @@ impl vortex_array::dtype::extensio pub fn V::deserialize(&self, data: &[u8], storage_dtype: vortex_array::dtype::DType) -> core::result::Result -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id pub trait vortex_array::dtype::extension::ExtVTable: 'static + core::marker::Sized + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + core::cmp::Eq + core::hash::Hash @@ -8980,7 +8824,7 @@ pub fn V::try_match<'a>(ext_dtype: &'a vortex_array::dtype::extension::ExtDTypeR pub type vortex_array::dtype::extension::ExtDTypePluginRef = alloc::sync::Arc -pub type vortex_array::dtype::extension::ExtId = arcref::ArcRef +pub type vortex_array::dtype::extension::ExtId = vortex_session::registry::Id pub mod vortex_array::dtype::flatbuffers @@ -17800,7 +17644,7 @@ impl vortex_array::scalar_fn::Scalar pub fn V::deserialize(&self, metadata: &[u8], session: &vortex_session::VortexSession) -> core::result::Result -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id pub trait vortex_array::scalar_fn::ScalarFnVTable: 'static + core::marker::Sized + core::clone::Clone + core::marker::Send + core::marker::Sync @@ -18624,7 +18468,7 @@ pub type vortex_array::scalar_fn::ChildName = arcref::ArcRef pub type vortex_array::scalar_fn::ReduceNodeRef = alloc::sync::Arc -pub type vortex_array::scalar_fn::ScalarFnId = arcref::ArcRef +pub type vortex_array::scalar_fn::ScalarFnId = vortex_session::registry::Id pub type vortex_array::scalar_fn::ScalarFnPluginRef = alloc::sync::Arc @@ -19422,7 +19266,7 @@ impl vortex_array::ArrayPlugin for V pub fn V::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> core::result::Result -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool @@ -23080,9 +22924,9 @@ pub fn V::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, met pub fn V::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> core::result::Result -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id -pub fn V::id(&self) -> arcref::ArcRef +pub fn V::id(&self) -> vortex_session::registry::Id pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool @@ -25340,7 +25184,7 @@ pub fn vortex_array::validity_to_child(validity: &vortex_array::validity::Validi pub type vortex_array::ArrayContext = vortex_session::registry::Context -pub type vortex_array::ArrayId = arcref::ArcRef +pub type vortex_array::ArrayId = vortex_session::registry::Id pub type vortex_array::ArrayPluginRef = alloc::sync::Arc diff --git a/vortex-array/src/aggregate_fn/accumulator.rs b/vortex-array/src/aggregate_fn/accumulator.rs index 00c80b38221..71ed56b3110 100644 --- a/vortex-array/src/aggregate_fn/accumulator.rs +++ b/vortex-array/src/aggregate_fn/accumulator.rs @@ -116,7 +116,7 @@ impl DynAccumulator for Accumulator { let kernels_r = kernels.read(); let batch_id = batch.encoding_id(); if let Some(result) = kernels_r - .get(&(batch_id.clone(), Some(self.aggregate_fn.id()))) + .get(&(batch_id, Some(self.aggregate_fn.id()))) .or_else(|| kernels_r.get(&(batch_id, None))) .and_then(|kernel| { kernel diff --git a/vortex-array/src/aggregate_fn/fns/count/mod.rs b/vortex-array/src/aggregate_fn/fns/count/mod.rs index f0617e96bfe..da12b85b0ee 100644 --- a/vortex-array/src/aggregate_fn/fns/count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/count/mod.rs @@ -27,7 +27,7 @@ impl AggregateFnVTable for Count { type Partial = u64; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.count") + AggregateFnId::new("vortex.count") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/first/mod.rs b/vortex-array/src/aggregate_fn/fns/first/mod.rs index 38e55607279..67affbd4fdc 100644 --- a/vortex-array/src/aggregate_fn/fns/first/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/first/mod.rs @@ -40,7 +40,7 @@ impl AggregateFnVTable for First { type Partial = FirstPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.first") + AggregateFnId::new("vortex.first") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index 4df47b770de..3edebb91333 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -256,7 +256,7 @@ impl AggregateFnVTable for IsConstant { type Partial = IsConstantPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.is_constant") + AggregateFnId::new("vortex.is_constant") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index 780fc492baa..e68e74da63d 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -227,7 +227,7 @@ impl AggregateFnVTable for IsSorted { type Partial = IsSortedPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.is_sorted") + AggregateFnId::new("vortex.is_sorted") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/last/mod.rs b/vortex-array/src/aggregate_fn/fns/last/mod.rs index 009b487ad2d..a1cbf032b88 100644 --- a/vortex-array/src/aggregate_fn/fns/last/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/last/mod.rs @@ -40,7 +40,7 @@ impl AggregateFnVTable for Last { type Partial = LastPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.last") + AggregateFnId::new("vortex.last") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs index 91d178a4814..17e886b1efb 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs @@ -173,7 +173,7 @@ impl AggregateFnVTable for MinMax { type Partial = MinMaxPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.min_max") + AggregateFnId::new("vortex.min_max") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs index 9a1a8e4793b..602963cc385 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs @@ -81,7 +81,7 @@ impl AggregateFnVTable for NanCount { type Partial = u64; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.nan_count") + AggregateFnId::new("vortex.nan_count") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/fns/sum/mod.rs b/vortex-array/src/aggregate_fn/fns/sum/mod.rs index 4d09eac4d5b..7b45aa65fa9 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/mod.rs @@ -71,7 +71,7 @@ impl AggregateFnVTable for Sum { type Partial = SumPartial; fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.sum") + AggregateFnId::new("vortex.sum") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/foreign.rs b/vortex-array/src/aggregate_fn/foreign.rs index e79d2456493..26672cb23b7 100644 --- a/vortex-array/src/aggregate_fn/foreign.rs +++ b/vortex-array/src/aggregate_fn/foreign.rs @@ -54,7 +54,7 @@ impl AggregateFnVTable for ForeignAggregateFnVTable { type Partial = (); fn id(&self) -> AggregateFnId { - self.id.clone() + self.id } fn serialize(&self, options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/mod.rs b/vortex-array/src/aggregate_fn/mod.rs index a22006cb981..b697265e62b 100644 --- a/vortex-array/src/aggregate_fn/mod.rs +++ b/vortex-array/src/aggregate_fn/mod.rs @@ -6,7 +6,7 @@ //! This module contains the [`AggregateFnVTable`] trait, the [`Accumulator`] trait, and the //! type-erasure infrastructure for aggregate functions. -use arcref::ArcRef; +use vortex_session::registry::Id; mod accumulator; pub use accumulator::*; @@ -38,7 +38,7 @@ pub mod proto; pub mod session; /// A unique identifier for an aggregate function. -pub type AggregateFnId = ArcRef; +pub type AggregateFnId = Id; /// Private module to seal [`typed::DynAggregateFn`]. mod sealed { diff --git a/vortex-array/src/aggregate_fn/proto.rs b/vortex-array/src/aggregate_fn/proto.rs index dfb86ad0ed9..b0e7d9828ed 100644 --- a/vortex-array/src/aggregate_fn/proto.rs +++ b/vortex-array/src/aggregate_fn/proto.rs @@ -1,9 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - -use arcref::ArcRef; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; @@ -38,11 +35,11 @@ impl AggregateFnRef { /// /// Note: the serialization format is not stable and may change between versions. pub fn from_proto(proto: &pb::AggregateFn, session: &VortexSession) -> VortexResult { - let agg_fn_id: AggregateFnId = ArcRef::new_arc(Arc::from(proto.id.as_str())); + let agg_fn_id: AggregateFnId = AggregateFnId::new(proto.id.as_str()); let agg_fn = if let Some(plugin) = session.aggregate_fns().registry().find(&agg_fn_id) { plugin.deserialize(proto.metadata(), session)? } else if session.allows_unknown() { - new_foreign_aggregate_fn(agg_fn_id.clone(), proto.metadata().to_vec()) + new_foreign_aggregate_fn(agg_fn_id, proto.metadata().to_vec()) } else { return Err(vortex_err!("unknown aggregate function id: {}", proto.id)); }; @@ -89,7 +86,7 @@ mod tests { type Partial = (); fn id(&self) -> AggregateFnId { - AggregateFnId::new_ref("vortex.test.proto") + AggregateFnId::new("vortex.test.proto") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/aggregate_fn/session.rs b/vortex-array/src/aggregate_fn/session.rs index 64e66195358..886b1ea9c2d 100644 --- a/vortex-array/src/aggregate_fn/session.rs +++ b/vortex-array/src/aggregate_fn/session.rs @@ -22,6 +22,7 @@ use crate::aggregate_fn::fns::sum::Sum; use crate::aggregate_fn::kernels::DynAggregateKernel; use crate::aggregate_fn::kernels::DynGroupedAggregateKernel; use crate::array::ArrayId; +use crate::array::VTable; use crate::arrays::Chunked; use crate::arrays::Dict; use crate::arrays::chunked::compute::aggregate::ChunkedArrayAggregate; @@ -61,10 +62,10 @@ impl Default for AggregateFnSession { this.register(Sum); // Register the built-in aggregate kernels. - this.register_aggregate_kernel(Chunked::ID, None, &ChunkedArrayAggregate); - this.register_aggregate_kernel(Dict::ID, Some(MinMax.id()), &DictMinMaxKernel); - this.register_aggregate_kernel(Dict::ID, Some(IsConstant.id()), &DictIsConstantKernel); - this.register_aggregate_kernel(Dict::ID, Some(IsSorted.id()), &DictIsSortedKernel); + this.register_aggregate_kernel(Chunked.id(), None::, &ChunkedArrayAggregate); + this.register_aggregate_kernel(Dict.id(), Some(MinMax.id()), &DictMinMaxKernel); + this.register_aggregate_kernel(Dict.id(), Some(IsConstant.id()), &DictIsConstantKernel); + this.register_aggregate_kernel(Dict.id(), Some(IsSorted.id()), &DictIsSortedKernel); this } @@ -86,11 +87,13 @@ impl AggregateFnSession { /// Register an aggregate function kernel for a specific aggregate function and array type. pub fn register_aggregate_kernel( &self, - array_id: ArrayId, - agg_fn_id: Option, + array_id: impl Into, + agg_fn_id: Option>, kernel: &'static dyn DynAggregateKernel, ) { - self.kernels.write().insert((array_id, agg_fn_id), kernel); + self.kernels + .write() + .insert((array_id.into(), agg_fn_id.map(|id| id.into())), kernel); } } diff --git a/vortex-array/src/array/foreign.rs b/vortex-array/src/array/foreign.rs index b29d76ba2b8..3233d2f4228 100644 --- a/vortex-array/src/array/foreign.rs +++ b/vortex-array/src/array/foreign.rs @@ -94,7 +94,7 @@ impl VTable for ForeignArray { type ValidityVTable = ForeignValidityVTable; fn id(&self) -> ArrayId { - self.id.clone() + self.id } fn validate( diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 9ef3e57cab2..fae2d486ab6 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -7,7 +7,6 @@ use std::fmt::Formatter; use std::hash::Hash; use std::hash::Hasher; -use arcref::ArcRef; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -15,6 +14,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::Id; use crate::ExecutionCtx; use crate::LEGACY_SESSION; @@ -510,4 +510,4 @@ impl Hasher for HasherWrapper<'_> { } /// ArrayId is a globally unique name for the array's vtable. -pub type ArrayId = ArcRef; +pub type ArrayId = Id; diff --git a/vortex-array/src/array/plugin.rs b/vortex-array/src/array/plugin.rs index f9411584f44..66845eb9a0a 100644 --- a/vortex-array/src/array/plugin.rs +++ b/vortex-array/src/array/plugin.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::fmt; +use std::fmt::Debug; +use std::fmt::Formatter; use std::sync::Arc; use vortex_error::VortexResult; @@ -62,8 +65,8 @@ pub trait ArrayPlugin: 'static + Send + Sync { } } -impl std::fmt::Debug for dyn ArrayPlugin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Debug for dyn ArrayPlugin { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_tuple("ArrayPlugin").field(&self.id()).finish() } } diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index d02a8da22cf..44ba2204a85 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -30,6 +30,8 @@ mod kernel; mod operations; mod validity; +use vortex_session::registry::CachedId; + use crate::Precision; use crate::array::ArrayId; use crate::arrays::bool::compute::rules::RULES; @@ -66,7 +68,8 @@ impl VTable for Bool { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.bool"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -190,10 +193,6 @@ impl VTable for Bool { #[derive(Clone, Debug)] pub struct Bool; -impl Bool { - pub const ID: ArrayId = ArrayId::new_ref("vortex.bool"); -} - #[cfg(test)] mod tests { use vortex_buffer::ByteBufferMut; diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 29e4b397d35..c0bc63bfdb6 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -48,10 +49,6 @@ pub type ChunkedArray = Array; #[derive(Clone, Debug)] pub struct Chunked; -impl Chunked { - pub const ID: ArrayId = ArrayId::new_ref("vortex.chunked"); -} - impl ArrayHash for ChunkedData { fn array_hash(&self, _state: &mut H, _precision: Precision) { // Chunk offsets are cached derived data. Slot 0 already stores the logical offsets array, @@ -72,9 +69,9 @@ impl VTable for Chunked { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.chunked"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index d6f1c6ab2d3..9b4f9dbb953 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -51,10 +52,6 @@ pub type ConstantArray = Array; #[derive(Clone, Debug)] pub struct Constant; -impl Constant { - pub const ID: ArrayId = ArrayId::new_ref("vortex.constant"); -} - impl ArrayHash for ConstantData { fn array_hash(&self, state: &mut H, _precision: Precision) { self.scalar.hash(state); @@ -74,7 +71,8 @@ impl VTable for Constant { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.constant"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 1894466ed6b..4fce4915013 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -32,6 +32,8 @@ mod validity; use std::hash::Hash; +use vortex_session::registry::CachedId; + use crate::Precision; use crate::array::ArrayId; use crate::arrays::decimal::array::SLOT_NAMES; @@ -68,7 +70,8 @@ impl VTable for Decimal { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.decimal"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -203,10 +206,6 @@ impl VTable for Decimal { #[derive(Clone, Debug)] pub struct Decimal; -impl Decimal { - pub const ID: ArrayId = ArrayId::new_ref("vortex.decimal"); -} - #[cfg(test)] mod tests { use vortex_buffer::ByteBufferMut; diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index c6817a39d58..bf148004af3 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -8,6 +8,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::Precision; use crate::array::ArrayView; +use crate::array::VTable; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; @@ -80,7 +81,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { tracing::trace!( "Not pushing down fallible scalar function {} over dictionary with sparse codes {}", parent.scalar_fn(), - Dict::ID, + Dict.id(), ); return Ok(None); } @@ -104,7 +105,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { tracing::trace!( "Not pushing down null-sensitive scalar function {} over dictionary with null codes {}", parent.scalar_fn(), - Dict::ID, + Dict.id(), ); return Ok(None); } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index b13374b9d8d..1fe15d5ce3d 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -12,6 +12,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use super::DictData; use super::DictMetadata; @@ -52,10 +53,6 @@ pub type DictArray = Array; #[derive(Clone, Debug)] pub struct Dict; -impl Dict { - pub const ID: ArrayId = ArrayId::new_ref("vortex.dict"); -} - impl ArrayHash for DictData { fn array_hash(&self, _state: &mut H, _precision: Precision) {} } @@ -73,7 +70,8 @@ impl VTable for Dict { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.dict"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index af1cbc832b9..3f5295f729e 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -86,7 +86,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("test_ext") + ExtId::new("test_ext") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { @@ -182,7 +182,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("test_ext_2") + ExtId::new("test_ext_2") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index ba403403dbe..b82d4b8e0df 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -14,6 +14,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -54,7 +55,8 @@ impl VTable for Extension { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.ext"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -165,7 +167,3 @@ impl VTable for Extension { #[derive(Clone, Debug)] pub struct Extension; - -impl Extension { - pub const ID: ArrayId = ArrayId::new_ref("vortex.ext"); -} diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 8eb7fb62213..05f49b5c941 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_mask::Mask; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::AnyCanonical; use crate::ArrayEq; @@ -48,10 +49,6 @@ pub type FilterArray = Array; #[derive(Clone, Debug)] pub struct Filter; -impl Filter { - pub const ID: ArrayId = ArrayId::new_ref("vortex.filter"); -} - impl ArrayHash for FilterData { fn array_hash(&self, state: &mut H, precision: Precision) { self.mask.array_hash(state, precision); @@ -68,9 +65,9 @@ impl VTable for Filter { type ArrayData = FilterData; type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.filter"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index b2c02ec6891..a3724b06fca 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -40,10 +41,6 @@ pub type FixedSizeListArray = Array; #[derive(Clone, Debug)] pub struct FixedSizeList; -impl FixedSizeList { - pub const ID: ArrayId = ArrayId::new_ref("vortex.fixed_size_list"); -} - impl ArrayHash for FixedSizeListData { fn array_hash(&self, state: &mut H, precision: Precision) { let _precision = precision; @@ -63,9 +60,9 @@ impl VTable for FixedSizeList { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.fixed_size_list"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index d9624f49560..8563c4ed27a 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -64,9 +65,9 @@ impl VTable for List { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.list"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -200,7 +201,3 @@ impl VTable for List { #[derive(Clone, Debug)] pub struct List; - -impl List { - pub const ID: ArrayId = ArrayId::new_ref("vortex.list"); -} diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 6338c6504c1..893edb6bf71 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -12,6 +12,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -42,10 +43,6 @@ pub type ListViewArray = Array; #[derive(Clone, Debug)] pub struct ListView; -impl ListView { - pub const ID: ArrayId = ArrayId::new_ref("vortex.listview"); -} - #[derive(Clone, prost::Message)] pub struct ListViewMetadata { #[prost(uint64, tag = "1")] @@ -73,9 +70,9 @@ impl VTable for ListView { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.listview"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index cc5ef29a758..5f1f09e0644 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -12,6 +12,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::AnyCanonical; use crate::ArrayEq; @@ -48,10 +49,6 @@ pub type MaskedArray = Array; #[derive(Clone, Debug)] pub struct Masked; -impl Masked { - pub const ID: ArrayId = ArrayId::new_ref("vortex.masked"); -} - impl ArrayHash for MaskedData { fn array_hash(&self, _state: &mut H, _precision: Precision) {} } @@ -69,7 +66,8 @@ impl VTable for Masked { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.masked"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index b2bdcc3a840..6e88b02aaf0 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -5,6 +5,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayRef; use crate::ExecutionCtx; @@ -36,7 +37,8 @@ impl VTable for Null { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.null"); + *ID } fn validate( @@ -139,10 +141,6 @@ impl VTable for Null { #[derive(Clone, Debug)] pub struct Null; -impl Null { - pub const ID: ArrayId = ArrayId::new_ref("vortex.null"); -} - impl Array { pub fn new(len: usize) -> Self { unsafe { diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index c49e681a411..5a1cdef3edf 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -17,6 +17,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayRef; use crate::Canonical; @@ -99,7 +100,8 @@ impl VTable for Patched { type ValidityVTable = ValidityVTableFromChild; fn id(&self) -> ArrayId { - ArrayId::new_ref("vortex.patched") + static ID: CachedId = CachedId::new("vortex.patched"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 806c95b6a8f..c1cdfc20d28 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -27,6 +27,7 @@ use std::hash::Hasher; use vortex_buffer::Alignment; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::Precision; use crate::array::ArrayId; @@ -57,7 +58,8 @@ impl VTable for Primitive { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.primitive"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -203,10 +205,6 @@ impl VTable for Primitive { #[derive(Clone, Debug)] pub struct Primitive; -impl Primitive { - pub const ID: ArrayId = ArrayId::new_ref("vortex.primitive"); -} - #[cfg(test)] mod tests { use vortex_buffer::ByteBufferMut; diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index c663ddfc64e..3f70088966b 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -71,7 +71,7 @@ impl VTable for ScalarFnVTable { type ValidityVTable = Self; fn id(&self) -> ArrayId { - self.id.clone() + self.id } fn validate( @@ -284,7 +284,7 @@ impl scalar_fn::ScalarFnVTable for ArrayExpr { type Options = FakeEq; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.array") + ScalarFnId::new("vortex.array") } fn arity(&self, _options: &Self::Options) -> Arity { diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 6be759c83eb..3f67738684d 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -7,6 +7,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayEq; use crate::ArrayHash; @@ -37,10 +38,6 @@ pub type SharedArray = Array; #[derive(Clone, Debug)] pub struct Shared; -impl Shared { - pub const ID: ArrayId = ArrayId::new_ref("vortex.shared"); -} - impl ArrayHash for SharedData { fn array_hash(&self, _state: &mut H, _precision: Precision) {} } @@ -55,9 +52,9 @@ impl VTable for Shared { type ArrayData = SharedData; type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.shared"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 2ac41a358b9..911de401aa4 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::AnyCanonical; use crate::ArrayEq; @@ -45,10 +46,6 @@ pub type SliceArray = Array; #[derive(Clone, Debug)] pub struct Slice; -impl Slice { - pub const ID: ArrayId = ArrayId::new_ref("vortex.slice"); -} - impl ArrayHash for SliceData { fn array_hash(&self, state: &mut H, _precision: Precision) { self.range.start.hash(state); @@ -66,9 +63,9 @@ impl VTable for Slice { type ArrayData = SliceData; type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Slice::ID + static ID: CachedId = CachedId::new("vortex.slice"); + *ID } fn validate( diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index f33cea360c3..5da40b56f24 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -29,6 +29,8 @@ mod kernel; mod operations; mod validity; +use vortex_session::registry::CachedId; + use crate::array::ArrayId; /// A [`Struct`]-encoded Vortex array. @@ -39,9 +41,9 @@ impl VTable for Struct { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.struct"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -205,7 +207,3 @@ impl VTable for Struct { #[derive(Clone, Debug)] pub struct Struct; - -impl Struct { - pub const ID: ArrayId = ArrayId::new_ref("vortex.struct"); -} diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index ffe44158c27..b72ac5eaa48 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -9,6 +9,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; +use vortex_session::registry::CachedId; use crate::ArrayRef; use crate::ExecutionCtx; @@ -68,9 +69,9 @@ impl VTable for VarBin { type OperationsVTable = Self; type ValidityVTable = Self; - fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.varbin"); + *ID } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -198,7 +199,3 @@ impl VTable for VarBin { #[derive(Clone, Debug)] pub struct VarBin; - -impl VarBin { - pub const ID: ArrayId = ArrayId::new_ref("vortex.varbin"); -} diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 453cd7b61d4..3827549744c 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayRef; use crate::ExecutionCtx; @@ -42,10 +43,6 @@ pub type VarBinViewArray = Array; #[derive(Clone, Debug)] pub struct VarBinView; -impl VarBinView { - pub const ID: ArrayId = ArrayId::new_ref("vortex.varbinview"); -} - impl ArrayHash for VarBinViewData { fn array_hash(&self, state: &mut H, precision: Precision) { for buffer in self.buffers.iter() { @@ -74,7 +71,8 @@ impl VTable for VarBinView { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.varbinview"); + *ID } fn nbuffers(array: ArrayView<'_, Self>) -> usize { diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 2fdf200b98f..25db7e16077 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -9,6 +9,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use vortex_session::registry::CachedId; use crate::ArrayRef; use crate::ExecutionCtx; @@ -29,10 +30,6 @@ pub type VariantArray = Array; #[derive(Clone, Debug)] pub struct Variant; -impl Variant { - pub const ID: ArrayId = ArrayId::new_ref("vortex.variant"); -} - impl VTable for Variant { type ArrayData = EmptyArrayData; @@ -41,7 +38,8 @@ impl VTable for Variant { type ValidityVTable = Self; fn id(&self) -> ArrayId { - Self::ID + static ID: CachedId = CachedId::new("vortex.variant"); + *ID } fn validate( diff --git a/vortex-array/src/dtype/extension/foreign.rs b/vortex-array/src/dtype/extension/foreign.rs index 7753a5b6b86..b7636225ff9 100644 --- a/vortex-array/src/dtype/extension/foreign.rs +++ b/vortex-array/src/dtype/extension/foreign.rs @@ -51,7 +51,7 @@ impl ExtVTable for ForeignExtDType { type NativeValue<'a> = &'a ScalarValue; fn id(&self) -> ExtId { - self.id.clone() + self.id } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/dtype/extension/mod.rs b/vortex-array/src/dtype/extension/mod.rs index 0eae1587b40..96dfa7e22a3 100644 --- a/vortex-array/src/dtype/extension/mod.rs +++ b/vortex-array/src/dtype/extension/mod.rs @@ -30,9 +30,10 @@ pub use erased::*; mod matcher; pub use matcher::*; +use vortex_session::registry::Id; /// A unique identifier for an extension type -pub type ExtId = arcref::ArcRef; +pub type ExtId = Id; /// Private module to seal [`typed::DynExtDType`]. mod sealed { diff --git a/vortex-array/src/dtype/serde/flatbuffers.rs b/vortex-array/src/dtype/serde/flatbuffers.rs index af7ed7b8af3..ea4470039f1 100644 --- a/vortex-array/src/dtype/serde/flatbuffers.rs +++ b/vortex-array/src/dtype/serde/flatbuffers.rs @@ -195,13 +195,10 @@ impl TryFrom for DType { let fb_ext = fb .type__as_extension() .ok_or_else(|| vortex_err!("failed to parse extension from flatbuffer"))?; - let id = ExtId::new_arc( - fb_ext - .id() - .ok_or_else(|| vortex_err!("failed to parse extension id from flatbuffer"))? - .to_string() - .into(), - ); + let id = + ExtId::new(fb_ext.id().ok_or_else(|| { + vortex_err!("failed to parse extension id from flatbuffer") + })?); let storage_dtype = fb_ext.storage_dtype().ok_or_else(|| { vortex_err!( Serde: "storage_dtype must be present on DType fbs message") diff --git a/vortex-array/src/dtype/serde/proto.rs b/vortex-array/src/dtype/serde/proto.rs index 4ca8642120b..1dfe60f522e 100644 --- a/vortex-array/src/dtype/serde/proto.rs +++ b/vortex-array/src/dtype/serde/proto.rs @@ -87,7 +87,7 @@ impl DType { )) } DtypeType::Extension(e) => { - let id = ExtId::new_arc(e.id.as_str().to_string().into()); + let id = ExtId::new(e.id.as_str()); let storage_dtype = DType::from_proto( e.storage_dtype .as_ref() diff --git a/vortex-array/src/dtype/serde/serde.rs b/vortex-array/src/dtype/serde/serde.rs index e89a1a98658..1f7ea6fc2ca 100644 --- a/vortex-array/src/dtype/serde/serde.rs +++ b/vortex-array/src/dtype/serde/serde.rs @@ -571,7 +571,7 @@ impl<'de> DeserializeSeed<'de> for DTypeSerde<'_, ExtDTypeRef> { } let id = id.ok_or_else(|| de::Error::missing_field("id"))?; - let id = ExtId::new_arc(id); + let id = ExtId::new(&id); let storage_dtype = storage_dtype.ok_or_else(|| de::Error::missing_field("storage_dtype"))?; let metadata = metadata.ok_or_else(|| de::Error::missing_field("metadata"))?; diff --git a/vortex-array/src/expr/proto.rs b/vortex-array/src/expr/proto.rs index 22c5aa8b467..27859235e88 100644 --- a/vortex-array/src/expr/proto.rs +++ b/vortex-array/src/expr/proto.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - use itertools::Itertools; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -41,7 +39,7 @@ impl ExprSerializeProtoExt for Expression { impl Expression { pub fn from_proto(expr: &pb::Expr, session: &VortexSession) -> VortexResult { - let expr_id = ScalarFnId::new_arc(Arc::from(expr.id.to_string())); + let expr_id = ScalarFnId::new(expr.id.as_str()); let children = expr .children .iter() diff --git a/vortex-array/src/extension/datetime/date.rs b/vortex-array/src/extension/datetime/date.rs index a0434b3ebc5..8f3ae954535 100644 --- a/vortex-array/src/extension/datetime/date.rs +++ b/vortex-array/src/extension/datetime/date.rs @@ -79,7 +79,7 @@ impl ExtVTable for Date { type NativeValue<'a> = DateValue; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.date") + ExtId::new("vortex.date") } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/extension/datetime/time.rs b/vortex-array/src/extension/datetime/time.rs index 08c43e9e20c..d3a62b5ef65 100644 --- a/vortex-array/src/extension/datetime/time.rs +++ b/vortex-array/src/extension/datetime/time.rs @@ -80,7 +80,7 @@ impl ExtVTable for Time { type NativeValue<'a> = TimeValue; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.time") + ExtId::new("vortex.time") } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/extension/datetime/timestamp.rs b/vortex-array/src/extension/datetime/timestamp.rs index b33571ed40e..ab65ce8b84c 100644 --- a/vortex-array/src/extension/datetime/timestamp.rs +++ b/vortex-array/src/extension/datetime/timestamp.rs @@ -114,7 +114,7 @@ impl ExtVTable for Timestamp { type NativeValue<'a> = TimestampValue<'a>; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.timestamp") + ExtId::new("vortex.timestamp") } // NOTE(ngates): unfortunately we're stuck with this hand-rolled serialization format for diff --git a/vortex-array/src/extension/tests/divisible_int.rs b/vortex-array/src/extension/tests/divisible_int.rs index 8b9612e9438..4ab08f7b9f5 100644 --- a/vortex-array/src/extension/tests/divisible_int.rs +++ b/vortex-array/src/extension/tests/divisible_int.rs @@ -35,7 +35,7 @@ impl ExtVTable for DivisibleInt { type NativeValue<'a> = u64; fn id(&self) -> ExtId { - ExtId::new_ref("test.divisible_int") + ExtId::new("test.divisible_int") } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/extension/uuid/vtable.rs b/vortex-array/src/extension/uuid/vtable.rs index c76c7352942..283de266684 100644 --- a/vortex-array/src/extension/uuid/vtable.rs +++ b/vortex-array/src/extension/uuid/vtable.rs @@ -27,7 +27,7 @@ impl ExtVTable for Uuid { type NativeValue<'a> = uuid::Uuid; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.uuid") + ExtId::new("vortex.uuid") } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/normalize.rs b/vortex-array/src/normalize.rs index 3235a480809..f4c1d8c5a8f 100644 --- a/vortex-array/src/normalize.rs +++ b/vortex-array/src/normalize.rs @@ -105,6 +105,7 @@ mod tests { use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; + use crate::array::VTable; use crate::arrays::Dict; use crate::arrays::DictArray; use crate::arrays::Primitive; @@ -203,22 +204,18 @@ mod tests { // Slice the dict array to get a SliceArray wrapping a DictArray. let sliced = SliceArray::new(dict, 1..4).into_array(); - assert_eq!(sliced.encoding_id(), Slice::ID); + assert_eq!(sliced.encoding_id(), Slice.id()); - let allowed = HashSet::from_iter([Dict::ID, Primitive::ID]); + let allowed = HashSet::from_iter([Dict.id(), Primitive.id()]); let mut ctx = ExecutionCtx::new(VortexSession::empty()); - println!("sliced {}", sliced.display_tree()); - let normalized = sliced.normalize(&mut NormalizeOptions { allowed: &allowed, operation: Operation::Execute(&mut ctx), })?; - println!("after {}", normalized.display_tree()); - // The normalized result should be a DictArray, not a SliceArray. - assert_eq!(normalized.encoding_id(), Dict::ID); + assert_eq!(normalized.encoding_id(), Dict.id()); assert_eq!(normalized.len(), 3); // Verify the data: codes [1,0,1] -> values [20, 10, 20] diff --git a/vortex-array/src/scalar/arrow.rs b/vortex-array/src/scalar/arrow.rs index 73142281b0d..811417144f3 100644 --- a/vortex-array/src/scalar/arrow.rs +++ b/vortex-array/src/scalar/arrow.rs @@ -455,7 +455,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("some_ext") + ExtId::new("some_ext") } fn serialize_metadata(&self, _options: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/scalar/tests/casting.rs b/vortex-array/src/scalar/tests/casting.rs index 4e6ecdd60b9..b73d4c83818 100644 --- a/vortex-array/src/scalar/tests/casting.rs +++ b/vortex-array/src/scalar/tests/casting.rs @@ -32,7 +32,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("apples") + ExtId::new("apples") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { @@ -247,7 +247,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("f16_ext") + ExtId::new("f16_ext") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { @@ -304,7 +304,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("struct_ext") + ExtId::new("struct_ext") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/scalar/typed_view/extension/tests.rs b/vortex-array/src/scalar/typed_view/extension/tests.rs index 5faa9550402..909f8a4b11a 100644 --- a/vortex-array/src/scalar/typed_view/extension/tests.rs +++ b/vortex-array/src/scalar/typed_view/extension/tests.rs @@ -21,7 +21,7 @@ impl ExtVTable for TestI32Ext { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("test_ext") + ExtId::new("test_ext") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { @@ -104,7 +104,7 @@ fn test_ext_scalar_partial_ord_different_types() { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("test_ext_2") + ExtId::new("test_ext_2") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { @@ -286,7 +286,7 @@ fn test_ext_scalar_with_metadata() { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("test_ext_metadata") + ExtId::new("test_ext_metadata") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index aaec8108808..9061b829b7c 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -185,7 +185,7 @@ impl ScalarFnVTable for Between { type Options = BetweenOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.between") + ScalarFnId::new("vortex.between") } fn serialize(&self, instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/binary/mod.rs b/vortex-array/src/scalar_fn/fns/binary/mod.rs index 83385268397..1dce67e6679 100644 --- a/vortex-array/src/scalar_fn/fns/binary/mod.rs +++ b/vortex-array/src/scalar_fn/fns/binary/mod.rs @@ -52,7 +52,7 @@ impl ScalarFnVTable for Binary { type Options = Operator; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.binary") + ScalarFnId::new("vortex.binary") } fn serialize(&self, instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index f30a17ee580..f88dbdaf356 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -79,7 +79,7 @@ impl ScalarFnVTable for CaseWhen { type Options = CaseWhenOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.case_when") + ScalarFnId::new("vortex.case_when") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/cast/mod.rs b/vortex-array/src/scalar_fn/fns/cast/mod.rs index 66a257d340b..4bb21961552 100644 --- a/vortex-array/src/scalar_fn/fns/cast/mod.rs +++ b/vortex-array/src/scalar_fn/fns/cast/mod.rs @@ -53,7 +53,7 @@ impl ScalarFnVTable for Cast { type Options = DType; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.cast") + ScalarFnId::new("vortex.cast") } fn serialize(&self, dtype: &DType) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/fill_null/mod.rs b/vortex-array/src/scalar_fn/fns/fill_null/mod.rs index 656029bf82d..16557cd77db 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/mod.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/mod.rs @@ -39,7 +39,7 @@ impl ScalarFnVTable for FillNull { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.fill_null") + ScalarFnId::new("vortex.fill_null") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/get_item.rs b/vortex-array/src/scalar_fn/fns/get_item.rs index 353ebc0925b..0b562420945 100644 --- a/vortex-array/src/scalar_fn/fns/get_item.rs +++ b/vortex-array/src/scalar_fn/fns/get_item.rs @@ -44,7 +44,7 @@ impl ScalarFnVTable for GetItem { type Options = FieldName; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.get_item") + ScalarFnId::new("vortex.get_item") } fn serialize(&self, instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/is_not_null.rs b/vortex-array/src/scalar_fn/fns/is_not_null.rs index fd0dcc53fc6..b008ebb5835 100644 --- a/vortex-array/src/scalar_fn/fns/is_not_null.rs +++ b/vortex-array/src/scalar_fn/fns/is_not_null.rs @@ -35,7 +35,7 @@ impl ScalarFnVTable for IsNotNull { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::new_ref("vortex.is_not_null") + ScalarFnId::new("vortex.is_not_null") } fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/like/mod.rs b/vortex-array/src/scalar_fn/fns/like/mod.rs index da52f730061..2288812b940 100644 --- a/vortex-array/src/scalar_fn/fns/like/mod.rs +++ b/vortex-array/src/scalar_fn/fns/like/mod.rs @@ -62,7 +62,7 @@ impl ScalarFnVTable for Like { type Options = LikeOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.like") + ScalarFnId::new("vortex.like") } fn serialize(&self, instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index b4d6c3e6bc4..331f57874f0 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -60,7 +60,7 @@ impl ScalarFnVTable for ListContains { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.list.contains") + ScalarFnId::new("vortex.list.contains") } fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/mask/mod.rs b/vortex-array/src/scalar_fn/fns/mask/mod.rs index eaf83307b35..cce76412fa6 100644 --- a/vortex-array/src/scalar_fn/fns/mask/mod.rs +++ b/vortex-array/src/scalar_fn/fns/mask/mod.rs @@ -46,7 +46,7 @@ impl ScalarFnVTable for Mask { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.mask") + ScalarFnId::new("vortex.mask") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/fns/not/mod.rs index c70c8794fe6..bdf7fdd9e76 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/fns/not/mod.rs @@ -36,7 +36,7 @@ impl ScalarFnVTable for Not { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.not") + ScalarFnId::new("vortex.not") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/root.rs b/vortex-array/src/scalar_fn/fns/root.rs index 7e04746db19..99bb6fecf6e 100644 --- a/vortex-array/src/scalar_fn/fns/root.rs +++ b/vortex-array/src/scalar_fn/fns/root.rs @@ -30,7 +30,7 @@ impl ScalarFnVTable for Root { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.root") + ScalarFnId::new("vortex.root") } fn serialize(&self, _instance: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index 2f93df5e789..c08f88f23cb 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -94,7 +94,7 @@ impl ScalarFnVTable for Select { fn child_name(&self, _instance: &FieldSelection, child_idx: usize) -> ChildName { match child_idx { - 0 => ChildName::new_ref("child"), + 0 => ChildName::from("child"), _ => unreachable!(), } } diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 5de3524be9a..54949ef5d70 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -46,7 +46,7 @@ impl ScalarFnVTable for Zip { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.zip") + ScalarFnId::new("vortex.zip") } fn serialize(&self, _options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/foreign.rs b/vortex-array/src/scalar_fn/foreign.rs index e7d3ee641cc..5cf896c129d 100644 --- a/vortex-array/src/scalar_fn/foreign.rs +++ b/vortex-array/src/scalar_fn/foreign.rs @@ -66,7 +66,7 @@ impl ScalarFnVTable for ForeignScalarFnVTable { type Options = ForeignScalarFnOptions; fn id(&self) -> ScalarFnId { - self.id.clone() + self.id } fn serialize(&self, options: &Self::Options) -> VortexResult>> { diff --git a/vortex-array/src/scalar_fn/mod.rs b/vortex-array/src/scalar_fn/mod.rs index 39e45f00e08..ce00fc08358 100644 --- a/vortex-array/src/scalar_fn/mod.rs +++ b/vortex-array/src/scalar_fn/mod.rs @@ -7,7 +7,7 @@ //! implementations. Expressions ([`crate::expr::Expression`]) reference scalar functions //! at each node. -use arcref::ArcRef; +use vortex_session::registry::Id; mod vtable; pub use vtable::*; @@ -34,7 +34,7 @@ pub mod fns; pub mod session; /// A unique identifier for a scalar function. -pub type ScalarFnId = ArcRef; +pub type ScalarFnId = Id; /// Private module to seal [`typed::DynScalarFn`]. mod sealed { diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index edbc31bdd0d..946a40518bc 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -762,8 +762,8 @@ mod tests { let ser = SerializedArray::from_array_tree(tree).unwrap(); let ctx = ReadContext::new([ - ArrayId::new_ref("vortex.test.foreign_array"), - ArrayId::new_ref("vortex.test.foreign_child"), + ArrayId::new("vortex.test.foreign_array"), + ArrayId::new("vortex.test.foreign_child"), ]); let session = VortexSession::empty() .with::() diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index ebeb94263b1..bec93d4336f 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -9,6 +9,7 @@ use itertools::zip_eq; use tracing::trace; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::arrays::Dict; use vortex::array::arrays::Primitive; use vortex::array::arrays::Slice; @@ -66,14 +67,14 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { } let id = array.encoding_id(); - if id == ALP::ID { + if id == ALP.id() { let arr = array.as_::(); return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; } - if id == BitPacked::ID { + if id == BitPacked.id() { return array.as_::().patches().is_none(); } - if id == Dict::ID { + if id == Dict.id() { let arr = array.as_::(); // Nullable codes could hold garbage values at null positions, causing // out-of-bounds shared memory reads in the DICT gather scalar op. @@ -91,7 +92,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { _ => false, }; } - if id == RunEnd::ID { + if id == RunEnd.id() { let arr = array.as_::(); // Nullable ends could hold garbage values at null positions, causing // unpredictable binary search / forward-scan behavior in the RUNEND @@ -110,11 +111,11 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { _ => false, }; } - id == FoR::ID - || id == ZigZag::ID - || id == Primitive::ID - || id == Slice::ID - || id == Sequence::ID + id == FoR.id() + || id == ZigZag.id() + || id == Primitive.id() + || id == Slice.id() + || id == Sequence.id() } /// An unmaterialized stage: a source op, scalar ops, and optional source buffer reference. @@ -435,23 +436,23 @@ impl FusedPlan { let id = array.encoding_id(); - if id == BitPacked::ID { + if id == BitPacked.id() { self.walk_bitpacked(array) - } else if id == FoR::ID { + } else if id == FoR.id() { self.walk_for(array, pending_subtrees) - } else if id == ZigZag::ID { + } else if id == ZigZag.id() { self.walk_zigzag(array, pending_subtrees) - } else if id == ALP::ID { + } else if id == ALP.id() { self.walk_alp(array, pending_subtrees) - } else if id == Dict::ID { + } else if id == Dict.id() { self.walk_dict(array, pending_subtrees) - } else if id == RunEnd::ID { + } else if id == RunEnd.id() { self.walk_runend(array, pending_subtrees) - } else if id == Primitive::ID { + } else if id == Primitive.id() { self.walk_primitive(array) - } else if id == Slice::ID { + } else if id == Slice.id() { self.walk_slice(array, pending_subtrees) - } else if id == Sequence::ID { + } else if id == Sequence.id() { self.walk_sequence(array) } else { vortex_bail!( @@ -597,7 +598,7 @@ impl FusedPlan { pending_subtrees: &mut Vec, ) -> VortexResult { let ptype = PType::try_from(child.dtype())?; - if child.encoding_id() == Primitive::ID { + if child.encoding_id() == Primitive.id() { return self.walk_primitive(child); } let buf_idx = self.source_buffers.len(); diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 2118d370d85..8234bfb8efe 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -16,6 +16,7 @@ use futures::future::BoxFuture; use tracing::debug; use tracing::trace; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; @@ -352,7 +353,7 @@ pub trait CudaArrayExt { impl CudaArrayExt for ArrayRef { #[expect(clippy::unwrap_used)] async fn execute_cuda(self, ctx: &mut CudaExecutionCtx) -> VortexResult { - if self.encoding_id() == Struct::ID { + if self.encoding_id() == Struct.id() { let len = self.len(); let StructDataParts { fields, diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 516968b8f82..ed69e235ad4 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -16,6 +16,7 @@ use futures::future::BoxFuture; use vortex::array::ArrayContext; use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::DeserializeMetadata; use vortex::array::MaskFuture; use vortex::array::ProstMetadata; @@ -127,7 +128,7 @@ impl VTable for CudaFlat { type Metadata = ProstMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.cuda_flat") + LayoutId::new("vortex.cuda_flat") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { @@ -551,7 +552,7 @@ fn extract_constant_buffers(chunk: &ArrayRef) -> Vec { let mut buffer_idx = 0u32; for array in chunk.depth_first_traversal() { let n = array.nbuffers(); - if array.encoding_id() == Constant::ID { + if array.encoding_id() == Constant.id() { for buf in array.buffers() { result.push(InlinedBuffer { buffer_index: buffer_idx, diff --git a/vortex-cuda/src/lib.rs b/vortex-cuda/src/lib.rs index 274fe31b18f..d23cbac1403 100644 --- a/vortex-cuda/src/lib.rs +++ b/vortex-cuda/src/lib.rs @@ -59,6 +59,7 @@ pub use session::CudaSession; pub use session::CudaSessionExt; pub use stream::VortexCudaStream; pub use stream_pool::VortexCudaStreamPool; +use vortex::array::ArrayVTable; use vortex::array::arrays::Constant; use vortex::array::arrays::Dict; use vortex::array::arrays::Filter; @@ -93,22 +94,22 @@ pub fn cuda_available() -> bool { /// Registers CUDA kernels. pub fn initialize_cuda(session: &CudaSession) { info!("Registering CUDA kernels"); - session.register_kernel(ALP::ID, &ALPExecutor); - session.register_kernel(BitPacked::ID, &BitPackedExecutor); - session.register_kernel(Constant::ID, &ConstantNumericExecutor); - session.register_kernel(DateTimeParts::ID, &DateTimePartsExecutor); - session.register_kernel(DecimalByteParts::ID, &DecimalBytePartsExecutor); - session.register_kernel(Dict::ID, &DictExecutor); - session.register_kernel(Shared::ID, &SharedExecutor); - session.register_kernel(FoR::ID, &FoRExecutor); - session.register_kernel(RunEnd::ID, &RunEndExecutor); - session.register_kernel(Sequence::ID, &SequenceExecutor); - session.register_kernel(ZigZag::ID, &ZigZagExecutor); - session.register_kernel(Zstd::ID, &ZstdExecutor); + session.register_kernel(ALP.id(), &ALPExecutor); + session.register_kernel(BitPacked.id(), &BitPackedExecutor); + session.register_kernel(Constant.id(), &ConstantNumericExecutor); + session.register_kernel(DateTimeParts.id(), &DateTimePartsExecutor); + session.register_kernel(DecimalByteParts.id(), &DecimalBytePartsExecutor); + session.register_kernel(Dict.id(), &DictExecutor); + session.register_kernel(Shared.id(), &SharedExecutor); + session.register_kernel(FoR.id(), &FoRExecutor); + session.register_kernel(RunEnd.id(), &RunEndExecutor); + session.register_kernel(Sequence.id(), &SequenceExecutor); + session.register_kernel(ZigZag.id(), &ZigZagExecutor); + session.register_kernel(Zstd.id(), &ZstdExecutor); #[cfg(feature = "unstable_encodings")] - session.register_kernel(ZstdBuffers::ID, &ZstdBuffersExecutor); + session.register_kernel(ZstdBuffers.id(), &ZstdBuffersExecutor); // Operation kernels - session.register_kernel(Filter::ID, &FilterExecutor); - session.register_kernel(Slice::ID, &SliceExecutor); + session.register_kernel(Filter.id(), &FilterExecutor); + session.register_kernel(Slice.id(), &SliceExecutor); } diff --git a/vortex-cuda/src/session.rs b/vortex-cuda/src/session.rs index e6bf4710dda..cbb2ff79726 100644 --- a/vortex-cuda/src/session.rs +++ b/vortex-cuda/src/session.rs @@ -85,8 +85,12 @@ impl CudaSession { /// /// * `array_id` - The encoding ID to register support for /// * `executor` - A static reference to the CUDA support implementation - pub fn register_kernel(&self, array_id: ArrayId, executor: &'static dyn CudaExecute) { - self.kernels.insert(array_id, executor); + pub fn register_kernel( + &self, + array_id: impl Into, + executor: &'static dyn CudaExecute, + ) { + self.kernels.insert(array_id.into(), executor); } /// Retrieves the CUDA support implementation for an encoding, if registered. diff --git a/vortex-duckdb/src/convert/dtype.rs b/vortex-duckdb/src/convert/dtype.rs index ad66787f02e..401b31a90c6 100644 --- a/vortex-duckdb/src/convert/dtype.rs +++ b/vortex-duckdb/src/convert/dtype.rs @@ -583,7 +583,7 @@ mod tests { type NativeValue<'a> = &'a str; fn id(&self) -> ExtId { - ExtId::new_ref("unknown.extension") + ExtId::new("unknown.extension") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-file/src/footer/mod.rs b/vortex-file/src/footer/mod.rs index e0d08568108..ac2153f7cd1 100644 --- a/vortex-file/src/footer/mod.rs +++ b/vortex-file/src/footer/mod.rs @@ -88,7 +88,7 @@ impl Footer { let layout_ids: Arc<[_]> = layout_specs .iter() .flat_map(|e| e.iter()) - .map(|encoding| LayoutEncodingId::new_arc(Arc::from(encoding.id()))) + .map(|encoding| LayoutEncodingId::new(encoding.id())) .collect(); let layout_read_ctx = ReadContext::new(layout_ids); @@ -97,7 +97,7 @@ impl Footer { let array_ids: Arc<[_]> = array_specs .iter() .flat_map(|e| e.iter()) - .map(|encoding| ArrayId::new_arc(Arc::from(encoding.id()))) + .map(|encoding| ArrayId::new(encoding.id())) .collect(); let array_read_ctx = ReadContext::new(array_ids); diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index d5f2cce336d..190461df96f 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -125,7 +125,7 @@ impl MessageDecoder { .encodings() .iter() .flat_map(|e| e.iter()) - .map(|id| ArrayId::new_arc(Arc::from(id.to_string()))) + .map(ArrayId::new) .collect(); let ctx = ReadContext::new(encoding_ids); diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index fd983e4e94c..4458a5a779c 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -2132,11 +2132,11 @@ pub type vortex_layout::ArrayFuture = futures_core::future::BoxFuture<'static, v pub type vortex_layout::LayoutContext = vortex_session::registry::Context -pub type vortex_layout::LayoutEncodingId = arcref::ArcRef +pub type vortex_layout::LayoutEncodingId = vortex_session::registry::Id pub type vortex_layout::LayoutEncodingRef = arcref::ArcRef -pub type vortex_layout::LayoutId = arcref::ArcRef +pub type vortex_layout::LayoutId = vortex_session::registry::Id pub type vortex_layout::LayoutReaderRef = alloc::sync::Arc diff --git a/vortex-layout/src/encoding.rs b/vortex-layout/src/encoding.rs index 271a94f9059..828634a8902 100644 --- a/vortex-layout/src/encoding.rs +++ b/vortex-layout/src/encoding.rs @@ -12,6 +12,7 @@ use vortex_array::dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use vortex_session::registry::Id; use vortex_session::registry::ReadContext; use crate::IntoLayout; @@ -20,7 +21,8 @@ use crate::LayoutRef; use crate::VTable; use crate::segments::SegmentId; -pub type LayoutEncodingId = ArcRef; +/// A unique identifier for a layout encoding. +pub type LayoutEncodingId = Id; pub type LayoutEncodingRef = ArcRef; pub trait LayoutEncoding: 'static + Send + Sync + Debug + private::Sealed { diff --git a/vortex-layout/src/flatbuffers.rs b/vortex-layout/src/flatbuffers.rs index 02e05b199c8..4063e23eff7 100644 --- a/vortex-layout/src/flatbuffers.rs +++ b/vortex-layout/src/flatbuffers.rs @@ -259,8 +259,8 @@ mod tests { ); let layout_ctx = ReadContext::new([ - LayoutEncodingId::new_ref("vortex.test.foreign_layout"), - LayoutEncodingId::new_ref("vortex.test.foreign_child_layout"), + LayoutEncodingId::new("vortex.test.foreign_layout"), + LayoutEncodingId::new("vortex.test.foreign_child_layout"), ]); let array_ctx = ReadContext::new([]); let layouts = LayoutSession::default().registry().clone(); diff --git a/vortex-layout/src/layout.rs b/vortex-layout/src/layout.rs index 0d805623c0f..11f0afd629c 100644 --- a/vortex-layout/src/layout.rs +++ b/vortex-layout/src/layout.rs @@ -7,7 +7,6 @@ use std::fmt::Display; use std::fmt::Formatter; use std::sync::Arc; -use arcref::ArcRef; use itertools::Itertools; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; @@ -16,6 +15,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; use vortex_session::VortexSession; +use vortex_session::registry::Id; use crate::LayoutEncodingId; use crate::LayoutEncodingRef; @@ -26,7 +26,8 @@ use crate::display::display_tree_with_segment_sizes; use crate::segments::SegmentId; use crate::segments::SegmentSource; -pub type LayoutId = ArcRef; +/// A unique identifier for a layout. +pub type LayoutId = Id; pub type LayoutRef = Arc; diff --git a/vortex-layout/src/layouts/chunked/mod.rs b/vortex-layout/src/layouts/chunked/mod.rs index af3c8bef4fa..82fcb650607 100644 --- a/vortex-layout/src/layouts/chunked/mod.rs +++ b/vortex-layout/src/layouts/chunked/mod.rs @@ -34,7 +34,7 @@ impl VTable for Chunked { type Metadata = EmptyMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.chunked") + LayoutId::new("vortex.chunked") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { diff --git a/vortex-layout/src/layouts/dict/mod.rs b/vortex-layout/src/layouts/dict/mod.rs index 866cf6587e9..7928b447fa9 100644 --- a/vortex-layout/src/layouts/dict/mod.rs +++ b/vortex-layout/src/layouts/dict/mod.rs @@ -40,7 +40,7 @@ impl VTable for Dict { type Metadata = ProstMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.dict") + LayoutId::new("vortex.dict") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 46d308fcdb8..1ffb669d08e 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -360,7 +360,7 @@ mod tests { )], Nullability::NonNullable, ); - assert!(layout.encoding_id() == LayoutId::new_ref("vortex.dict")); + assert!(layout.encoding_id() == LayoutId::new("vortex.dict")); let actual = layout .new_reader("".into(), segments, &session) .unwrap() @@ -504,7 +504,7 @@ mod tests { .unwrap(); let expression = is_not_null(root()); - assert_eq!(layout.encoding_id(), LayoutId::new_ref("vortex.dict")); + assert_eq!(layout.encoding_id(), LayoutId::new("vortex.dict")); let actual = layout .new_reader("".into(), segments, &session) .unwrap() diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs index 6fa95a5e256..186ada897a7 100644 --- a/vortex-layout/src/layouts/flat/mod.rs +++ b/vortex-layout/src/layouts/flat/mod.rs @@ -43,7 +43,7 @@ impl VTable for Flat { type Metadata = ProstMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.flat") + LayoutId::new("vortex.flat") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { diff --git a/vortex-layout/src/layouts/foreign/mod.rs b/vortex-layout/src/layouts/foreign/mod.rs index 358e2a42be1..9abacc8b26e 100644 --- a/vortex-layout/src/layouts/foreign/mod.rs +++ b/vortex-layout/src/layouts/foreign/mod.rs @@ -40,7 +40,7 @@ impl LayoutEncoding for ForeignLayoutEncoding { } fn id(&self) -> LayoutEncodingId { - self.id.clone() + self.id } fn build( @@ -57,7 +57,7 @@ impl LayoutEncoding for ForeignLayoutEncoding { .collect::>>()?; Ok(new_foreign_layout( - self.id.clone(), + self.id, dtype.clone(), row_count, metadata.to_vec(), diff --git a/vortex-layout/src/layouts/row_idx/expr.rs b/vortex-layout/src/layouts/row_idx/expr.rs index c235193ac6c..2d85d6e43f5 100644 --- a/vortex-layout/src/layouts/row_idx/expr.rs +++ b/vortex-layout/src/layouts/row_idx/expr.rs @@ -25,7 +25,7 @@ impl ScalarFnVTable for RowIdx { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.row_idx") + ScalarFnId::new("vortex.row_idx") } fn arity(&self, _options: &Self::Options) -> Arity { diff --git a/vortex-layout/src/layouts/struct_/mod.rs b/vortex-layout/src/layouts/struct_/mod.rs index 3924394ff11..29843f096d8 100644 --- a/vortex-layout/src/layouts/struct_/mod.rs +++ b/vortex-layout/src/layouts/struct_/mod.rs @@ -42,7 +42,7 @@ impl VTable for Struct { type Metadata = EmptyMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.struct") + LayoutId::new("vortex.struct") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 5167025ff3a..3c524bedead 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -46,7 +46,7 @@ impl VTable for Zoned { type Metadata = ZonedMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new_ref("vortex.stats") // For legacy reasons, this is called stats + LayoutId::new("vortex.stats") // For legacy reasons, this is called stats } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index b73fc9c6fb0..e18972bdaf2 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -49,7 +49,7 @@ impl<'py> FromPyObject<'_, 'py> for PythonArray { let python_array = ob_cast.get(); Ok(Self { vtable: PythonVTable { - id: python_array.id.clone(), + id: python_array.id, }, object: Arc::new(ob.to_owned().unbind()), len: python_array.len, diff --git a/vortex-python/src/arrays/py/mod.rs b/vortex-python/src/arrays/py/mod.rs index 761d8a3d7df..c0aab2cb98c 100644 --- a/vortex-python/src/arrays/py/mod.rs +++ b/vortex-python/src/arrays/py/mod.rs @@ -9,7 +9,6 @@ pub(crate) use array::*; use pyo3::Bound; use pyo3::PyAny; use pyo3::exceptions::PyValueError; -use pyo3::intern; use pyo3::prelude::PyAnyMethods; pub(crate) use python::*; use vortex::array::ArrayId; @@ -19,15 +18,14 @@ use crate::error::PyVortexResult; /// Extract the array id from a Python class `id` attribute. pub fn id_from_obj(cls: &Bound) -> PyVortexResult { - Ok(ArrayId::new_arc( - cls.getattr(intern!(cls.py(), "id")) - .map_err(|_| { - PyValueError::new_err(format!( - "PyEncoding subclass {cls:?} must have an 'id' attribute" - )) - })? - .extract::() - .map_err(|_| PyValueError::new_err("'id' attribute must be a string"))? - .into(), - )) + let id_str: String = cls + .getattr("id") + .map_err(|_| { + PyValueError::new_err(format!( + "PyEncoding subclass {cls:?} must have an 'id' attribute" + )) + })? + .extract() + .map_err(|_| PyValueError::new_err("'id' attribute must be a string"))?; + Ok(ArrayId::new(&id_str)) } diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index aa646e971fe..4329c19261f 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -54,7 +54,7 @@ impl VTable for PythonVTable { type ValidityVTable = Self; fn id(&self) -> ArrayId { - self.id.clone() + self.id } fn validate( diff --git a/vortex-python/src/serde/context.rs b/vortex-python/src/serde/context.rs index 6a4f6d5d3a2..9e88a523749 100644 --- a/vortex-python/src/serde/context.rs +++ b/vortex-python/src/serde/context.rs @@ -74,9 +74,7 @@ impl PyReadContext { #[new] fn new(ids: Vec) -> Self { Self(ReadContext::new( - ids.into_iter() - .map(|i| Id::new_arc(Arc::from(i))) - .collect::>(), + ids.into_iter().map(|i| Id::new(&i)).collect::>(), )) } diff --git a/vortex-session/Cargo.toml b/vortex-session/Cargo.toml index f6af94e1260..263c8c8500c 100644 --- a/vortex-session/Cargo.toml +++ b/vortex-session/Cargo.toml @@ -22,6 +22,7 @@ workspace = true [dependencies] arcref = { workspace = true } dashmap = { workspace = true } +lasso = { workspace = true } parking_lot = { workspace = true } vortex-error = { workspace = true } vortex-utils = { workspace = true, features = ["dashmap"] } diff --git a/vortex-session/public-api.lock b/vortex-session/public-api.lock index 55413e768ef..a63e955f77a 100644 --- a/vortex-session/public-api.lock +++ b/vortex-session/public-api.lock @@ -2,6 +2,18 @@ pub mod vortex_session pub mod vortex_session::registry +pub struct vortex_session::registry::CachedId + +impl vortex_session::registry::CachedId + +pub const fn vortex_session::registry::CachedId::new(s: &'static str) -> Self + +impl core::ops::deref::Deref for vortex_session::registry::CachedId + +pub type vortex_session::registry::CachedId::Target = vortex_session::registry::Id + +pub fn vortex_session::registry::CachedId::deref(&self) -> &vortex_session::registry::Id + pub struct vortex_session::registry::Context impl vortex_session::registry::Context @@ -28,6 +40,66 @@ impl core::default::Default for vortex_session::registry::Context pub fn vortex_session::registry::Context::default() -> Self +pub struct vortex_session::registry::Id(_) + +impl vortex_session::registry::Id + +pub fn vortex_session::registry::Id::as_str(&self) -> &str + +pub fn vortex_session::registry::Id::new(s: &str) -> Self + +pub fn vortex_session::registry::Id::new_static(s: &'static str) -> Self + +impl core::clone::Clone for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::clone(&self) -> vortex_session::registry::Id + +impl core::cmp::Eq for vortex_session::registry::Id + +impl core::cmp::Ord for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::cmp(&self, other: &Self) -> core::cmp::Ordering + +impl core::cmp::PartialEq for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::eq(&self, other: &vortex_session::registry::Id) -> bool + +impl core::cmp::PartialEq<&vortex_session::registry::Id> for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::eq(&self, other: &&vortex_session::registry::Id) -> bool + +impl core::cmp::PartialEq for &vortex_session::registry::Id + +pub fn &vortex_session::registry::Id::eq(&self, other: &vortex_session::registry::Id) -> bool + +impl core::cmp::PartialOrd for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::partial_cmp(&self, other: &Self) -> core::option::Option + +impl core::convert::AsRef for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::as_ref(&self) -> &str + +impl core::convert::From<&str> for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::from(s: &str) -> Self + +impl core::fmt::Debug for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl core::fmt::Display for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl core::hash::Hash for vortex_session::registry::Id + +pub fn vortex_session::registry::Id::hash<__H: core::hash::Hasher>(&self, state: &mut __H) + +impl core::marker::Copy for vortex_session::registry::Id + +impl core::marker::StructuralPartialEq for vortex_session::registry::Id + pub struct vortex_session::registry::ReadContext impl vortex_session::registry::ReadContext @@ -76,8 +148,6 @@ impl core::default::Default for vortex_session::registry::Registry pub fn vortex_session::registry::Registry::default() -> Self -pub type vortex_session::registry::Id = arcref::ArcRef - pub struct vortex_session::Ref<'a, T>(_) impl<'a, T> vortex_session::Ref<'a, T> diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 35a65b086a3..a739b9fdda3 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -4,17 +4,139 @@ //! Many session types use a registry of objects that can be looked up by name to construct //! contexts. This module provides a generic registry type for that purpose. +use std::cmp::Ordering; +use std::fmt; use std::fmt::Debug; +use std::fmt::Display; +use std::fmt::Formatter; use std::ops::Deref; use std::sync::Arc; +use std::sync::LazyLock; +use std::sync::OnceLock; -use arcref::ArcRef; +use lasso::Spur; +use lasso::ThreadedRodeo; use parking_lot::RwLock; use vortex_error::VortexExpect; use vortex_utils::aliases::dash_map::DashMap; -/// An identifier for an item in a registry. -pub type Id = ArcRef; +/// Global string interner for [`Id`] values. +static INTERNER: LazyLock = LazyLock::new(ThreadedRodeo::new); + +/// A lightweight, copyable identifier backed by a global string interner. +/// +/// Used for array encoding IDs, scalar function IDs, layout IDs, and similar +/// globally-unique string identifiers throughout Vortex. Equality and hashing +/// are O(1) symbol comparisons. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Id(Spur); + +impl Id { + /// Intern a string and return its `Id`. + pub fn new(s: &str) -> Self { + Self(INTERNER.get_or_intern(s)) + } + + /// Intern a string and return its `Id`. + pub fn new_static(s: &'static str) -> Self { + Self(INTERNER.get_or_intern_static(s)) + } + + /// Returns the interned string. + pub fn as_str(&self) -> &str { + let s = INTERNER.resolve(&self.0); + // SAFETY: INTERNER is 'static and its arena is append-only, so resolved string + // pointers are stable for the lifetime of the program. + unsafe { &*(s as *const str) } + } +} + +impl From<&str> for Id { + fn from(s: &str) -> Self { + Self::new(s) + } +} + +impl Display for Id { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Debug for Id { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "Id(\"{}\")", self.as_str()) + } +} + +impl PartialOrd for Id { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Id { + fn cmp(&self, other: &Self) -> Ordering { + self.as_str().cmp(other.as_str()) + } +} + +impl AsRef for Id { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl PartialEq<&Id> for Id { + fn eq(&self, other: &&Id) -> bool { + self == *other + } +} + +impl PartialEq for &Id { + fn eq(&self, other: &Id) -> bool { + *self == other + } +} + +/// A lazily-initialized, cached [`Id`] for use as a `static`. +/// +/// Avoids repeated interner write-lock acquisition by storing the interned [`Id`] +/// on first access and returning the cached copy on all subsequent calls. +/// +/// # Example +/// +/// ``` +/// use vortex_session::registry::{CachedId, Id}; +/// +/// static MY_ID: CachedId = CachedId::new("my.encoding"); +/// +/// fn get_id() -> Id { +/// *MY_ID +/// } +/// ``` +pub struct CachedId { + s: &'static str, + cached: OnceLock, +} + +impl CachedId { + /// Create a new `CachedId` that will intern `s` on first access. + pub const fn new(s: &'static str) -> Self { + Self { + s, + cached: OnceLock::new(), + } + } +} + +impl Deref for CachedId { + type Target = Id; + + fn deref(&self) -> &Id { + self.cached.get_or_init(|| Id::new(self.s)) + } +} /// A registry of items that are keyed by a string identifier. #[derive(Clone, Debug)] @@ -33,7 +155,7 @@ impl Registry { /// List the IDs in the registry. pub fn ids(&self) -> impl Iterator + '_ { - self.0.iter().map(|i| i.key().clone()) + self.0.iter().map(|i| *i.key()) } /// List the items in the registry. @@ -156,7 +278,7 @@ impl Context { idx < u16::MAX as usize, "Cannot have more than u16::MAX items" ); - ids.push(id.clone()); + ids.push(*id); Some(u16::try_from(idx).vortex_expect("checked already")) } diff --git a/vortex-tensor/src/fixed_shape/vtable.rs b/vortex-tensor/src/fixed_shape/vtable.rs index 21ab1ef5336..52e32ef0cf6 100644 --- a/vortex-tensor/src/fixed_shape/vtable.rs +++ b/vortex-tensor/src/fixed_shape/vtable.rs @@ -22,7 +22,7 @@ impl ExtVTable for FixedShapeTensor { type NativeValue<'a> = &'a ScalarValue; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.tensor.fixed_shape_tensor") + ExtId::new("vortex.fixed_shape_tensor") } fn serialize_metadata(&self, metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index c0f0f54fb12..181322855cf 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -139,7 +139,7 @@ impl ScalarFnVTable for L2Denorm { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::new_ref("vortex.tensor.l2_denorm") + ScalarFnId::new("vortex.tensor.l2_denorm") } fn arity(&self, _options: &Self::Options) -> Arity { diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs index 69ae671bc85..5c77b48eb87 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -48,7 +48,7 @@ impl ScalarFnVTable for SorfTransform { type Options = SorfOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::new_ref("vortex.tensor.sorf_transform") + ScalarFnId::new("vortex.tensor.sorf_transform") } fn arity(&self, _options: &Self::Options) -> Arity { diff --git a/vortex-tensor/src/vector/vtable.rs b/vortex-tensor/src/vector/vtable.rs index 2dda05b7363..19fe57fed41 100644 --- a/vortex-tensor/src/vector/vtable.rs +++ b/vortex-tensor/src/vector/vtable.rs @@ -20,7 +20,7 @@ impl ExtVTable for Vector { type NativeValue<'a> = &'a ScalarValue; fn id(&self) -> ExtId { - ExtId::new_ref("vortex.tensor.vector") + ExtId::new("vortex.tensor.vector") } fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult> { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/bool.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/bool.rs index e710438a2ce..97a05f8458b 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/bool.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/bool.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::Bool; use vortex_array::arrays::BoolArray; @@ -25,7 +26,7 @@ impl FlatLayoutFixture for BooleansFixture { } fn expected_encodings(&self) -> Vec { - vec![Bool::ID] + vec![Bool.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/chunked.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/chunked.rs index 683d5b809a3..8f5782fb36e 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/chunked.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/chunked.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::Primitive; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for ChunkedFixture { } fn expected_encodings(&self) -> Vec { - vec![Primitive::ID] + vec![Primitive.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs index 9571136ca38..792901d2bec 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::Extension; use vortex_array::arrays::PrimitiveArray; @@ -30,7 +31,7 @@ impl FlatLayoutFixture for DateTimeFixture { } fn expected_encodings(&self) -> Vec { - vec![Extension::ID] + vec![Extension.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs index d7e261cdbf4..98f857f5988 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::Decimal; use vortex_array::arrays::DecimalArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for DecimalFixture { } fn expected_encodings(&self) -> Vec { - vec![Decimal::ID] + vec![Decimal.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/fixed_size_list.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/fixed_size_list.rs index c1c2e615b76..4b10ea607cf 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/fixed_size_list.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/fixed_size_list.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::FixedSizeList; use vortex_array::arrays::FixedSizeListArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for FixedSizeListFixture { } fn expected_encodings(&self) -> Vec { - vec![FixedSizeList::ID] + vec![FixedSizeList.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/list.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/list.rs index dd09e002630..ab844ab01b7 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/list.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/list.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::List; use vortex_array::arrays::ListArray; @@ -28,7 +29,7 @@ impl FlatLayoutFixture for ListFixture { } fn expected_encodings(&self) -> Vec { - vec![List::ID] + vec![List.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs index 0a9ab2191ef..d81aaadfa5d 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::ListView; use vortex_array::arrays::ListViewArray; @@ -28,7 +29,7 @@ impl FlatLayoutFixture for ListViewFixture { } fn expected_encodings(&self) -> Vec { - vec![ListView::ID] + vec![ListView.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/null.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/null.rs index 0d937a1c225..1c7dfa25f84 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/null.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/null.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::Null; use vortex_array::arrays::NullArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for NullFixture { } fn expected_encodings(&self) -> Vec { - vec![Null::ID] + vec![Null.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/primitive.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/primitive.rs index f77d5f42f39..a84c59f3fb5 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/primitive.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/primitive.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for PrimitivesFixture { } fn expected_encodings(&self) -> Vec { - vec![Primitive::ID] + vec![Primitive.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/struct_nested.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/struct_nested.rs index cc02d856aab..d5d3eb4bd08 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/struct_nested.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/struct_nested.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::Struct; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for StructNestedFixture { } fn expected_encodings(&self) -> Vec { - vec![Struct::ID] + vec![Struct.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbin.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbin.rs index 72ac683ee44..26bbd3d554e 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbin.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbin.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBin; @@ -25,7 +26,7 @@ impl FlatLayoutFixture for VarBinFixture { } fn expected_encodings(&self) -> Vec { - vec![VarBin::ID] + vec![VarBin.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbinview.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbinview.rs index f795dafce49..ae32129a082 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbinview.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbinview.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayRef; +use vortex_array::ArrayVTable; use vortex_array::IntoArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinView; @@ -25,7 +26,7 @@ impl FlatLayoutFixture for VarBinViewFixture { } fn expected_encodings(&self) -> Vec { - vec![VarBinView::ID] + vec![VarBinView.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs index 9de554a8ee7..6cf60cf4270 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -55,7 +56,7 @@ impl FlatLayoutFixture for AlpFixture { } fn expected_encodings(&self) -> Vec { - vec![ALP::ID] + vec![ALP.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs index 977fc7cab36..04b5115e94d 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -41,7 +42,7 @@ impl FlatLayoutFixture for AlprdFixture { } fn expected_encodings(&self) -> Vec { - vec![ALPRD::ID] + vec![ALPRD.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs index c0dc53817b3..9f3b67291d9 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for BitPackedFixture { } fn expected_encodings(&self) -> Vec { - vec![BitPacked::ID] + vec![BitPacked.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs index 53cf79463d6..a410d2354c0 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for ByteBoolFixture { } fn expected_encodings(&self) -> Vec { - vec![ByteBool::ID] + vec![ByteBool.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs index 64feb357093..38e97d9f448 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::Constant; use vortex::array::arrays::ConstantArray; @@ -34,7 +35,7 @@ impl FlatLayoutFixture for ConstantFixture { } fn expected_encodings(&self) -> Vec { - vec![Constant::ID] + vec![Constant.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs index bfbf2cba21c..c77b6c6a731 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -35,7 +36,7 @@ impl FlatLayoutFixture for DateTimePartsFixture { } fn expected_encodings(&self) -> Vec { - vec![DateTimeParts::ID] + vec![DateTimeParts.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/decimal_byte_parts.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/decimal_byte_parts.rs index 723df6c5378..79f6659c065 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/decimal_byte_parts.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/decimal_byte_parts.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for DecimalBytePartsFixture { } fn expected_encodings(&self) -> Vec { - vec![DecimalByteParts::ID] + vec![DecimalByteParts.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/delta.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/delta.rs index f69e54c4631..21519d2a3b1 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/delta.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/delta.rs @@ -4,6 +4,7 @@ use vortex::VortexSessionDefault; use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; @@ -30,7 +31,7 @@ impl FlatLayoutFixture for DeltaFixture { } fn expected_encodings(&self) -> Vec { - vec![Delta::ID] + vec![Delta.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/dict.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/dict.rs index 84107b08b2a..add08b5f8e3 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/dict.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/dict.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::Dict; use vortex::array::arrays::PrimitiveArray; @@ -28,7 +29,7 @@ impl FlatLayoutFixture for DictFixture { } fn expected_encodings(&self) -> Vec { - vec![Dict::ID] + vec![Dict.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/for_.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/for_.rs index 3c1a96b6bb1..70fe9d8d8ec 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/for_.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/for_.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for FoRFixture { } fn expected_encodings(&self) -> Vec { - vec![FoR::ID] + vec![FoR.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs index 91d0f8dce07..4365fb1a01c 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinArray; @@ -28,7 +29,7 @@ impl FlatLayoutFixture for FsstFixture { } fn expected_encodings(&self) -> Vec { - vec![FSST::ID] + vec![FSST.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs index a80be0c5c13..f6902fdc2c5 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for PcoFixture { } fn expected_encodings(&self) -> Vec { - vec![Pco::ID] + vec![Pco.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs index ac0d4ba85e7..bebcb9fa12b 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for RleFixture { } fn expected_encodings(&self) -> Vec { - vec![RLE::ID] + vec![RLE.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs index e5594edcc7f..da81a2b07c4 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::PrimitiveArray; @@ -29,7 +30,7 @@ impl FlatLayoutFixture for RunEndFixture { } fn expected_encodings(&self) -> Vec { - vec![RunEnd::ID] + vec![RunEnd.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sequence.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sequence.rs index 61bd87252fb..9d9493f99c1 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sequence.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sequence.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -26,7 +27,7 @@ impl FlatLayoutFixture for SequenceFixture { } fn expected_encodings(&self) -> Vec { - vec![Sequence::ID] + vec![Sequence.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs index eeff8c36c1b..724b41abf30 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::ConstantArray; @@ -31,7 +32,7 @@ impl FlatLayoutFixture for SparseFixture { } fn expected_encodings(&self) -> Vec { - vec![Sparse::ID] + vec![Sparse.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs index ec852bf8deb..16e553854a0 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zigzag.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for ZigZagFixture { } fn expected_encodings(&self) -> Vec { - vec![ZigZag::ID] + vec![ZigZag.id()] } fn build(&self) -> VortexResult { diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs index 655cbba8c06..16736672878 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs @@ -3,6 +3,7 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; +use vortex::array::ArrayVTable; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -27,7 +28,7 @@ impl FlatLayoutFixture for ZstdFixture { } fn expected_encodings(&self) -> Vec { - vec![Zstd::ID] + vec![Zstd.id()] } fn build(&self) -> VortexResult { From c690c2ce8a84805e1d3e288fc9bff728e836af8e Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Tue, 14 Apr 2026 15:26:59 -0400 Subject: [PATCH 043/250] binstall for vortex-tui (#7434) ## Summary Adds support for downloading via `cargo binstall` for the `vx` CLI tool. This is more targeted at folks in the Rust community who have a Rust toolchain but might not be as familiar with Python toolchains. Updates documentation to reference both install pathways (`cargo binstall` and `uvx`) from our README and getting started docs --------- Signed-off-by: Andrew Duffy --- .github/workflows/release-binaries.yml | 60 ++++++++++++++++++++++++++ README.md | 9 ++-- docs/getting-started/install.md | 15 +++++++ vortex-tui/Cargo.toml | 8 ++++ 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release-binaries.yml diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml new file mode 100644 index 00000000000..8c750fe7085 --- /dev/null +++ b/.github/workflows/release-binaries.yml @@ -0,0 +1,60 @@ +name: Release Binaries + +on: + release: + types: [published] + +permissions: + contents: write + +jobs: + build: + name: Build ${{ matrix.target }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + runs-on: macos-latest + archive: tgz + - target: x86_64-apple-darwin + runs-on: macos-15-intel + archive: tgz + - target: aarch64-unknown-linux-gnu + runs-on: ubuntu-24.04-arm + archive: tgz + - target: x86_64-unknown-linux-gnu + runs-on: ubuntu-24.04 + archive: tgz + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + targets: ${{ matrix.target }} + enable-sccache: "false" + + - name: Build release binary + run: cargo build --release --package vortex-tui --bin vx --target ${{ matrix.target }} + + - name: Create archive (tgz) + if: matrix.archive == 'tgz' + run: | + cd target/${{ matrix.target }}/release + tar -czvf ../../../vx-${{ matrix.target }}.tar.gz vx + + - name: Create archive (zip) + if: matrix.archive == 'zip' + run: | + cd target/${{ matrix.target }}/release + zip ../../../vx-${{ matrix.target }}.zip vx.exe + + - name: Upload release asset + run: gh release upload "${{ github.event.release.tag_name }}" vx-${{ matrix.target }}.${{ matrix.archive == 'tgz' && 'tar.gz' || 'zip' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 92650b0db67..b36509bdc1b 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,14 @@ uv add vortex-data For browsing the structure of Vortex files, you can use the `vx` command-line tool. ```bash -# Install latest release -cargo install vortex-tui --locked +# Install pre-built binary (fast, recommended) +cargo binstall vortex-tui # Or build from source -cargo install --path vortex-tui --locked +cargo install vortex-tui --locked + +# Or run via Python without installing +uvx --from vortex-data vx --help # Usage vx browse diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index 59e808f9f75..9ff30a30fcd 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -7,6 +7,13 @@ the terminal. ::::{tab-set} +:::{tab-item} Binstall (recommended) +```bash +cargo binstall vortex-tui +``` +Downloads a pre-built binary. Requires [cargo-binstall](https://github.com/cargo-bins/cargo-binstall). +::: + :::{tab-item} pip ```bash pip install vortex-data @@ -14,10 +21,18 @@ pip install vortex-data This also installs the Python library. See the [Python quickstart](python.rst) for library usage. ::: +:::{tab-item} uvx +```bash +uvx --from vortex-data vx --help +``` +Runs the CLI without installing. Requires [uv](https://docs.astral.sh/uv/). +::: + :::{tab-item} Cargo ```bash cargo install vortex-tui ``` +Builds from source. This can be slow due to the large dependency tree. ::: :::: diff --git a/vortex-tui/Cargo.toml b/vortex-tui/Cargo.toml index ad4d0b052f2..e671ce935f5 100644 --- a/vortex-tui/Cargo.toml +++ b/vortex-tui/Cargo.toml @@ -90,5 +90,13 @@ web-sys = { version = "0.3.81", features = [ "Window", ] } +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/{ version }/vx-{ target }{ archive-suffix }" +bin-dir = "{ bin }{ binary-ext }" +pkg-fmt = "tgz" + +[package.metadata.binstall.overrides.'cfg(target_os = "windows")'] +pkg-fmt = "zip" + [lints] workspace = true From f7472016ec08393578be80f86a7afe7cb5aa38bf Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:09:35 -0400 Subject: [PATCH 044/250] Remove metadata from `DynArray` (#7435) ## Summary Apparently we just don't need this now? And we want to use `array_serialize` to get metadata instead. This also fixes a few incorrect tests that we had. ## API Changes Removes the `metadata` method on `DynArray`. ## Testing Fixes some tests. --------- Signed-off-by: Connor Tsui --- encodings/alp/src/alp/plugin.rs | 6 +- encodings/bytebool/src/array.rs | 3 +- encodings/fastlanes/src/bitpacking/plugin.rs | 6 +- encodings/parquet-variant/src/vtable.rs | 9 +- encodings/pco/src/lib.rs | 5 +- encodings/pco/src/{test.rs => tests.rs} | 2 +- encodings/runend/src/arrow.rs | 8 +- encodings/zstd/src/zstd_buffers.rs | 6 +- vortex-array/public-api.lock | 2 - vortex-array/src/array/erased.rs | 6 - vortex-array/src/array/mod.rs | 10 -- vortex-array/src/arrow/executor/run_end.rs | 6 +- vortex-array/src/serde.rs | 112 +------------------ vortex-array/src/session/mod.rs | 11 +- vortex-cuda/src/hybrid_dispatch/mod.rs | 3 +- vortex-python/src/io.rs | 4 +- 16 files changed, 44 insertions(+), 155 deletions(-) rename encodings/pco/src/{test.rs => tests.rs} (99%) diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 3b2c36ebd02..2ac3ac85c54 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -133,7 +133,7 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let metadata = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -182,7 +182,7 @@ mod tests { let array = alp_encoded.as_array(); - let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let metadata = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -213,7 +213,7 @@ mod tests { fn primitive_array_returns_error() { let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array(); - let metadata = array.metadata(&SESSION).unwrap().unwrap_or_default(); + let metadata = SESSION.array_serialize(&array).unwrap().unwrap(); let children = array.children(); let buffers = array .buffers() diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 28205d973be..e98c5bd656d 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -352,7 +352,6 @@ impl From>> for ByteBoolData { mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::assert_arrays_eq; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; @@ -407,7 +406,7 @@ mod tests { let serialized = array .clone() .into_array() - .serialize(&ctx, &LEGACY_SESSION, &SerializeOptions::default()) + .serialize(&ctx, &session, &SerializeOptions::default()) .unwrap(); let mut concat = ByteBufferMut::empty(); diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index fec101ff895..49511a10748 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -133,7 +133,7 @@ mod tests { let array = bitpacked.as_array(); - let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let metadata = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -182,7 +182,7 @@ mod tests { let array = bitpacked.as_array(); - let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let metadata = SESSION.array_serialize(array)?.unwrap(); let children = array.children(); let buffers = array .buffers() @@ -212,7 +212,7 @@ mod tests { fn primitive_array_returns_error() -> VortexResult<()> { let array = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); - let metadata = array.metadata(&SESSION)?.unwrap_or_default(); + let metadata = SESSION.array_serialize(&array)?.unwrap(); let children = array.children(); let buffers = array .buffers() diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 7dc3b3c90b2..75ea8036c9a 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -244,13 +244,13 @@ mod tests { use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::arrays::VarBinViewArray; - use vortex_array::arrays::Variant; use vortex_array::arrays::VariantArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; + use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -261,11 +261,14 @@ mod tests { use crate::ParquetVariant; use crate::array::ParquetVariantArrayExt; + fn roundtrip(array: ArrayRef) -> ArrayRef { let dtype = array.dtype().clone(); let len = array.len(); - let session = VortexSession::empty().with::(); + let session = VortexSession::empty().with::(); + session.arrays().register(ParquetVariant); + let ctx = ArrayContext::empty(); let serialized = array .serialize(&ctx, &session, &SerializeOptions::default()) @@ -276,8 +279,6 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } let concat = concat.freeze(); - session.arrays().register(ParquetVariant); - session.arrays().register(Variant); let parts = SerializedArray::try_from(concat).unwrap(); parts diff --git a/encodings/pco/src/lib.rs b/encodings/pco/src/lib.rs index 924322c2a7e..fcf9a9397fb 100644 --- a/encodings/pco/src/lib.rs +++ b/encodings/pco/src/lib.rs @@ -5,8 +5,6 @@ mod array; mod compute; mod rules; mod slice; -#[cfg(test)] -mod test; pub use array::*; @@ -35,3 +33,6 @@ pub struct PcoMetadata { #[prost(message, repeated, tag = "2")] pub chunks: Vec, } + +#[cfg(test)] +mod tests; diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/tests.rs similarity index 99% rename from encodings/pco/src/test.rs rename to encodings/pco/src/tests.rs index f674fb7d1bb..d694a0efdbe 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/tests.rs @@ -184,7 +184,7 @@ fn test_serde() -> VortexResult<()> { let bytes = pco .serialize( &context, - &LEGACY_SESSION, + &SESSION, &SerializeOptions { offset: 0, include_padding: true, diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 564c1d06cb2..206a014fa7f 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -90,6 +90,7 @@ mod tests { use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::session::ArraySession; + use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; @@ -99,8 +100,11 @@ mod tests { use crate::RunEnd; use crate::ops::find_slice_end_index; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + session.arrays().register(RunEnd); + session + }); fn decode_run_array( array: &RunArray, diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index ffeb70210ae..5d00fd5a6a4 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -59,9 +59,9 @@ impl ZstdBuffers { session: &VortexSession, ) -> VortexResult { let encoding_id = array.encoding_id(); - let metadata = array - .metadata(session)? - .ok_or_else(|| vortex_err!("Array does not support serialization"))?; + let metadata = session + .array_serialize(array)? + .ok_or_else(|| vortex_err!("[ZstdBuffers]: Array does not support serialization"))?; let buffer_handles = array.buffer_handles(); let children = array.children(); diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 8f26230791a..25736c6df84 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -22024,8 +22024,6 @@ pub fn vortex_array::ArrayRef::is_valid(&self, index: usize) -> vortex_error::Vo pub fn vortex_array::ArrayRef::len(&self) -> usize -pub fn vortex_array::ArrayRef::metadata(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> - pub fn vortex_array::ArrayRef::metadata_fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn vortex_array::ArrayRef::named_buffers(&self) -> alloc::vec::Vec<(alloc::string::String, vortex_array::buffer::BufferHandle)> diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index cfefe316ca4..f951f9423ae 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -15,7 +15,6 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_mask::Mask; -use vortex_session::VortexSession; use crate::AnyCanonical; use crate::Array; @@ -561,11 +560,6 @@ impl ArrayRef { self.0.slot_name(self, idx) } - /// Returns the serialized metadata of the array. - pub fn metadata(&self, session: &VortexSession) -> VortexResult>> { - self.0.metadata(self, session) - } - /// Formats a human-readable metadata description. pub fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { self.0.metadata_fmt(f) diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index fae2d486ab6..eada9e79a0a 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -13,7 +13,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_session::VortexSession; use vortex_session::registry::Id; use crate::ExecutionCtx; @@ -132,10 +131,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// Returns the name of the slot at the given index. fn slot_name(&self, this: &ArrayRef, idx: usize) -> String; - /// Returns the serialized metadata of the array, or `None` if the array does not - /// support serialization. - fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult>>; - /// Formats a human-readable metadata description. fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result; @@ -341,11 +336,6 @@ impl DynArray for ArrayInner { V::slot_name(view, idx) } - fn metadata(&self, this: &ArrayRef, session: &VortexSession) -> VortexResult>> { - let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::serialize(view, session) - } - fn metadata_fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.data, f) } diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-array/src/arrow/executor/run_end.rs index 69aca62a994..b9424bc866b 100644 --- a/vortex-array/src/arrow/executor/run_end.rs +++ b/vortex-array/src/arrow/executor/run_end.rs @@ -24,6 +24,7 @@ use crate::IntoArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrow::ArrowArrayExecutor; +use crate::session::ArraySessionExt; /// The encoding ID used by `vortex-runend`. We match on this string to avoid a crate dependency. const VORTEX_RUNEND_ID: &str = "vortex.runend"; @@ -79,8 +80,9 @@ fn run_end_to_arrow( ctx: &mut ExecutionCtx, ) -> VortexResult { let length = array.len(); - let metadata_bytes = array - .metadata(ctx.session())? + let metadata_bytes = ctx + .session() + .array_serialize(&array)? .ok_or_else(|| vortex_err!("RunEndArray missing metadata"))?; let metadata = RunEndMetadata::decode(&*metadata_bytes) .map_err(|e| vortex_err!("Failed to decode RunEndMetadata: {e}"))?; diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index 946a40518bc..c560aa9fb7e 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -170,16 +170,6 @@ impl<'a> ArrayNodeFlatBuffer<'a> { session: &'a VortexSession, array: &'a ArrayRef, ) -> VortexResult { - // Depth-first traversal of the array to ensure it supports serialization. - // FIXME(ngates): this serializes the metadata and throws it away! - for child in array.depth_first_traversal() { - if child.metadata(session)?.is_none() { - vortex_bail!( - "Array {} does not support serialization", - child.encoding_id() - ); - } - } let n_buffers_recursive = array.nbuffers_recursive(); if n_buffers_recursive > u16::MAX as usize { vortex_bail!( @@ -210,13 +200,13 @@ impl<'a> ArrayNodeFlatBuffer<'a> { ) })?; - let metadata = self.array.metadata(self.session)?.ok_or_else(|| { + let metadata_bytes = self.session.array_serialize(self.array)?.ok_or_else(|| { vortex_err!( "Array {} does not support serialization", self.array.encoding_id() ) })?; - let metadata = Some(fbb.create_vector(metadata.as_slice())); + let metadata = Some(fbb.create_vector(metadata_bytes.as_slice())); // Assign buffer indices for all child arrays. let nbuffers = u16::try_from(self.array.nbuffers()) @@ -701,101 +691,3 @@ impl TryFrom for SerializedArray { Self::try_from(value.try_to_host_sync()?) } } - -#[cfg(test)] -mod tests { - use std::sync::LazyLock; - - use flatbuffers::FlatBufferBuilder; - use vortex_session::VortexSession; - use vortex_session::registry::ReadContext; - - use super::SerializeOptions; - use super::SerializedArray; - use crate::ArrayContext; - use crate::array::ArrayId; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::flatbuffers as fba; - use crate::session::ArraySession; - - static SESSION: LazyLock = LazyLock::new(VortexSession::empty); - - #[test] - fn unknown_array_encoding_allow_unknown() { - let mut fbb = FlatBufferBuilder::new(); - - let child_metadata = fbb.create_vector(&[9u8]); - let child = fba::ArrayNode::create( - &mut fbb, - &fba::ArrayNodeArgs { - encoding: 1, - metadata: Some(child_metadata), - children: None, - buffers: None, - stats: None, - }, - ); - - let children = fbb.create_vector(&[child]); - let metadata = fbb.create_vector(&[1u8, 2, 3]); - let root = fba::ArrayNode::create( - &mut fbb, - &fba::ArrayNodeArgs { - encoding: 0, - metadata: Some(metadata), - children: Some(children), - buffers: None, - stats: None, - }, - ); - let array = fba::Array::create( - &mut fbb, - &fba::ArrayArgs { - root: Some(root), - buffers: None, - }, - ); - fbb.finish_minimal(array); - let (buf, start) = fbb.collapse(); - let tree = vortex_buffer::ByteBuffer::from(buf).slice(start..); - - let ser = SerializedArray::from_array_tree(tree).unwrap(); - let ctx = ReadContext::new([ - ArrayId::new("vortex.test.foreign_array"), - ArrayId::new("vortex.test.foreign_child"), - ]); - let session = VortexSession::empty() - .with::() - .allow_unknown(); - - let decoded = ser - .decode(&DType::Variant(Nullability::Nullable), 5, &ctx, &session) - .unwrap(); - assert_eq!(decoded.encoding_id().as_ref(), "vortex.test.foreign_array"); - assert_eq!(decoded.nchildren(), 1); - assert_eq!( - decoded.nth_child(0).unwrap().encoding_id().as_ref(), - "vortex.test.foreign_child" - ); - assert_eq!(decoded.metadata(&SESSION).unwrap().unwrap(), vec![1, 2, 3]); - assert_eq!( - decoded - .nth_child(0) - .unwrap() - .metadata(&SESSION) - .unwrap() - .unwrap(), - vec![9] - ); - - let serialized = decoded - .serialize( - &ArrayContext::default(), - &SESSION, - &SerializeOptions::default(), - ) - .unwrap(); - assert!(!serialized.is_empty()); - } -} diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index 62ed46cb34c..034a369694f 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::registry::Registry; @@ -15,6 +16,7 @@ use crate::arrays::Bool; use crate::arrays::Chunked; use crate::arrays::Constant; use crate::arrays::Decimal; +use crate::arrays::Dict; use crate::arrays::Extension; use crate::arrays::FixedSizeList; use crate::arrays::List; @@ -26,6 +28,7 @@ use crate::arrays::Primitive; use crate::arrays::Struct; use crate::arrays::VarBin; use crate::arrays::VarBinView; +use crate::arrays::Variant; pub type ArrayRegistry = Registry; @@ -68,11 +71,13 @@ impl Default for ArraySession { this.register(ListView); this.register(FixedSizeList); this.register(Struct); + this.register(Variant); this.register(Extension); // Register the utility encodings. this.register(Chunked); this.register(Constant); + this.register(Dict); this.register(List); this.register(Masked); this.register(Patched); @@ -92,8 +97,12 @@ pub trait ArraySessionExt: SessionExt { /// Serialize an array using a plugin from the registry. fn array_serialize(&self, array: &ArrayRef) -> VortexResult>> { let Some(plugin) = self.arrays().registry.find(&array.encoding_id()) else { - return Ok(None); + vortex_bail!( + "Array {} is not registered for serializations", + array.encoding_id() + ); }; + plugin.serialize(array, &self.session()) } } diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 0845a46fdc3..a5ca2560d43 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -257,8 +257,7 @@ mod tests { 0u32.into(), ) .vortex_expect("for"); - let vals = ZstdBuffers::compress(&vals.into_array(), 3, &VortexSession::empty()) - .vortex_expect("zstd"); + let vals = ZstdBuffers::compress(&vals.into_array(), 3, &session).vortex_expect("zstd"); // codes = FoR(BitPacked) let codes = PrimitiveArray::new( diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index edf1061188d..1d1cb63cce7 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -280,7 +280,7 @@ impl PyVortexWriteOptions { /// >>> vx.io.VortexWriteOptions.default().write(sprl, "chonky.vortex") /// >>> import os /// >>> os.path.getsize('chonky.vortex') - /// 216004 + /// 216036 /// ``` /// /// Wow, Vortex manages to use about two bytes per integer! So advanced. So tiny. @@ -292,7 +292,7 @@ impl PyVortexWriteOptions { /// ```python /// >>> vx.io.VortexWriteOptions.compact().write(sprl, "tiny.vortex") /// >>> os.path.getsize('tiny.vortex') - /// 55120 + /// 55152 /// ``` /// /// Random numbers are not (usually) composed of random bytes! From c4ad4e2addf9dd435143205d67318b2462eb523e Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 14 Apr 2026 18:54:07 -0400 Subject: [PATCH 045/250] Remove ListViewBuilder scalar_at (#7438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit | Benchmark | Before (median) | After (median) | Speedup | |---|---:|---:|---:| | `extend_from_array_non_zctl_overlapping (1000, 8)` | 902.2 µs | 63.66 µs | 14.17x | | `extend_from_array_non_zctl_overlapping (1000, 32)` | 2.446 ms | 76.54 µs | 31.96x | | `extend_from_array_non_zctl_overlapping (10000, 8)` | 9.821 ms | 561.5 µs | 17.49x | | `extend_from_array_zctl (1000, 8)` | 1.057 ms | 13.95 µs | 75.77x | | `extend_from_array_zctl (1000, 64)` | 5.416 ms | 38.29 µs | 141.45x | | `extend_from_array_zctl (10000, 8)` | 10.52 ms | 111.3 µs | 94.52x | Signed-off-by: Nicholas Gates --- vortex-array/Cargo.toml | 4 + .../benches/listview_builder_extend.rs | 97 +++++++++++++++++++ vortex-array/src/builders/listview.rs | 75 +++++++++----- 3 files changed, 152 insertions(+), 24 deletions(-) create mode 100644 vortex-array/benches/listview_builder_extend.rs diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index e324c70aadb..f9adbeb99db 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -187,3 +187,7 @@ harness = false [[bench]] name = "listview_rebuild" harness = false + +[[bench]] +name = "listview_builder_extend" +harness = false diff --git a/vortex-array/benches/listview_builder_extend.rs b/vortex-array/benches/listview_builder_extend.rs new file mode 100644 index 00000000000..75487564ea8 --- /dev/null +++ b/vortex-array/benches/listview_builder_extend.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::cast_possible_wrap)] + +use std::sync::Arc; + +use divan::Bencher; +use vortex_array::IntoArray; +use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builders::ArrayBuilder; +use vortex_array::builders::ListViewBuilder; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability::NonNullable; +use vortex_array::dtype::Nullability::Nullable; +use vortex_array::dtype::PType::I32; +use vortex_array::validity::Validity; +use vortex_buffer::Buffer; + +fn main() { + divan::main(); +} + +const ZCTL_ARGS: &[(usize, usize)] = &[ + // num_lists, list_size + (1_000, 8), + (1_000, 64), + (10_000, 8), +]; + +const NON_ZCTL_ARGS: &[(usize, usize)] = &[ + // num_lists, list_size + (1_000, 8), + (1_000, 32), + (10_000, 8), +]; + +fn make_listview( + num_lists: usize, + list_size: usize, + step: usize, + with_nulls: bool, +) -> ListViewArray { + let element_count = step * num_lists + list_size; + let elements = PrimitiveArray::from_iter(0..element_count as i32).into_array(); + let offsets: Buffer = (0..num_lists).map(|i| (i * step) as u32).collect(); + let sizes: Buffer = std::iter::repeat_n(list_size as u16, num_lists).collect(); + let validity = if with_nulls { + Validity::from_iter((0..num_lists).map(|i| i % 5 != 0)) + } else { + Validity::NonNullable + }; + + ListViewArray::new(elements, offsets.into_array(), sizes.into_array(), validity) +} + +#[divan::bench(args = ZCTL_ARGS)] +fn extend_from_array_zctl(bencher: Bencher, (num_lists, list_size): (usize, usize)) { + let source = make_listview(num_lists, list_size, list_size, false); + debug_assert!(source.is_zero_copy_to_list()); + let source = source.into_array(); + + bencher.with_inputs(|| &source).bench_refs(|source| { + let mut builder = ListViewBuilder::::with_capacity( + Arc::new(DType::Primitive(I32, NonNullable)), + NonNullable, + num_lists * list_size, + num_lists, + ); + builder.extend_from_array(source); + divan::black_box(builder.finish_into_listview()) + }); +} + +#[divan::bench(args = NON_ZCTL_ARGS)] +fn extend_from_array_non_zctl_overlapping( + bencher: Bencher, + (num_lists, list_size): (usize, usize), +) { + // `step = 1` creates heavily overlapping lists, which forces the non-ZCTL extend path. + let source = make_listview(num_lists, list_size, 1, true); + debug_assert!(!source.is_zero_copy_to_list()); + let source = source.into_array(); + + bencher.with_inputs(|| &source).bench_refs(|source| { + let mut builder = ListViewBuilder::::with_capacity( + Arc::new(DType::Primitive(I32, NonNullable)), + Nullable, + num_lists * list_size, + num_lists, + ); + builder.extend_from_array(source); + divan::black_box(builder.finish_into_listview()) + }); +} diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index b339dc430f7..a694bbbf035 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -296,33 +296,15 @@ impl ArrayBuilder for ListViewBuilder { return; } - // If we do not have the guarantee that the array is zero-copyable to a list, then we have - // to manually append each scalar. - if !listview.is_zero_copy_to_list() { - for i in 0..listview.len() { - let list = listview - .scalar_at(i) - .vortex_expect("scalar_at failed in extend_from_array_unchecked"); - - self.append_scalar(&list) - .vortex_expect("was unable to extend the `ListViewBuilder`") - } - - return; - } - - // Otherwise, after removing any leading and trailing elements, we can simply bulk append - // the entire array. + // Normalize to an exact zero-copy-to-list layout and then bulk append. This avoids the + // very expensive scalar_at-per-list path for overlapping / out-of-order list views. let listview = listview .rebuild(ListViewRebuildMode::MakeExact) .vortex_expect("ListViewArray::rebuild(MakeExact) failed in extend_from_array"); debug_assert!(listview.is_zero_copy_to_list()); - self.nulls.append_validity_mask( - array - .validity_mask() - .vortex_expect("validity_mask in extend_from_array_unchecked"), - ); + self.nulls + .append_validity_mask(listview.listview_validity_mask()); // Bulk append the new elements (which should have no gaps or overlaps). let old_elements_len = self.elements_builder.len(); @@ -429,11 +411,13 @@ fn adjust_and_extend_offsets<'a, O: IntegerPType, A: IntegerPType>( mod tests { use std::sync::Arc; + use vortex_buffer::buffer; use vortex_error::VortexExpect; use super::ListViewBuilder; use crate::IntoArray; use crate::arrays::ListArray; + use crate::arrays::ListViewArray; use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; @@ -443,6 +427,7 @@ mod tests { use crate::dtype::Nullability::Nullable; use crate::dtype::PType::I32; use crate::scalar::Scalar; + use crate::validity::Validity; #[test] fn test_empty() { @@ -696,6 +681,50 @@ mod tests { ); } + #[test] + fn test_extend_from_array_overlapping_listview() { + let dtype: Arc = Arc::new(I32.into()); + + // Non-ZCTL source: + // - List 0: [10, 20] + // - List 1: null (size is intentionally non-zero in source metadata) + // - List 2: [10] + let source = unsafe { + ListViewArray::new_unchecked( + buffer![10i32, 20, 30].into_array(), + buffer![0u32, 1, 0].into_array(), + buffer![2u8, 2, 1].into_array(), + Validity::from_iter([true, false, true]), + ) + }; + assert!(!source.is_zero_copy_to_list()); + + let mut builder = + ListViewBuilder::::with_capacity(Arc::clone(&dtype), Nullable, 0, 0); + builder.extend_from_array(&source.into_array()); + + let listview = builder.finish_into_listview(); + assert_eq!(listview.len(), 3); + assert!(listview.is_zero_copy_to_list()); + + assert_arrays_eq!( + listview.list_elements_at(0).unwrap(), + PrimitiveArray::from_iter([10i32, 20]) + ); + assert!( + !listview + .validity() + .vortex_expect("listview validity should be derivable") + .is_valid(1) + .unwrap() + ); + assert_eq!(listview.list_elements_at(1).unwrap().len(), 0); + assert_arrays_eq!( + listview.list_elements_at(2).unwrap(), + PrimitiveArray::from_iter([10i32]) + ); + } + #[test] fn test_error_append_null_to_non_nullable() { let dtype: Arc = Arc::new(I32.into()); @@ -719,8 +748,6 @@ mod tests { #[test] fn test_append_array_as_list() { - use vortex_buffer::buffer; - let dtype: Arc = Arc::new(I32.into()); let mut builder = ListViewBuilder::::with_capacity(Arc::clone(&dtype), NonNullable, 20, 10); From 4a5b7d73ac16cf6e7fc142559c2df4355c8ac87c Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 14 Apr 2026 21:43:41 -0400 Subject: [PATCH 046/250] Support serializion tensor scalar fns (#7437) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Adds serialization support to the tensor scalar fns. This is gated by the `VORTEX_TENSOR_SCALAR_FN_ARRAY_PLUGINS` environment variable. ## Testing Basic testing. ## Unresolved Questions - I should probably write this to a file and roundtrip that, right? - It is also not clear if it makes sense to have a `SorfTransformMetadata` when it is basically identical to `SorfOptions` but with prost. --------- Signed-off-by: Connor Tsui --- vortex-tensor/public-api.lock | 30 ++++ .../src/encodings/turboquant/tests/mod.rs | 8 +- vortex-tensor/src/lib.rs | 35 ++++- .../src/scalar_fns/cosine_similarity.rs | 85 +++++++++- vortex-tensor/src/scalar_fns/inner_product.rs | 148 +++++++++++++++++- vortex-tensor/src/scalar_fns/l2_denorm.rs | 112 ++++++++++++- vortex-tensor/src/scalar_fns/l2_norm.rs | 89 ++++++++++- .../src/scalar_fns/sorf_transform/tests.rs | 66 +++++++- .../src/scalar_fns/sorf_transform/vtable.rs | 93 +++++++++++ 9 files changed, 624 insertions(+), 42 deletions(-) diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 4f6c2dddbfb..01ff0a6f477 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -248,6 +248,12 @@ impl core::clone::Clone for vortex_tensor::scalar_fns::cosine_similarity::Cosine pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::clone(&self) -> vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity + +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::deserialize(&self, _dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity pub type vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::Options = vortex_array::scalar_fn::vtable::EmptyOptions @@ -284,6 +290,12 @@ impl core::clone::Clone for vortex_tensor::scalar_fns::inner_product::InnerProdu pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::clone(&self) -> vortex_tensor::scalar_fns::inner_product::InnerProduct +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::inner_product::InnerProduct + +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::deserialize(&self, _dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::inner_product::InnerProduct pub type vortex_tensor::scalar_fns::inner_product::InnerProduct::Options = vortex_array::scalar_fn::vtable::EmptyOptions @@ -322,6 +334,12 @@ impl core::clone::Clone for vortex_tensor::scalar_fns::l2_denorm::L2Denorm pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::clone(&self) -> vortex_tensor::scalar_fns::l2_denorm::L2Denorm +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::l2_denorm::L2Denorm + +pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::deserialize(&self, _dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::l2_denorm::L2Denorm pub type vortex_tensor::scalar_fns::l2_denorm::L2Denorm::Options = vortex_array::scalar_fn::vtable::EmptyOptions @@ -362,6 +380,12 @@ impl core::clone::Clone for vortex_tensor::scalar_fns::l2_norm::L2Norm pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::clone(&self) -> vortex_tensor::scalar_fns::l2_norm::L2Norm +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::l2_norm::L2Norm + +pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::deserialize(&self, _dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::l2_norm::L2Norm pub type vortex_tensor::scalar_fns::l2_norm::L2Norm::Options = vortex_array::scalar_fn::vtable::EmptyOptions @@ -444,6 +468,12 @@ impl core::clone::Clone for vortex_tensor::scalar_fns::sorf_transform::SorfTrans pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::clone(&self) -> vortex_tensor::scalar_fns::sorf_transform::SorfTransform +impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::sorf_transform::SorfTransform + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> + +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> + impl vortex_array::scalar_fn::vtable::ScalarFnVTable for vortex_tensor::scalar_fns::sorf_transform::SorfTransform pub type vortex_tensor::scalar_fns::sorf_transform::SorfTransform::Options = vortex_tensor::scalar_fns::sorf_transform::SorfOptions diff --git a/vortex-tensor/src/encodings/turboquant/tests/mod.rs b/vortex-tensor/src/encodings/turboquant/tests/mod.rs index 4c01affa27d..b111c6e28ba 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/mod.rs @@ -8,8 +8,6 @@ mod nullable; mod roundtrip; mod structural; -use std::sync::LazyLock; - use rand::SeedableRng; use rand::rngs::StdRng; use rand_distr::Distribution; @@ -29,22 +27,18 @@ use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::dtype::extension::ExtDType; use vortex_array::extension::EmptyMetadata; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_session::VortexSession; use crate::encodings::turboquant::TurboQuantConfig; use crate::encodings::turboquant::turboquant_encode_unchecked; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use crate::tests::SESSION; use crate::vector::Vector; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Create a FixedSizeListArray of random f32 vectors with the given validity. fn make_fsl_with_validity( num_rows: usize, diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index b3cf6c21695..d2fb02a6c7f 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -5,8 +5,10 @@ //! including unit vectors, spherical coordinates, and similarity measures such as cosine //! similarity. +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; +use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; use crate::fixed_shape::FixedShapeTensor; @@ -34,9 +36,32 @@ pub fn initialize(session: &VortexSession) { session.dtypes().register(Vector); session.dtypes().register(FixedShapeTensor); - session.scalar_fns().register(CosineSimilarity); - session.scalar_fns().register(InnerProduct); - session.scalar_fns().register(L2Denorm); - session.scalar_fns().register(L2Norm); - session.scalar_fns().register(SorfTransform); + let session_fns = session.scalar_fns(); + let session_arrays = session.arrays(); + + session_fns.register(CosineSimilarity); + session_fns.register(InnerProduct); + session_fns.register(L2Denorm); + session_fns.register(L2Norm); + session_fns.register(SorfTransform); + + session_arrays.register(ScalarFnArrayPlugin::new(CosineSimilarity)); + session_arrays.register(ScalarFnArrayPlugin::new(InnerProduct)); + session_arrays.register(ScalarFnArrayPlugin::new(L2Denorm)); + session_arrays.register(ScalarFnArrayPlugin::new(L2Norm)); + session_arrays.register(ScalarFnArrayPlugin::new(SorfTransform)); +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use vortex_array::session::ArraySession; + use vortex_session::VortexSession; + + pub static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + crate::initialize(&session); + session + }); } diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 2a717b90b7b..6c06d165a0e 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -12,6 +12,9 @@ use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -25,11 +28,14 @@ use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure; +use vortex_session::VortexSession; +use crate::scalar_fns::inner_product::BinaryTensorOpMetadata; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::try_build_constant_l2_denorm; @@ -221,6 +227,37 @@ impl ScalarFnVTable for CosineSimilarity { } } +impl ScalarFnArrayVTable for CosineSimilarity { + fn serialize( + &self, + view: &ScalarFnArrayView, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(BinaryTensorOpMetadata::encode_from_view(view)?)) + } + + fn deserialize( + &self, + _dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult> { + let reconstructed = BinaryTensorOpMetadata::decode_children( + metadata, + len, + children, + session, + "CosineSimilarity", + )?; + Ok(ScalarFnArrayParts { + options: EmptyOptions, + children: reconstructed, + }) + } +} + impl CosineSimilarity { /// Both sides are `L2Denorm`: treat the normalized children as authoritative, so /// `cosine_similarity = dot(n_l, n_r)`. @@ -292,31 +329,28 @@ impl CosineSimilarity { #[cfg(test)] mod tests { - use std::sync::LazyLock; use rstest::rstest; + use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::session::ArraySession; + use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::validity::Validity; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::scalar_fns::cosine_similarity::CosineSimilarity; use crate::scalar_fns::l2_denorm::L2Denorm; + use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; use crate::utils::test_helpers::constant_vector_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Evaluates cosine similarity between two tensor arrays and returns the result as `Vec`. fn eval_cosine_similarity(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { let scalar_fn = CosineSimilarity::new().erased(); @@ -693,4 +727,43 @@ mod tests { ); Ok(()) } + + #[rstest] + #[case::vector( + vector_array(3, &[1.0, 0.0, 0.0, 3.0, 4.0, 0.0]).unwrap(), + vector_array(3, &[0.0, 1.0, 0.0, 3.0, 4.0, 0.0]).unwrap(), + 2, + )] + #[case::fixed_shape_tensor( + tensor_array(&[2], &[1.0, 0.0, 3.0, 4.0]).unwrap(), + tensor_array(&[2], &[0.0, 1.0, 3.0, 4.0]).unwrap(), + 2, + )] + fn serde_round_trip( + #[case] lhs: ArrayRef, + #[case] rhs: ArrayRef, + #[case] len: usize, + ) -> VortexResult<()> { + let original = CosineSimilarity::try_new_array(lhs.clone(), rhs.clone(), len)?.into_array(); + + let plugin = ScalarFnArrayPlugin::new(CosineSimilarity); + let metadata = plugin + .serialize(&original, &SESSION)? + .expect("CosineSimilarity serialize must produce metadata"); + + let children = vec![lhs, rhs]; + let recovered = plugin.deserialize( + original.dtype(), + original.len(), + &metadata, + &[], + &children, + &SESSION, + )?; + + assert_eq!(recovered.dtype(), original.dtype()); + assert_eq!(recovered.len(), original.len()); + assert_eq!(recovered.encoding_id(), original.encoding_id()); + Ok(()) + } } diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 0d6b2bf76aa..3f4678626bd 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -7,6 +7,7 @@ use std::fmt::Formatter; use std::sync::Arc; use num_traits::Float; +use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -18,15 +19,21 @@ use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeList; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::extension::ExtDType; +use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::expr::and; use vortex_array::extension::EmptyMetadata; @@ -39,12 +46,14 @@ use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::serde::ArrayChildren; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_denorm::L2Denorm; @@ -245,6 +254,97 @@ impl ScalarFnVTable for InnerProduct { } } +/// Metadata for a serialized binary tensor-op array (shared by [`InnerProduct`] and +/// [`CosineSimilarity`]). Both operands share the same extension dtype up to nullability +/// (enforced by their `return_dtype` checks), but their individual nullabilities are lost in the +/// parent's unioned output, so both are persisted. +/// +/// [`CosineSimilarity`]: crate::scalar_fns::cosine_similarity::CosineSimilarity +#[derive(Clone, prost::Message)] +pub(crate) struct BinaryTensorOpMetadata { + #[prost(message, optional, tag = "1")] + pub(crate) lhs_dtype: Option, + #[prost(message, optional, tag = "2")] + pub(crate) rhs_dtype: Option, +} + +impl BinaryTensorOpMetadata { + /// Encodes the two children of `view` into a [`BinaryTensorOpMetadata`] byte blob. + pub(crate) fn encode_from_view( + view: &ScalarFnArrayView, + ) -> VortexResult> { + let scalar_fn_array = view.as_::(); + let lhs_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); + let rhs_dtype = Some(scalar_fn_array.child_at(1).dtype().try_into()?); + Ok(Self { + lhs_dtype, + rhs_dtype, + } + .encode_to_vec()) + } + + /// Decodes `metadata` and fetches both children from `children` using the decoded dtypes, + /// validating that `lhs` and `rhs` agree modulo nullability. + pub(crate) fn decode_children( + metadata: &[u8], + len: usize, + children: &dyn ArrayChildren, + session: &VortexSession, + scalar_fn_name: &str, + ) -> VortexResult> { + let metadata = Self::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode BinaryTensorOpMetadata: {e}"))?; + let lhs_pb = metadata + .lhs_dtype + .as_ref() + .ok_or_else(|| vortex_err!("{scalar_fn_name} metadata missing lhs_dtype"))?; + let rhs_pb = metadata + .rhs_dtype + .as_ref() + .ok_or_else(|| vortex_err!("{scalar_fn_name} metadata missing rhs_dtype"))?; + let lhs_dtype = DType::from_proto(lhs_pb, session)?; + let rhs_dtype = DType::from_proto(rhs_pb, session)?; + vortex_ensure!( + lhs_dtype.eq_ignore_nullability(&rhs_dtype), + "{scalar_fn_name} operand dtype mismatch: {lhs_dtype} vs {rhs_dtype}" + ); + let lhs = children.get(0, &lhs_dtype, len)?; + let rhs = children.get(1, &rhs_dtype, len)?; + Ok(vec![lhs, rhs]) + } +} + +impl ScalarFnArrayVTable for InnerProduct { + fn serialize( + &self, + view: &ScalarFnArrayView, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(BinaryTensorOpMetadata::encode_from_view(view)?)) + } + + fn deserialize( + &self, + _dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult> { + let reconstructed = BinaryTensorOpMetadata::decode_children( + metadata, + len, + children, + session, + "InnerProduct", + )?; + Ok(ScalarFnArrayParts { + options: EmptyOptions, + children: reconstructed, + }) + } +} + impl InnerProduct { /// Both sides are `L2Denorm`: `inner_product = s_l * s_r * dot(n_l, n_r)`. fn execute_both_denorm( @@ -592,29 +692,26 @@ fn execute_dict_constant_inner_product( #[cfg(test)] mod tests { - use std::sync::LazyLock; use rstest::rstest; + use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::session::ArraySession; + use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::validity::Validity; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; + use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Evaluates inner product between two tensor arrays and returns the result as `Vec`. fn eval_inner_product(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { let scalar_fn = InnerProduct::new().erased(); @@ -810,6 +907,45 @@ mod tests { Ok(()) } + #[rstest] + #[case::vector( + vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), + vector_array(3, &[7.0, 8.0, 9.0, 10.0, 11.0, 12.0]).unwrap(), + 2, + )] + #[case::fixed_shape_tensor( + tensor_array(&[2], &[1.0, 2.0, 3.0, 4.0]).unwrap(), + tensor_array(&[2], &[5.0, 6.0, 7.0, 8.0]).unwrap(), + 2, + )] + fn serde_round_trip( + #[case] lhs: ArrayRef, + #[case] rhs: ArrayRef, + #[case] len: usize, + ) -> VortexResult<()> { + let original = InnerProduct::try_new_array(lhs.clone(), rhs.clone(), len)?.into_array(); + + let plugin = ScalarFnArrayPlugin::new(InnerProduct); + let metadata = plugin + .serialize(&original, &SESSION)? + .expect("InnerProduct serialize must produce metadata"); + + let children = vec![lhs, rhs]; + let recovered = plugin.deserialize( + original.dtype(), + original.len(), + &metadata, + &[], + &children, + &SESSION, + )?; + + assert_eq!(recovered.dtype(), original.dtype()); + assert_eq!(recovered.len(), original.len()); + assert_eq!(recovered.encoding_id(), original.encoding_id()); + Ok(()) + } + // ---- Tests for the `SorfTransform + constant` and `Dict + constant` fast paths ---- #[allow( diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 181322855cf..01164a71dfe 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -8,6 +8,7 @@ use std::fmt::Formatter; use num_traits::Float; use num_traits::ToPrimitive; use num_traits::Zero; +use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -18,13 +19,19 @@ use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; +use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::expr::and; use vortex_array::match_each_float_ptype; @@ -38,6 +45,7 @@ use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -46,6 +54,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; +use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_norm::L2Norm; @@ -272,6 +282,64 @@ impl ScalarFnVTable for L2Denorm { } } +/// Metadata for a serialized [`L2Denorm`] array: both children's full [`DType`]s. The parent's +/// dtype is `normalized.union_nullability(norms.nullability())`, which loses both children's +/// individual nullabilities, so we persist them directly. +#[derive(Clone, prost::Message)] +pub(super) struct L2DenormMetadata { + #[prost(message, optional, tag = "1")] + normalized_dtype: Option, + #[prost(message, optional, tag = "2")] + norms_dtype: Option, +} + +impl ScalarFnArrayVTable for L2Denorm { + fn serialize( + &self, + view: &ScalarFnArrayView, + _session: &VortexSession, + ) -> VortexResult>> { + let scalar_fn_array = view.as_::(); + let normalized_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); + let norms_dtype = Some(scalar_fn_array.child_at(1).dtype().try_into()?); + Ok(Some( + L2DenormMetadata { + normalized_dtype, + norms_dtype, + } + .encode_to_vec(), + )) + } + + fn deserialize( + &self, + _dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult> { + let metadata = L2DenormMetadata::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode L2DenormMetadata: {e}"))?; + let normalized_pb = metadata + .normalized_dtype + .as_ref() + .ok_or_else(|| vortex_err!("L2DenormMetadata missing normalized_dtype"))?; + let norms_pb = metadata + .norms_dtype + .as_ref() + .ok_or_else(|| vortex_err!("L2DenormMetadata missing norms_dtype"))?; + let normalized_dtype = DType::from_proto(normalized_pb, session)?; + let norms_dtype = DType::from_proto(norms_pb, session)?; + let normalized = children.get(0, &normalized_dtype, len)?; + let norms = children.get(1, &norms_dtype, len)?; + Ok(ScalarFnArrayParts { + options: EmptyOptions, + children: vec![normalized, norms], + }) + } +} + /// Optimized execution when the norms array is constant. fn execute_l2_denorm_constant_norms( normalized_ref: ArrayRef, @@ -631,8 +699,9 @@ fn validate_l2_normalized_rows_impl( #[cfg(test)] mod tests { - use std::sync::LazyLock; + use rstest::rstest; + use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; @@ -646,6 +715,7 @@ mod tests { use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; + use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; @@ -653,17 +723,16 @@ mod tests { use vortex_array::extension::datetime::Date; use vortex_array::extension::datetime::TimeUnit; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::fixed_shape::FixedShapeTensor; use crate::fixed_shape::FixedShapeTensorMetadata; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows; + use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; use crate::utils::test_helpers::constant_vector_array; @@ -671,9 +740,6 @@ mod tests { use crate::utils::test_helpers::vector_array; use crate::vector::Vector; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Evaluates L2 denorm on a tensor/vector array and returns the executed array. fn eval_l2_denorm(normalized: ArrayRef, norms: ArrayRef, len: usize) -> VortexResult { let mut ctx = SESSION.create_execution_ctx(); @@ -1032,4 +1098,38 @@ mod tests { assert_tensor_arrays_eq(actual, expected)?; Ok(()) } + + /// Build an `L2Denorm` array from a raw input (which may have nullable storage) by running + /// `normalize_as_l2_denorm`. The normalized child ends up non-nullable, and the norms child + /// inherits the input's nullability, giving us two different per-child nullabilities to + /// round-trip. + #[rstest] + #[case::vector(vector_array(3, &[3.0, 4.0, 0.0, 0.0, 0.0, 0.0]).unwrap())] + #[case::fixed_shape_tensor(tensor_array(&[2, 2], &[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0]).unwrap())] + fn serde_round_trip(#[case] input: ArrayRef) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let original = normalize_as_l2_denorm(input, &mut ctx)?.into_array(); + + let scalar_fn_array = original.as_::(); + let children = scalar_fn_array.children(); + + let plugin = ScalarFnArrayPlugin::new(L2Denorm); + let metadata = plugin + .serialize(&original, &SESSION)? + .expect("L2Denorm serialize must produce metadata"); + + let recovered = plugin.deserialize( + original.dtype(), + original.len(), + &metadata, + &[], + &children, + &SESSION, + )?; + + assert_eq!(recovered.dtype(), original.dtype()); + assert_eq!(recovered.len(), original.len()); + assert_eq!(recovered.encoding_id(), original.encoding_id()); + Ok(()) + } } diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 59bd9d77009..13245dd880b 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -6,17 +6,24 @@ use std::fmt::Formatter; use num_traits::Float; +use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; +use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; +use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::match_each_float_ptype; use vortex_array::scalar_fn::Arity; @@ -26,10 +33,13 @@ use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::serde::ArrayChildren; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; +use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_denorm::L2Denorm; @@ -172,6 +182,49 @@ impl ScalarFnVTable for L2Norm { } } +/// Metadata for a serialized [`L2Norm`] array: the single `input` child's [`DType`], which carries +/// the extension type (`FixedShapeTensor` vs `Vector`), dimension, and nullability that are not +/// recoverable from the parent's primitive-float output. +#[derive(Clone, prost::Message)] +pub(super) struct L2NormMetadata { + #[prost(message, optional, tag = "1")] + input_dtype: Option, +} + +impl ScalarFnArrayVTable for L2Norm { + fn serialize( + &self, + view: &ScalarFnArrayView, + _session: &VortexSession, + ) -> VortexResult>> { + let scalar_fn_array = view.as_::(); + let input_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); + Ok(Some(L2NormMetadata { input_dtype }.encode_to_vec())) + } + + fn deserialize( + &self, + _dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + session: &VortexSession, + ) -> VortexResult> { + let metadata = L2NormMetadata::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode L2NormMetadata: {e}"))?; + let input_pb = metadata + .input_dtype + .as_ref() + .ok_or_else(|| vortex_err!("L2NormMetadata missing input_dtype"))?; + let input_dtype = DType::from_proto(input_pb, session)?; + let child = children.get(0, &input_dtype, len)?; + Ok(ScalarFnArrayParts { + options: EmptyOptions, + children: vec![child], + }) + } +} + /// Computes the L2 norm (Euclidean norm) of a float slice. /// /// Returns `sqrt(sum(v_i^2))`. A zero-length or all-zero input produces `0.0`. @@ -185,28 +238,25 @@ fn l2_norm_row(v: &[T]) -> T { #[cfg(test)] mod tests { - use std::sync::LazyLock; use rstest::rstest; + use vortex_array::ArrayPlugin; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; - use vortex_array::session::ArraySession; + use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::validity::Validity; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::scalar_fns::l2_norm::L2Norm; + use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Evaluates L2 norm on a tensor/vector array and returns the result as `Vec`. fn eval_l2_norm(input: ArrayRef, len: usize) -> VortexResult> { let scalar_fn = L2Norm::new().erased(); @@ -275,4 +325,31 @@ mod tests { assert_close(&[prim.as_slice::()[0]], &[5.0]); Ok(()) } + + #[rstest] + #[case::fixed_shape_tensor(tensor_array(&[3], &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] + #[case::vector(vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] + fn serde_round_trip(#[case] child: ArrayRef, #[case] len: usize) -> VortexResult<()> { + let original = L2Norm::try_new_array(child.clone(), len)?.into_array(); + + let plugin = ScalarFnArrayPlugin::new(L2Norm); + let metadata = plugin + .serialize(&original, &SESSION)? + .expect("L2Norm serialize must produce metadata"); + + let children = vec![child]; + let recovered = plugin.deserialize( + original.dtype(), + original.len(), + &metadata, + &[], + &children, + &SESSION, + )?; + + assert_eq!(recovered.dtype(), original.dtype()); + assert_eq!(recovered.len(), original.len()); + assert_eq!(recovered.encoding_id(), original.encoding_id()); + Ok(()) + } } diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs index 99d97fbe87f..64308c39562 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs @@ -6,8 +6,9 @@ #![allow(clippy::cast_possible_truncation)] use std::sync::Arc; -use std::sync::LazyLock; +use vortex_array::ArrayPlugin; +use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; @@ -16,16 +17,17 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArray; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::extension::ExtDType; use vortex_array::extension::EmptyMetadata; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; +use vortex_buffer::Buffer; use vortex_buffer::BufferMut; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_session::VortexSession; use super::SorfOptions; use super::SorfTransform; @@ -33,11 +35,9 @@ use super::rotation::SorfMatrix; use crate::encodings::turboquant::centroids::compute_centroid_boundaries; use crate::encodings::turboquant::centroids::find_nearest_centroid; use crate::encodings::turboquant::centroids::get_centroids; +use crate::tests::SESSION; use crate::vector::Vector; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - /// Build a unit-normalized input vector array and forward-transform + quantize it, returning /// `(input_f32, Vector(FSL(Dict(codes, centroids))), padded_dim)`. /// @@ -436,3 +436,57 @@ fn f64_output_type() -> VortexResult<()> { Ok(()) } + +/// Build a trivial `Vector>` child populated with zeroes. The values +/// are irrelevant for the serde round-trip test; only the dtype shape matters. +fn trivial_padded_vector(padded_dim: u32, num_rows: usize, validity: Validity) -> ArrayRef { + let elements = PrimitiveArray::new( + Buffer::::zeroed(num_rows * padded_dim as usize), + Validity::NonNullable, + ); + let fsl = FixedSizeListArray::try_new(elements.into_array(), padded_dim, validity, num_rows) + .vortex_expect("fsl must build"); + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) + .vortex_expect("ext dtype must build") + .erased(); + ExtensionArray::new(ext_dtype, fsl.into_array()).into_array() +} + +#[rstest::rstest] +// Non-power-of-two dimension to exercise `padded_dim = dim.next_power_of_two()`. +#[case::power_of_two_dim(128, Validity::NonNullable)] +#[case::non_power_of_two_dim(100, Validity::NonNullable)] +// Nullable top-level Vector to verify child nullability is reconstructed from the parent output. +#[case::nullable_child(100, Validity::AllValid)] +fn serde_round_trip(#[case] dimension: u32, #[case] validity: Validity) -> VortexResult<()> { + let padded_dim = dimension.next_power_of_two(); + let num_rows = 4; + let options = SorfOptions { + seed: 42, + num_rounds: 3, + dimension, + element_ptype: PType::F32, + }; + let child = trivial_padded_vector(padded_dim, num_rows, validity); + let original = SorfTransform::try_new_array(&options, child.clone(), num_rows)?.into_array(); + + let plugin = ScalarFnArrayPlugin::new(SorfTransform); + let metadata = plugin + .serialize(&original, &SESSION)? + .expect("SorfTransform serialize must produce metadata"); + + let children = vec![child]; + let recovered = plugin.deserialize( + original.dtype(), + original.len(), + &metadata, + &[], + &children, + &SESSION, + )?; + + assert_eq!(recovered.dtype(), original.dtype()); + assert_eq!(recovered.len(), original.len()); + assert_eq!(recovered.encoding_id(), original.encoding_id()); + Ok(()) +} diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs index 5c77b48eb87..64f92da384f 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use num_traits::Float; use num_traits::FromPrimitive; +use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -17,6 +18,9 @@ use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; +use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; @@ -30,12 +34,14 @@ use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::ExecutionArgs; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure_eq; use vortex_error::vortex_err; +use vortex_session::VortexSession; use super::SorfOptions; use super::SorfTransform; @@ -188,6 +194,93 @@ impl ScalarFnVTable for SorfTransform { } } +/// Metadata for a serialized [`SorfTransform`] array. +/// +/// Stores the full [`SorfOptions`] inline. The child [`DType`] is not serialized because it is +/// fully determined by the options: the child is always a [`Vector`] extension wrapping +/// `FSL`. The child's nullability is recovered from the +/// parent output dtype at deserialize time, since `SorfTransform::return_dtype` propagates child +/// nullability into the output FSL (see `return_dtype` above). +#[derive(Clone, prost::Message)] +pub(super) struct SorfTransformMetadata { + #[prost(uint64, tag = "1")] + seed: u64, + /// Rust `u8` widened to `u32` for protobuf (no `u8` on the wire). + #[prost(uint32, tag = "2")] + num_rounds: u32, + #[prost(uint32, tag = "3")] + dimension: u32, + #[prost(enumeration = "PType", tag = "4")] + element_ptype: i32, +} + +impl ScalarFnArrayVTable for SorfTransform { + fn serialize( + &self, + view: &ScalarFnArrayView, + _session: &VortexSession, + ) -> VortexResult>> { + let options = view.options; + Ok(Some( + SorfTransformMetadata { + seed: options.seed, + num_rounds: u32::from(options.num_rounds), + dimension: options.dimension, + element_ptype: options.element_ptype as i32, + } + .encode_to_vec(), + )) + } + + fn deserialize( + &self, + dtype: &DType, + len: usize, + metadata: &[u8], + children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + let metadata = SorfTransformMetadata::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode SorfTransformMetadata: {e}"))?; + let options = SorfOptions { + seed: metadata.seed, + num_rounds: u8::try_from(metadata.num_rounds).map_err(|_| { + vortex_err!( + "SorfTransform num_rounds {} does not fit in u8", + metadata.num_rounds + ) + })?, + dimension: metadata.dimension, + element_ptype: metadata.element_ptype(), + }; + validate_sorf_options(&options)?; + + // `return_dtype` sets the output FSL's nullability to the child's nullability (see + // `return_dtype` above), so we read the child nullability back from the parent dtype. + let child_nullability = dtype + .as_extension_opt() + .ok_or_else(|| { + vortex_err!("SorfTransform parent dtype must be a Vector extension, got {dtype}") + })? + .storage_dtype() + .nullability(); + let padded_dim = options.dimension.next_power_of_two(); + let child_storage = DType::FixedSizeList( + Arc::new(DType::Primitive(PType::F32, Nullability::NonNullable)), + padded_dim, + child_nullability, + ); + let child_ext = ExtDType::::try_new(EmptyMetadata, child_storage)?.erased(); + let child_dtype = DType::Extension(child_ext); + let child = children.get(0, &child_dtype, len)?; + + Ok(ScalarFnArrayParts { + options, + children: vec![child], + }) + } +} + /// Convert an f32 value to a float type `T`. /// /// `FromPrimitive::from_f32` is infallible for all Vortex float types: f16 saturates via the From 2f07feb5ea46fdcb23121175c3a263ff7a639303 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 15 Apr 2026 15:21:26 +0100 Subject: [PATCH 047/250] chore[gpu]: don't inline bitunpack lane impls (#7441) Fully inlining the bitunpack kernels adds memory pressure such that the dynamic dispatch kernel previously operated at the 32 registers per thread max. Preventing to inline the bit unpack line impls drops the register count per thread to 24. This is relevant for future changes which will requires more registers such as patches support on the GPU. The performance hit we take for bitunpack kernels here is 10%. It's probably worthwhile to investigate whether there might a tradeoff here to get similar perf with less aggressive inlining in the future. One thing we could also look at is trading in register spills via launch bounds for more occupancy. Splitting out the lane implementations into headers is needed such that the dynamic dispatch kernel ptx can be compiled without the standalone bitunpack kernels which are not relevant for the dynamic dispatch kernel. This reduces the amount of assembly for the dynamic dispatch to 48k lines from 128k lines. Besides nvcc compile times, this is relevant for the dynamic dispatch kernel in terms of ptx to device compilation which should be as fast as possible. --------- Signed-off-by: Alexander Droste --- .github/workflows/ci.yml | 1 + vortex-cuda/build.rs | 26 +- vortex-cuda/kernels/src/bit_unpack.cuh | 28 +- vortex-cuda/kernels/src/bit_unpack_16.cu | 864 +- .../kernels/src/bit_unpack_16_lanes.cuh | 867 ++ vortex-cuda/kernels/src/bit_unpack_32.cu | 3232 +--- .../kernels/src/bit_unpack_32_lanes.cuh | 3235 ++++ vortex-cuda/kernels/src/bit_unpack_64.cu | 12576 +-------------- .../kernels/src/bit_unpack_64_lanes.cuh | 12579 ++++++++++++++++ vortex-cuda/kernels/src/bit_unpack_8.cu | 256 +- .../kernels/src/bit_unpack_8_lanes.cuh | 259 + vortex-cuda/src/bit_unpack_gen.rs | 41 +- 12 files changed, 17012 insertions(+), 16952 deletions(-) create mode 100644 vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh create mode 100644 vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh create mode 100644 vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh create mode 100644 vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e454301989f..290612d099b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -277,6 +277,7 @@ jobs: git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \ | grep -E '\.(cpp|hpp|cu|cuh|h)$' \ | grep -v 'kernels/src/bit_unpack_.*\.cu$' \ + | grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \ | xargs clang-format --dry-run --Werror --style=file rust-lint-no-default: diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index eb82c6a9563..b8673cdda83 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -14,7 +14,8 @@ use std::process::Command; use fastlanes::FastLanes; -use crate::bit_unpack_gen::generate_cuda_unpack; +use crate::bit_unpack_gen::generate_cuda_unpack_kernels; +use crate::bit_unpack_gen::generate_cuda_unpack_lanes; #[path = "src/bit_unpack_gen.rs"] pub mod bit_unpack_gen; @@ -72,7 +73,11 @@ fn main() { match path.extension().and_then(|e| e.to_str()) { Some("cuh") | Some("h") => { - println!("cargo:rerun-if-changed={}", path.display()) + // Only watch hand-written .cuh/.h files, not generated ones + // (generated files are rebuilt when cuda_kernel_generator changes) + if !is_generated { + println!("cargo:rerun-if-changed={}", path.display()); + } } Some("cu") => { // Only watch hand-written .cu files, not generated ones @@ -94,10 +99,19 @@ fn main() { } fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result { - let path = output_dir.join(format!("bit_unpack_{}.cu", T::T)); - let mut cu_file = File::create(&path)?; - generate_cuda_unpack::(&mut cu_file, thread_count)?; - Ok(path) + // Generate the lanes header (.cuh) — device functions only, no __global__ kernels. + // This is what dynamic_dispatch.cu includes (via bit_unpack.cuh). + let cuh_path = output_dir.join(format!("bit_unpack_{}_lanes.cuh", T::T)); + let mut cuh_file = File::create(&cuh_path)?; + generate_cuda_unpack_lanes::(&mut cuh_file)?; + + // Generate the standalone kernels (.cu) — includes the lanes header, + // adds _device template + __global__ wrappers. Compiled to its own PTX. + let cu_path = output_dir.join(format!("bit_unpack_{}.cu", T::T)); + let mut cu_file = File::create(&cu_path)?; + generate_cuda_unpack_kernels::(&mut cu_file, thread_count)?; + + Ok(cu_path) } fn nvcc_compile_ptx( diff --git a/vortex-cuda/kernels/src/bit_unpack.cuh b/vortex-cuda/kernels/src/bit_unpack.cuh index fb814e3f4fa..8e6bc9fec93 100644 --- a/vortex-cuda/kernels/src/bit_unpack.cuh +++ b/vortex-cuda/kernels/src/bit_unpack.cuh @@ -7,10 +7,10 @@ #include #include -#include "bit_unpack_8.cu" -#include "bit_unpack_16.cu" -#include "bit_unpack_32.cu" -#include "bit_unpack_64.cu" +#include "bit_unpack_8_lanes.cuh" +#include "bit_unpack_16_lanes.cuh" +#include "bit_unpack_32_lanes.cuh" +#include "bit_unpack_64_lanes.cuh" #include "patches.h" /// Decodes a single lane of packed data. @@ -26,22 +26,22 @@ /// * `lane` - Lane index within the block (used to determine which packed words to process) /// * `bit_width` - Number of bits with which each value is encoded template -__device__ inline void bit_unpack_lane(const T *__restrict packed_chunk, - T *__restrict output_buffer, - T reference, - unsigned int lane, - uint32_t bit_width); +__device__ __noinline__ void bit_unpack_lane(const T *__restrict packed_chunk, + T *__restrict output_buffer, + T reference, + unsigned int lane, + uint32_t bit_width); /// Template specializations for `bitunpack_lane_to_smem` for different integer types. /// /// Generates template specializations for each supported integer size (8, 16, 32, 64 bits). #define BIT_UNPACK_LANE(bits) \ template <> \ - __device__ inline void bit_unpack_lane(const uint##bits##_t *in, \ - uint##bits##_t *out, \ - uint##bits##_t reference, \ - unsigned int lane, \ - uint32_t bw) { \ + __device__ __noinline__ void bit_unpack_lane(const uint##bits##_t *in, \ + uint##bits##_t *out, \ + uint##bits##_t reference, \ + unsigned int lane, \ + uint32_t bw) { \ bit_unpack_##bits##_lane(in, out, reference, lane, bw); \ } diff --git a/vortex-cuda/kernels/src/bit_unpack_16.cu b/vortex-cuda/kernels/src/bit_unpack_16.cu index c329e85157a..63eb5f19bf9 100644 --- a/vortex-cuda/kernels/src/bit_unpack_16.cu +++ b/vortex-cuda/kernels/src/bit_unpack_16.cu @@ -1,869 +1,7 @@ // AUTO-GENERATED. Do not edit by hand! -#include -#include -#include -#include "fastlanes_common.cuh" +#include "bit_unpack_16_lanes.cuh" #include "patches.cuh" -template -__device__ void _bit_unpack_16_lane(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_16_lane<0>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 16; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_16_lane<1>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<2>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<3>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 1)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<4>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<5>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 4)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 2)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<6>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<7>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 5)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 1)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 2)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<8>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 0)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<9>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 2)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 6)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 1)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 3)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 5)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<10>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 4)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 8)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 2)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 0)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 4)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 6)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<11>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 6)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 1)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 7)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 8)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 3)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 9)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 10)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 5)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<12>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 0)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 8)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 4)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<13>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 10)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 7)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 4)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 11)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 8)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 5)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 12)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 9)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 6)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 13); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<14>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 12)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 10)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 8)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 4)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 2)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 0)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint16_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 10)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 8)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 6)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint16_t, 2)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<15>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - uint16_t src; - uint16_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint16_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint16_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint16_t, 14)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint16_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint16_t, 13)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint16_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint16_t, 12)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint16_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint16_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint16_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint16_t, 10)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint16_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint16_t, 9)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint16_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint16_t, 8)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint16_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint16_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint16_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint16_t, 6)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint16_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint16_t, 5)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint16_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint16_t, 4)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint16_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint16_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint16_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint16_t, 2)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint16_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint16_t, 1)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint16_t, 15); - out[INDEX(15, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_16_lane<16>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; - #pragma unroll - for (int row = 0; row < 16; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ inline void bit_unpack_16_lane( - const uint16_t *__restrict in, - uint16_t *__restrict out, - uint16_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_16_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_16_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_16_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_16_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_16_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_16_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_16_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_16_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_16_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_16_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_16_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_16_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_16_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_16_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_16_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_16_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_16_lane<16>(in, out, reference, lane); break; - } -} - template __device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint16_t shared_out[1024]; diff --git a/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh new file mode 100644 index 00000000000..48c62fde2d4 --- /dev/null +++ b/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh @@ -0,0 +1,867 @@ +// AUTO-GENERATED. Do not edit by hand! +#pragma once + +#include +#include +#include +#include "fastlanes_common.cuh" + +template +__device__ void _bit_unpack_16_lane(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane); + +template <> +__device__ void _bit_unpack_16_lane<0>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + #pragma unroll + for (int row = 0; row < 16; row++) { + out[INDEX(row, lane)] = reference; + } +} + +template <> +__device__ void _bit_unpack_16_lane<1>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 1); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 1); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 1); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 1); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 1); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 1); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 1); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 1); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 1); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 1); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 1); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 1); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 1); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 1); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 1); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<2>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 2); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 2); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 2); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 2); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 2); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 2); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 2); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 0)) << 2; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 2); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 2); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 2); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 2); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 2); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 2); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 2); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<3>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 3); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 3); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 3); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 3); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 3); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 2)) << 1; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 3); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 3); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 3); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 3); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 1)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 3); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 3); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 3); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 3); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<4>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 4); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 4); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 4); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 0)) << 4; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 4); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 4); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 4); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 0)) << 4; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 4); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 4); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 4); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 0)) << 4; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 4); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 4); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 4); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<5>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 5); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 5); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 5); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 4)) << 1; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 5); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 5); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 3)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 5); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 5); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 2)) << 3; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 5); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 5); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 1)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 5); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 5); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<6>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 6); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 6); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 2)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 6); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 6); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 4)) << 2; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 6); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 0)) << 6; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 6); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 6); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 2)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 6); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 6); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 4)) << 2; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 6); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<7>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 7); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 7); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 5)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 7); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 3)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 7); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 1)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 7); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 7); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 6)) << 1; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 7); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 4)) << 3; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 7); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 2)) << 5; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 7); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 7); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<8>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 0)) << 8; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 8); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<9>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 9); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 7); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 2)) << 7; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 9); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 4)) << 5; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 9); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 6)) << 3; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 9); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 8)) << 1; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 1)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 9); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 3)) << 6; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 9); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 5)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 9); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 7)) << 2; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 9); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<10>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 10); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 4)) << 6; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 10); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 8)) << 2; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 2)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 10); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 6)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 0)) << 10; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 10); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 4)) << 6; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 10); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 8)) << 2; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 2)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 10); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 6)) << 4; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<11>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 11); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 6)) << 5; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 1)) << 10; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 11); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 7)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 9); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 2)) << 9; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 11); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 8)) << 3; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 3)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 11); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 9)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 7); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 4)) << 7; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 11); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 10)) << 1; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint16_t, 5)) << 6; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 11); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<12>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 12); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 8)) << 4; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 4)) << 8; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 0)) << 12; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 12); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 8)) << 4; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 4)) << 8; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 0)) << 12; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 12); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 8)) << 4; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 4)) << 8; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 0)) << 12; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 12); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint16_t, 8)) << 4; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint16_t, 4)) << 8; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<13>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 13); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 10)) << 3; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 7)) << 6; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 9); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 4)) << 9; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 1)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 13); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 11)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 8)) << 5; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 5)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 11); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 2)) << 11; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 13); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 12)) << 1; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint16_t, 9)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 7); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint16_t, 6)) << 7; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint16_t, 3)) << 10; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 13); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<14>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 14); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 12)) << 2; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 10)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 8)) << 6; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 6)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 4)) << 10; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 2)) << 12; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 0)) << 14; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint16_t, 14); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 12)) << 2; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 10)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint16_t, 8)) << 6; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint16_t, 6)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint16_t, 4)) << 10; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint16_t, 2)) << 12; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 14); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<15>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + uint16_t src; + uint16_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint16_t, 15); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint16_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint16_t, 14)) << 1; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint16_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint16_t, 13)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint16_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint16_t, 12)) << 3; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint16_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint16_t, 11)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint16_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint16_t, 10)) << 5; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint16_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint16_t, 9)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint16_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint16_t, 8)) << 7; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint16_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint16_t, 7)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint16_t, 9); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint16_t, 6)) << 9; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint16_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint16_t, 5)) << 10; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint16_t, 11); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint16_t, 4)) << 11; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint16_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint16_t, 3)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint16_t, 13); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint16_t, 2)) << 13; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint16_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint16_t, 1)) << 14; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint16_t, 15); + out[INDEX(15, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_16_lane<16>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 64; + #pragma unroll + for (int row = 0; row < 16; row++) { + out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; + } +} + +/// Runtime dispatch to the optimized lane decoder for the given bit width. +__device__ __noinline__ void bit_unpack_16_lane( + const uint16_t *__restrict in, + uint16_t *__restrict out, + uint16_t reference, + unsigned int lane, + uint32_t bit_width +) { + switch (bit_width) { + case 0: _bit_unpack_16_lane<0>(in, out, reference, lane); break; + case 1: _bit_unpack_16_lane<1>(in, out, reference, lane); break; + case 2: _bit_unpack_16_lane<2>(in, out, reference, lane); break; + case 3: _bit_unpack_16_lane<3>(in, out, reference, lane); break; + case 4: _bit_unpack_16_lane<4>(in, out, reference, lane); break; + case 5: _bit_unpack_16_lane<5>(in, out, reference, lane); break; + case 6: _bit_unpack_16_lane<6>(in, out, reference, lane); break; + case 7: _bit_unpack_16_lane<7>(in, out, reference, lane); break; + case 8: _bit_unpack_16_lane<8>(in, out, reference, lane); break; + case 9: _bit_unpack_16_lane<9>(in, out, reference, lane); break; + case 10: _bit_unpack_16_lane<10>(in, out, reference, lane); break; + case 11: _bit_unpack_16_lane<11>(in, out, reference, lane); break; + case 12: _bit_unpack_16_lane<12>(in, out, reference, lane); break; + case 13: _bit_unpack_16_lane<13>(in, out, reference, lane); break; + case 14: _bit_unpack_16_lane<14>(in, out, reference, lane); break; + case 15: _bit_unpack_16_lane<15>(in, out, reference, lane); break; + case 16: _bit_unpack_16_lane<16>(in, out, reference, lane); break; + } +} + diff --git a/vortex-cuda/kernels/src/bit_unpack_32.cu b/vortex-cuda/kernels/src/bit_unpack_32.cu index 7eb2d49cc49..521183eee7c 100644 --- a/vortex-cuda/kernels/src/bit_unpack_32.cu +++ b/vortex-cuda/kernels/src/bit_unpack_32.cu @@ -1,3237 +1,7 @@ // AUTO-GENERATED. Do not edit by hand! -#include -#include -#include -#include "fastlanes_common.cuh" +#include "bit_unpack_32_lanes.cuh" #include "patches.cuh" -template -__device__ void _bit_unpack_32_lane(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_32_lane<0>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 32; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_32_lane<1>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 1); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 1); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 1); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 1); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 1); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 1); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 1); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 1); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 1); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 1); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 1); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 1); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 1); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 1); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 1); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 1); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<2>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 2); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 2; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 2); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 2); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 2); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 2); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 2); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 2); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 2); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 2); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 2); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 2); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 2); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 2); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 2); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 2); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 2); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<3>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 3); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 1)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 3); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 3); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 3); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 3); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 3); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 3); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 1; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 3); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 3); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 3); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 3); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 3); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 3); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 3); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 3); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 3); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<4>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 4; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 4); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 4); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 4); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 4); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 4); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 4); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 4); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<5>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 5); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 5); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 5); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 5); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 5); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 5); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 1; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 5); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 5); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 5); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 5); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 5); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 3; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 5); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 5); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 5); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 5); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 5); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<6>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 6); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 6); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 6); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 6; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 6); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 6); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 6); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 6); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 6); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 2; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 6); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 6); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 6); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 6); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 6); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 6); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 6); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 6); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<7>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 7); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 7); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 7); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 2)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 7); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 7); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 7); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 5)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 7); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 7); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 7); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 1)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 7); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 7); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 7); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 7); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 3; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 7); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 7); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 7); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<8>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 8; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 8); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 8); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 8); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<9>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 9); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 9); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 9); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 3)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 9); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 9); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 9); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 7; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 9); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 9); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 9); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 3; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 9); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 9); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 9); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 9); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 9); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 5)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 9); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 9); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<10>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 10); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 8)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 10); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 10); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 10); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 10); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 10; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 10); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 10); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 10); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 2; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 10); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 10); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 6)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 10); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 10); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 6; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 10); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 10); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 2)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 10); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 10); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<11>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 11); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 1)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 11); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 11); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 3)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 11); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 11); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 5)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 11); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 11); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 5; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 11); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 11); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 7)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 11); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 11); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 3; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 11); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 11); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 9)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 11); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 11); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 1; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 11); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<12>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 12; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 12); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 12); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 12); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 12); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 4; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 12); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<13>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 13); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 7)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 13); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 13); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 8)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 13); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 13); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 9)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 13); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 13); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 13); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 3; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 13); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 9; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 13); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 13); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 11)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 13); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 5)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 13); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 13); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 1; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 13); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 7; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 13); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<14>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 14); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 10)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 14); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 14); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 2)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 14); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 14); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 14); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 14); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 14; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 14); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 14); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 10)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 14); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 6)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 14); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 2)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 14); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 14); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 2; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 14); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 8)) << 6; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 14); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 10; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 14); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<15>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 15); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 13)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 15); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 15); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 9)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 15); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 15); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 5)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 15); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 15); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 15); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 15); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 1; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 15); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 3; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 15); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 5; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 15); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 7; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 15); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 9; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 15); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 11; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 15); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 13; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 15); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<16>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 16; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 16); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<17>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 17); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 2)) << 15; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 17); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 4)) << 13; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 17); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 6)) << 11; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 17); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 8)) << 9; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 17); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 10)) << 7; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 17); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 12)) << 5; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 17); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 14)) << 3; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 17); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 16)) << 1; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 1)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 17); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 3)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 17); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 5)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 17); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 7)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 17); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 9)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 17); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 11)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 17); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 13)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 17); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 15)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<18>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 18); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 4)) << 14; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 18); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 10; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 18); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 12)) << 6; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 18); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 16)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 18); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 18); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 18); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 18; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 18); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 14; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 18); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 10; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 18); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 12)) << 6; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 18); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 2; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 18); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 6)) << 12; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 18); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 10)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 18); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 14)) << 4; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<19>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 19); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 6)) << 13; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 19); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 12)) << 7; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 19); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 18)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 5)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 19); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 11)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 19); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 17)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 4)) << 15; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 19); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 10)) << 9; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 19); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 16)) << 3; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 3)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 19); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 9)) << 10; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 19); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 15)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 2)) << 17; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 19); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 8)) << 11; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 19); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 14)) << 5; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 1)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 19); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 7)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 19); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 13)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<20>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 20; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 20); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 8)) << 12; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 20); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 16)) << 4; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 4)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 20); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 12)) << 8; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<21>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 21); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 10)) << 11; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 21); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 20)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 9)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 21); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 19)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 13; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 21); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 18)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 7)) << 14; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 21); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 17)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 6)) << 15; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 21); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 5; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 5)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 21); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 15)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 17; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 21); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 14)) << 7; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 3)) << 18; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 21); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 13)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 2)) << 19; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 21); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 12)) << 9; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 1)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 21); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 11)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<22>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 22); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 12)) << 10; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 2)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 22); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 14)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 4)) << 18; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 22); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 16)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 6)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 22); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 18)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 14; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 22); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 20)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 0)) << 22; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 22); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 12)) << 10; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 2)) << 20; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 22); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 14)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 4)) << 18; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 22); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 16)) << 6; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 6)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 22); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 18)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 8)) << 14; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 22); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 20)) << 2; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 10)) << 12; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<23>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 23); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 14)) << 9; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 5)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 23); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 19)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 10)) << 13; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 1)) << 22; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 23); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 15)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 6)) << 17; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 23); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 20)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 11)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 2)) << 21; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 23); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 16)) << 7; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 7)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 23); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 21)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 12)) << 11; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 3)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 23); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 17)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 8)) << 15; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 23); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 22)) << 1; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 13)) << 10; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 4)) << 19; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 23); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 18)) << 5; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 9)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<24>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 0)) << 24; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 24); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 16)) << 8; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 8)) << 16; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<25>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 25); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 18)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 11)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 4)) << 21; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 25); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 22)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 15)) << 10; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 8)) << 17; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 1)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 25); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 19)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 13; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 5)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 25); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 23)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 16)) << 9; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 9)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 23; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 25); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 20)) << 5; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 13)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 6)) << 19; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 25); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 24)) << 1; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 17)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 10)) << 15; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 3)) << 22; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 25); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 21)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 14)) << 11; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 7)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<26>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 26); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 20)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 14)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 8)) << 18; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 2)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 26); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 22)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 16)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 10)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 4)) << 22; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 26); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 24)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 18)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 14; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 0)) << 26; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 26); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 20)) << 6; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 14)) << 12; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 8)) << 18; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 2)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 26); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 22)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 16)) << 10; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 10)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 4)) << 22; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 26); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 24)) << 2; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 18)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 12)) << 14; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 6)) << 20; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<27>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 27); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 22)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 17)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 12)) << 15; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 7)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 2)) << 25; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 27); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 24)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 19)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 13; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 9)) << 18; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 4)) << 23; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 27); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 26)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 21)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 16)) << 11; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 11)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 6)) << 21; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 1)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 27); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 23)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 18)) << 9; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 13)) << 14; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 8)) << 19; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 3)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 27); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 25)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 20)) << 7; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 15)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 10)) << 17; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 5)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<28>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 0)) << 28; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 28); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 24)) << 4; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 20)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 16)) << 12; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 12)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 8)) << 20; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 4)) << 24; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<29>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 29); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 26)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 23)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 20)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 17)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 14)) << 15; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 11)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 8)) << 21; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 5)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 2)) << 27; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 29); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 28)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 25)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 22)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 19)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 16)) << 13; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 13)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 10)) << 19; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 7)) << 22; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 4)) << 25; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 1)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 29); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 27)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 24)) << 5; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 21)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 18)) << 11; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 15)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 12)) << 17; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 9)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 6)) << 23; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 3)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 29); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<30>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 30); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 28)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 26)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 24)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 22)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 20)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 18)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 16)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 14)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 12)) << 18; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 10)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 8)) << 22; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 6)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 4)) << 26; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 2)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 0)) << 30; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint32_t, 30); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 28)) << 2; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 26)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 24)) << 6; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 22)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 20)) << 10; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 18)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 16)) << 14; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 14)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 12)) << 18; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 10)) << 20; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 8)) << 22; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 6)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 4)) << 26; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint32_t, 2)) << 28; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<31>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - uint32_t src; - uint32_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint32_t, 31); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint32_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint32_t, 30)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint32_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint32_t, 29)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint32_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint32_t, 28)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint32_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint32_t, 27)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint32_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint32_t, 26)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint32_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint32_t, 25)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint32_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint32_t, 24)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint32_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint32_t, 23)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint32_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint32_t, 22)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint32_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint32_t, 21)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint32_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint32_t, 20)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint32_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint32_t, 19)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint32_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint32_t, 18)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint32_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint32_t, 17)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint32_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint32_t, 16)) << 15; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint32_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint32_t, 15)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint32_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint32_t, 14)) << 17; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint32_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint32_t, 13)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint32_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint32_t, 12)) << 19; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint32_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint32_t, 11)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint32_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint32_t, 10)) << 21; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint32_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint32_t, 9)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint32_t, 23); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint32_t, 8)) << 23; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint32_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint32_t, 7)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint32_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint32_t, 6)) << 25; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint32_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint32_t, 5)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint32_t, 27); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint32_t, 4)) << 27; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint32_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint32_t, 3)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint32_t, 29); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint32_t, 2)) << 29; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint32_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint32_t, 1)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint32_t, 31); - out[INDEX(31, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_32_lane<32>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; - #pragma unroll - for (int row = 0; row < 32; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ inline void bit_unpack_32_lane( - const uint32_t *__restrict in, - uint32_t *__restrict out, - uint32_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_32_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_32_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_32_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_32_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_32_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_32_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_32_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_32_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_32_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_32_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_32_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_32_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_32_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_32_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_32_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_32_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_32_lane<16>(in, out, reference, lane); break; - case 17: _bit_unpack_32_lane<17>(in, out, reference, lane); break; - case 18: _bit_unpack_32_lane<18>(in, out, reference, lane); break; - case 19: _bit_unpack_32_lane<19>(in, out, reference, lane); break; - case 20: _bit_unpack_32_lane<20>(in, out, reference, lane); break; - case 21: _bit_unpack_32_lane<21>(in, out, reference, lane); break; - case 22: _bit_unpack_32_lane<22>(in, out, reference, lane); break; - case 23: _bit_unpack_32_lane<23>(in, out, reference, lane); break; - case 24: _bit_unpack_32_lane<24>(in, out, reference, lane); break; - case 25: _bit_unpack_32_lane<25>(in, out, reference, lane); break; - case 26: _bit_unpack_32_lane<26>(in, out, reference, lane); break; - case 27: _bit_unpack_32_lane<27>(in, out, reference, lane); break; - case 28: _bit_unpack_32_lane<28>(in, out, reference, lane); break; - case 29: _bit_unpack_32_lane<29>(in, out, reference, lane); break; - case 30: _bit_unpack_32_lane<30>(in, out, reference, lane); break; - case 31: _bit_unpack_32_lane<31>(in, out, reference, lane); break; - case 32: _bit_unpack_32_lane<32>(in, out, reference, lane); break; - } -} - template __device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint32_t shared_out[1024]; diff --git a/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh new file mode 100644 index 00000000000..4d1206e9f88 --- /dev/null +++ b/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh @@ -0,0 +1,3235 @@ +// AUTO-GENERATED. Do not edit by hand! +#pragma once + +#include +#include +#include +#include "fastlanes_common.cuh" + +template +__device__ void _bit_unpack_32_lane(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane); + +template <> +__device__ void _bit_unpack_32_lane<0>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + #pragma unroll + for (int row = 0; row < 32; row++) { + out[INDEX(row, lane)] = reference; + } +} + +template <> +__device__ void _bit_unpack_32_lane<1>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 1); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 1); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 1); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 1); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 1); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 1); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 1); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 1); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 1); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 1); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 1); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 1); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 1); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 1); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 1); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 1); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 1); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 1); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 1); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 1); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 1); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 1); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 1); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 1); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 1); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 1); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 1); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 1); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 1); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 1); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 1); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<2>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 2); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 2); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 2); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 2); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 2); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 2); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 2); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 2); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 2); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 2); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 2); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 2); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 2); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 2); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 2); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 0)) << 2; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 2); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 2); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 2); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 2); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 2); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 2); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 2); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 2); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 2); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 2); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 2); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 2); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 2); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 2); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 2); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<3>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 3); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 3); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 3); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 3); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 3); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 3); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 3); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 3); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 3); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 3); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 1)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 3); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 3); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 3); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 3); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 3); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 3); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 3); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 3); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 3); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 3); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 2)) << 1; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 3); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 3); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 3); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 3); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 3); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 3); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 3); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 3); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 3); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<4>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 4); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 4); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 4); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 4); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 4); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 4); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 4); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 0)) << 4; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 4); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 4); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 4); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 4); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 4); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 4); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 4); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 0)) << 4; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 4); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 4); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 4); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 4); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 4); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 4); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 4); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 4; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 4); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 4); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 4); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 4); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 4); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 4); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 4); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<5>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 5); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 5); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 5); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 5); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 5); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 5); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 3)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 5); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 5); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 5); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 5); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 5); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 1)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 5); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 5); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 5); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 5); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 5); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 5); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 4)) << 1; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 5); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 5); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 5); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 5); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 5); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 2)) << 3; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 5); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 5); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 5); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 5); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 5); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<6>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 6); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 6); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 6); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 6); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 6); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 4)) << 2; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 6); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 6); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 6); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 6); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 2)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 6); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 6); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 6); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 6); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 6; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 6); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 6); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 6); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 6); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 6); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 4)) << 2; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 6); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 6); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 6); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 6); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 2)) << 4; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 6); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 6); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 6); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 6); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<7>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 7); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 7); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 7); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 7); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 3)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 7); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 7); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 7); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 7); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 6)) << 1; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 7); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 7); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 7); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 2)) << 5; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 7); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 7); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 7); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 7); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 5)) << 2; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 7); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 7); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 7); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 1)) << 6; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 7); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 7); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 7); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 7); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 4)) << 3; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 7); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 7); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 7); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<8>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 0)) << 8; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 8); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 8); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 8); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<9>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 9); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 9); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 9); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 4)) << 5; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 9); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 9); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 9); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 8)) << 1; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 9); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 9); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 3)) << 6; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 9); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 9); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 9); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 7)) << 2; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 9); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 9); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 2)) << 7; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 9); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 9); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 9); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 6)) << 3; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 9); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 9); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 1)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 9); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 9); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 9); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 5)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 9); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 9); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<10>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 10); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 10); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 10); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 8)) << 2; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 10); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 10); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 6)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 10); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 10); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 4)) << 6; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 10); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 10); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 2)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 10); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 10); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 0)) << 10; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 10); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 10); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 10); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 8)) << 2; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 10); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 10); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 6)) << 4; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 10); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 10); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 4)) << 6; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 10); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 10); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 2)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 10); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 10); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<11>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 11); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 11); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 1)) << 10; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 11); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 11); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 2)) << 9; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 11); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 11); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 3)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 11); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 11); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 4)) << 7; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 11); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 11); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 5)) << 6; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 11); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 11); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 6)) << 5; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 11); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 11); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 7)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 11); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 11); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 8)) << 3; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 11); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 11); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 9)) << 2; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 11); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 11); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 10)) << 1; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 11); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<12>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 12); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 12); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 4)) << 8; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 12); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 12); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 8)) << 4; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 12); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 12; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 12); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 12); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 4)) << 8; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 12); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 12); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 8)) << 4; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 12); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 0)) << 12; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 12); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 12); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 4)) << 8; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 12); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 12); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 8)) << 4; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 12); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 0)) << 12; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 12); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 12); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 4)) << 8; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 12); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 12); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 4; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 12); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<13>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 13); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 13); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 7)) << 6; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 13); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 1)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 13); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 13); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 8)) << 5; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 13); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 2)) << 11; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 13); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 13); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 9)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 13); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 3)) << 10; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 13); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 13); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 10)) << 3; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 13); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 4)) << 9; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 13); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 13); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 11)) << 2; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 13); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 5)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 13); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 13); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 12)) << 1; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 13); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 6)) << 7; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 13); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<14>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 14); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 14); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 10)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 14); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 6)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 14); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 2)) << 12; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 14); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 14); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 12)) << 2; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 14); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 8)) << 6; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 14); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 4)) << 10; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 14); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 0)) << 14; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 14); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 14); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 10)) << 4; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 14); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 6)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 14); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 2)) << 12; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 14); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 14); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 12)) << 2; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 14); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 8)) << 6; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 14); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 10; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 14); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<15>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 15); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 15); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 13)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 15); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 11)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 15); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 9)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 15); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 7)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 15); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 5)) << 10; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 15); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 3)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 15); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 1)) << 14; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 15); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 15); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 14)) << 1; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 15); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 12)) << 3; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 15); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 10)) << 5; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 15); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 7; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 15); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 6)) << 9; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 15); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 11; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 15); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 2)) << 13; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 15); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<16>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 0)) << 16; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 16); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<17>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 17); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 2)) << 15; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 17); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 4)) << 13; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 17); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 6)) << 11; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 17); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 8)) << 9; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 17); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 10)) << 7; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 17); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 12)) << 5; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 17); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 14)) << 3; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 17); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 16)) << 1; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 1)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 17); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 3)) << 14; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 17); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 5)) << 12; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 17); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 7)) << 10; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 17); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 9)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 17); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 11)) << 6; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 17); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 13)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 17); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 15)) << 2; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<18>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 18); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 4)) << 14; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 18); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 8)) << 10; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 18); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 12)) << 6; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 18); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 16)) << 2; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 2)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 18); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 6)) << 12; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 18); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 10)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 18); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 14)) << 4; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 0)) << 18; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 18); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 4)) << 14; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 18); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 10; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 18); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 12)) << 6; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 18); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 16)) << 2; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 2)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 18); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 6)) << 12; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 18); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 10)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 18); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 14)) << 4; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<19>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 19); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 6)) << 13; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 19); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 12)) << 7; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 19); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 18)) << 1; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 5)) << 14; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 19); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 11)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 19); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 17)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 4)) << 15; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 19); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 10)) << 9; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 19); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 16)) << 3; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 3)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 19); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 9)) << 10; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 19); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 15)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 2)) << 17; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 19); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 8)) << 11; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 19); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 14)) << 5; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 1)) << 18; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 19); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 7)) << 12; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 19); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 13)) << 6; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<20>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 20); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 8)) << 12; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 20); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 16)) << 4; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 4)) << 16; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 20); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 12)) << 8; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 0)) << 20; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 20); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 8)) << 12; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 20); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 16)) << 4; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 4)) << 16; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 20); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 12)) << 8; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 0)) << 20; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 20); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 12; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 20); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 16)) << 4; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 16; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 20); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 12)) << 8; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 0)) << 20; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 20); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 8)) << 12; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 20); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 16)) << 4; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 4)) << 16; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 20); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 12)) << 8; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<21>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 21); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 10)) << 11; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 21); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 20)) << 1; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 9)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 21); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 19)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 8)) << 13; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 21); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 18)) << 3; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 7)) << 14; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 21); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 17)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 6)) << 15; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 21); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 16)) << 5; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 5)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 21); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 15)) << 6; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 17; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 21); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 14)) << 7; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 3)) << 18; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 21); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 13)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 2)) << 19; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 21); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 12)) << 9; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 1)) << 20; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 21); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 11)) << 10; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<22>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 22); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 12)) << 10; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 2)) << 20; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 22); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 14)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 4)) << 18; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 22); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 16)) << 6; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 6)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 22); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 18)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 8)) << 14; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 22); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 20)) << 2; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 10)) << 12; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 0)) << 22; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 22); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 12)) << 10; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 2)) << 20; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 22); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 14)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 4)) << 18; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 22); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 16)) << 6; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 6)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 22); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 18)) << 4; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 8)) << 14; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 22); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 20)) << 2; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 10)) << 12; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<23>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 23); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 14)) << 9; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 5)) << 18; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 23); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 19)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 10)) << 13; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 1)) << 22; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 23); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 15)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 6)) << 17; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 23); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 20)) << 3; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 11)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 2)) << 21; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 23); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 16)) << 7; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 7)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 23); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 21)) << 2; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 12)) << 11; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 3)) << 20; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 23); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 17)) << 6; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 8)) << 15; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 23); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 22)) << 1; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 13)) << 10; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 4)) << 19; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 23); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 18)) << 5; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 9)) << 14; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 23); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<24>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 0)) << 24; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 24); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 16)) << 8; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 8)) << 16; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<25>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 25); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 18)) << 7; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 11)) << 14; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 4)) << 21; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 25); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 22)) << 3; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 15)) << 10; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 8)) << 17; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 1)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 25); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 19)) << 6; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 12)) << 13; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 5)) << 20; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 25); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 23)) << 2; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 16)) << 9; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 9)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 23); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 2)) << 23; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 25); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 20)) << 5; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 13)) << 12; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 6)) << 19; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 25); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 24)) << 1; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 17)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 10)) << 15; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 3)) << 22; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 25); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 21)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 14)) << 11; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 7)) << 18; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 25); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<26>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 26); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 20)) << 6; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 14)) << 12; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 8)) << 18; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 2)) << 24; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 26); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 22)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 16)) << 10; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 10)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 4)) << 22; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 26); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 24)) << 2; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 18)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 12)) << 14; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 6)) << 20; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 0)) << 26; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 26); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 20)) << 6; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 14)) << 12; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 8)) << 18; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 2)) << 24; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 26); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 22)) << 4; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 16)) << 10; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 10)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 4)) << 22; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 26); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 24)) << 2; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 18)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 12)) << 14; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 6)) << 20; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<27>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 27); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 22)) << 5; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 17)) << 10; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 12)) << 15; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 7)) << 20; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 25); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 2)) << 25; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 27); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 24)) << 3; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 19)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 14)) << 13; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 9)) << 18; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 23); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 4)) << 23; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 27); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 26)) << 1; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 21)) << 6; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 16)) << 11; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 11)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 6)) << 21; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 1)) << 26; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 27); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 23)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 18)) << 9; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 13)) << 14; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 8)) << 19; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 3)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 27); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 25)) << 2; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 20)) << 7; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 15)) << 12; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 10)) << 17; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint32_t, 5)) << 22; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 27); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<28>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 28); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 24)) << 4; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 20)) << 8; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 16)) << 12; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 12)) << 16; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 8)) << 20; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 4)) << 24; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 0)) << 28; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 28); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 24)) << 4; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 20)) << 8; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 16)) << 12; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 12)) << 16; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 8)) << 20; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 24; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 0)) << 28; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 28); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 24)) << 4; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 20)) << 8; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 16)) << 12; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 12)) << 16; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 8)) << 20; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 4)) << 24; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 0)) << 28; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 28); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 24)) << 4; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 20)) << 8; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 16)) << 12; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 12)) << 16; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint32_t, 8)) << 20; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint32_t, 4)) << 24; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<29>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 29); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 26)) << 3; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 23)) << 6; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 20)) << 9; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 17)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 14)) << 15; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 11)) << 18; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 8)) << 21; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 5)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 27); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 2)) << 27; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 29); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 28)) << 1; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 25)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 22)) << 7; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 19)) << 10; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 16)) << 13; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 13)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 10)) << 19; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 7)) << 22; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 25); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 4)) << 25; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 1)) << 28; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 29); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 27)) << 2; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 24)) << 5; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 21)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 18)) << 11; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 15)) << 14; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 12)) << 17; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint32_t, 9)) << 20; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 23); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint32_t, 6)) << 23; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint32_t, 3)) << 26; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 29); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<30>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 30); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 28)) << 2; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 26)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 24)) << 6; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 22)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 20)) << 10; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 18)) << 12; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 16)) << 14; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 14)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 12)) << 18; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 10)) << 20; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 8)) << 22; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 6)) << 24; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 4)) << 26; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 2)) << 28; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 30); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 0)) << 30; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint32_t, 30); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 28)) << 2; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 26)) << 4; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 24)) << 6; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 22)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 20)) << 10; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 18)) << 12; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 16)) << 14; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 14)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 12)) << 18; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 10)) << 20; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint32_t, 8)) << 22; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint32_t, 6)) << 24; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint32_t, 4)) << 26; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint32_t, 2)) << 28; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 30); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<31>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + uint32_t src; + uint32_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint32_t, 31); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint32_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint32_t, 30)) << 1; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint32_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint32_t, 29)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint32_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint32_t, 28)) << 3; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint32_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint32_t, 27)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint32_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint32_t, 26)) << 5; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint32_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint32_t, 25)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint32_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint32_t, 24)) << 7; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint32_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint32_t, 23)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint32_t, 9); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint32_t, 22)) << 9; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint32_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint32_t, 21)) << 10; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint32_t, 11); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint32_t, 20)) << 11; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint32_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint32_t, 19)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint32_t, 13); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint32_t, 18)) << 13; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint32_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint32_t, 17)) << 14; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint32_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint32_t, 16)) << 15; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint32_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint32_t, 15)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint32_t, 17); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint32_t, 14)) << 17; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint32_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint32_t, 13)) << 18; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint32_t, 19); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint32_t, 12)) << 19; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint32_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint32_t, 11)) << 20; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint32_t, 21); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint32_t, 10)) << 21; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint32_t, 22); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint32_t, 9)) << 22; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint32_t, 23); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint32_t, 8)) << 23; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint32_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint32_t, 7)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint32_t, 25); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint32_t, 6)) << 25; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint32_t, 26); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint32_t, 5)) << 26; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint32_t, 27); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint32_t, 4)) << 27; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint32_t, 28); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint32_t, 3)) << 28; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint32_t, 29); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint32_t, 2)) << 29; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint32_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint32_t, 1)) << 30; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint32_t, 31); + out[INDEX(31, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_32_lane<32>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 32; + #pragma unroll + for (int row = 0; row < 32; row++) { + out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; + } +} + +/// Runtime dispatch to the optimized lane decoder for the given bit width. +__device__ __noinline__ void bit_unpack_32_lane( + const uint32_t *__restrict in, + uint32_t *__restrict out, + uint32_t reference, + unsigned int lane, + uint32_t bit_width +) { + switch (bit_width) { + case 0: _bit_unpack_32_lane<0>(in, out, reference, lane); break; + case 1: _bit_unpack_32_lane<1>(in, out, reference, lane); break; + case 2: _bit_unpack_32_lane<2>(in, out, reference, lane); break; + case 3: _bit_unpack_32_lane<3>(in, out, reference, lane); break; + case 4: _bit_unpack_32_lane<4>(in, out, reference, lane); break; + case 5: _bit_unpack_32_lane<5>(in, out, reference, lane); break; + case 6: _bit_unpack_32_lane<6>(in, out, reference, lane); break; + case 7: _bit_unpack_32_lane<7>(in, out, reference, lane); break; + case 8: _bit_unpack_32_lane<8>(in, out, reference, lane); break; + case 9: _bit_unpack_32_lane<9>(in, out, reference, lane); break; + case 10: _bit_unpack_32_lane<10>(in, out, reference, lane); break; + case 11: _bit_unpack_32_lane<11>(in, out, reference, lane); break; + case 12: _bit_unpack_32_lane<12>(in, out, reference, lane); break; + case 13: _bit_unpack_32_lane<13>(in, out, reference, lane); break; + case 14: _bit_unpack_32_lane<14>(in, out, reference, lane); break; + case 15: _bit_unpack_32_lane<15>(in, out, reference, lane); break; + case 16: _bit_unpack_32_lane<16>(in, out, reference, lane); break; + case 17: _bit_unpack_32_lane<17>(in, out, reference, lane); break; + case 18: _bit_unpack_32_lane<18>(in, out, reference, lane); break; + case 19: _bit_unpack_32_lane<19>(in, out, reference, lane); break; + case 20: _bit_unpack_32_lane<20>(in, out, reference, lane); break; + case 21: _bit_unpack_32_lane<21>(in, out, reference, lane); break; + case 22: _bit_unpack_32_lane<22>(in, out, reference, lane); break; + case 23: _bit_unpack_32_lane<23>(in, out, reference, lane); break; + case 24: _bit_unpack_32_lane<24>(in, out, reference, lane); break; + case 25: _bit_unpack_32_lane<25>(in, out, reference, lane); break; + case 26: _bit_unpack_32_lane<26>(in, out, reference, lane); break; + case 27: _bit_unpack_32_lane<27>(in, out, reference, lane); break; + case 28: _bit_unpack_32_lane<28>(in, out, reference, lane); break; + case 29: _bit_unpack_32_lane<29>(in, out, reference, lane); break; + case 30: _bit_unpack_32_lane<30>(in, out, reference, lane); break; + case 31: _bit_unpack_32_lane<31>(in, out, reference, lane); break; + case 32: _bit_unpack_32_lane<32>(in, out, reference, lane); break; + } +} + diff --git a/vortex-cuda/kernels/src/bit_unpack_64.cu b/vortex-cuda/kernels/src/bit_unpack_64.cu index 581ee159376..9be1262f0f4 100644 --- a/vortex-cuda/kernels/src/bit_unpack_64.cu +++ b/vortex-cuda/kernels/src/bit_unpack_64.cu @@ -1,12581 +1,7 @@ // AUTO-GENERATED. Do not edit by hand! -#include -#include -#include -#include "fastlanes_common.cuh" +#include "bit_unpack_64_lanes.cuh" #include "patches.cuh" -template -__device__ void _bit_unpack_64_lane(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_64_lane<0>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 64; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_64_lane<1>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 1); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 1); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 1); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 1); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 1); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 1); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 1); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 1); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 1); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 1); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 1); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 1); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 1); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 1); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 1); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 1); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 1); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 1); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 1); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 1); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 1); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 1); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 1); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 1); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 1); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 1); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 1); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 1); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 1); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 1); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 1); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 1); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 1); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 1); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 1); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 1); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 1); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 1); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 1); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 1); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 1); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 1); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 1); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 1); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 1); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 1); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 1); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 1); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 1); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 1); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 1); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 1); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 1); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 1); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 1); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 1); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<2>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 2); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 2); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 2); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 2); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 2); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 2); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 2); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 2); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 2); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 2); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 2); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 2); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 2); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 2); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 2); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 2); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 2); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 2); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 2); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 2); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 2); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 2); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 2); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 2); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 2); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 2; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 2); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 2); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 2); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 2); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 2); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 2); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 2); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 2); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 2); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 2); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 2); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 2); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 2); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 2); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 2); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 2); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 2); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 2); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 2); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 2); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 2); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 2); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 2); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 2); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 2); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 2); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 2); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 2); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 2); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 2); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 2); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<3>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 3); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 3); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 3); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 3); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 3); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 3); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 3); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 3); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 3); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 3); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 3); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 3); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 3); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 3); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 3); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 3); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 1; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 3); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 3); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 3); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 3); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 3); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 3); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 3); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 3); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 3); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 3); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 3); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 3); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 3); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 3); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 3); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 3); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 3); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 3); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 3); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 3); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 1)) << 2; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 3); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 3); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 3); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 3); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 3); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 3); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 3); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 3); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 3); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 3); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 3); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 3); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 3); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 3); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 3); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 3); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 3); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 3); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 3); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 3); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<4>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 4; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 4); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 4); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 4); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 4); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 4); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 4); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 4); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 4); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 4); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 4); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 4); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 4); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 4); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 4); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 4); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<5>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 5); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 5); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 5); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 5); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 5); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 5); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 5); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 5); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 5); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 1)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 5); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 5); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 5); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 5); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 5); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 5); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 5); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 5); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 5); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 5); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 5); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 5); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 3; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 5); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 5); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 5); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 5); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 5); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 5); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 5); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 5); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 5); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 5); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 5); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 5); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 2; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 5); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 5); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 5); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 5); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 5); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 5); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 5); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 5); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 5); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 5); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 5); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 5); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 1; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 5); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 5); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 5); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 5); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 5); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 5); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 5); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 5); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 5); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 5); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 5); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<6>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 6); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 6); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 6); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 6); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 6); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 6); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 6); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 6); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 6); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 6); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 6); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 6); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 6); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 6); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 6); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 6); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 6); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 6); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 2; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 6); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 6); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 6); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 6); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 6); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 6); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 6); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 6); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 6); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 6; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 6); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 6); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 6); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 6); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 6); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 6); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 6); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 6); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 6); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 6); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 2)) << 4; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 6); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 6); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 6); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 6); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 6); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 6); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 6); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 6); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 6); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 6); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 2; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 6); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 6); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 6); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 6); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 6); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 6); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 6); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 6); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 6); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<7>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 7); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 7); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 7); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 7); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 7); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 7); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 7); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 7); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 1; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 7); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 7); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 7); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 7); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 7); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 7); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 7); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 7); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 5)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 7); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 7); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 7); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 7); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 7); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 7); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 7); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 7); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 3; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 7); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 7); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 7); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 7); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 7); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 7); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 7); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 7); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 3)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 7); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 7); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 7); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 7); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 7); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 7); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 7); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 7); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 5; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 7); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 7); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 7); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 7); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 7); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 7); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 7); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 7); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 6; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 7); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 7); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 7); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 7); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 7); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 7); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 7); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 7); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<8>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 8; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 8); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 8); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 8); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 8); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 8); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 8); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 8); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<9>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 9); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 9); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 9); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 9); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 9); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 9); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 9); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 1; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 9); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 9); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 9); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 9); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 9); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 9); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 9); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 9); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 9); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 9); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 9); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 9); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 3; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 9); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 9); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 9); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 9); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 9); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 9); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 5)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 9); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 9); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 9); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 9); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 9); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 9); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 5; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 9); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 9); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 9); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 9); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 9); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 9); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 3)) << 6; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 9); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 9); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 9); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 9); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 9); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 9); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 7; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 9); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 9); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 9); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 9); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 9); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 9); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 9); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 9); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 9); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 9); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 9); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 9); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<10>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 10); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 10); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 10); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 10); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 10); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 10); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 10); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 10); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 10); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 10); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 10); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 10); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 10); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 10); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 10); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 10); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 10); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 2; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 10); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 10); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 10); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 10); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 10); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 6; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 10); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 10); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 10); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 10); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 10); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 10; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 10); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 10); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 10); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 10); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 10); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 10); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 4; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 10); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 10); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 10); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 10); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 10); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 10); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 10); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 10); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 10); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 10); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 10); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 2; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 10); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 10); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 10); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 10); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 10); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 6; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 10); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 10); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 10); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 10); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 10); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<11>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 11); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 11); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 11); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 11); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 11); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 9; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 11); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 11); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 11); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 11); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 11); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 7; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 11); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 11); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 11); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 11); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 11); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 5; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 11); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 11); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 11); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 11); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 11); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 3; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 11); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 11); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 11); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 11); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 11); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 1; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 11); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 11); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 11); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 11); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 10; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 11); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 11); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 11); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 11); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 11); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 3)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 11); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 11); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 11); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 11); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 11); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 5)) << 6; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 11); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 11); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 11); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 11); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 11); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 7)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 11); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 11); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 11); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 11); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 11); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 9)) << 2; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 11); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 11); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 11); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 11); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<12>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 12; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 12); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 12); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 12); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 12); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 12); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 4; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 12); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 12); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 12); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 12); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 4)) << 8; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 12); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 12); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 12); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 12); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<13>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 13); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 13); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 13); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 13); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 1)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 13); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 13); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 13); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 13); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 11; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 13); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 13); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 13); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 13); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 13); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 13); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 13); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 13); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 9; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 13); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 13); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 13); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 13); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 5)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 13); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 13); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 13); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 13); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 7; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 13); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 13); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 13); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 13); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 7)) << 6; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 13); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 13); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 13); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 13); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 5; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 13); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 13); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 13); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 13); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 13); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 13); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 13); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 13); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 3; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 13); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 13); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 13); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 13); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 11)) << 2; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 13); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 13); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 13); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 13); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 1; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 13); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 13); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 13); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<14>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 14); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 14); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 14); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 14); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 14); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 14); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 14); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 14); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 2; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 14); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 14); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 14); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 10; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 14); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 14); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 14); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 14); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 14); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 14); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 14); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 14); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 14); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 14); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 14); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 6; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 14); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 14); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 14); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 14; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 14); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 14); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 14); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 14); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 6)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 14); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 14); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 14); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 14); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 2; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 14); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 14); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 14); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 10; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 14); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 14); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 14); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 14); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 4; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 14); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 14); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 14); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 12; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 14); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 14); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 14); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 14); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 6; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 14); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 14); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 14); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<15>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 15); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 15); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 15); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 15); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 11)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 15); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 15); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 15); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 15); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 15); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 15); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 15); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 15); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 15); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 15); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 1; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 15); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 15); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 15); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 5; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 15); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 15); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 15); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 9; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 15); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 15); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 15); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 13; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 15); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 15); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 15); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 15); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 2; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 15); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 15); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 15); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 6; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 15); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 15); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 15); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 5)) << 10; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 15); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 15); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 15); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 1)) << 14; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 15); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 15); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 15); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 15); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 3; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 15); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 15); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 15); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 7; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 15); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 15); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 15); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 11; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 15); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 15); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 15); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<16>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 16; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 16); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 16); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 16); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<17>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 17); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 17); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 17); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 4)) << 13; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 17); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 17); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 17); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 8)) << 9; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 17); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 17); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 17); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 5; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 17); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 17); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 17); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 16)) << 1; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 17); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 17); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 3)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 17); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 17); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 17); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 7)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 17); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 17); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 17); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 17); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 17); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 17); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 15)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 17); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 17); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 2)) << 15; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 17); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 17); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 17); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 11; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 17); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 17); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 17); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 7; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 17); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 17); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 17); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 3; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 17); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 17); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 1)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 17); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 17); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 17); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 5)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 17); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 17); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 17); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 17); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 17); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 17); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 13)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 17); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 17); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<18>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 18); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 18); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 18); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 10; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 18); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 18); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 18); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 2; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 18); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 18); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 18); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 18); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 18); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 18); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 18); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 14; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 18); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 18); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 18); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 6; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 18); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 18); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 18); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 18); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 18); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 18); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 18); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 18; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 18); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 18); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 18); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 10; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 18); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 18); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 18); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 2; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 18); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 18); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 6)) << 12; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 18); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 18); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 18); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 14)) << 4; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 18); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 18); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 14; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 18); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 18); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 18); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 6; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 18); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 18); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 18); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 18); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 18); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 10)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 18); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 18); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<19>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 19); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 19); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 19); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 12)) << 7; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 19); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 19); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 5)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 19); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 19); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 19); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 17)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 19); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 19); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 9; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 19); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 19); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 3)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 19); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 19); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 19); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 19); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 19); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 11; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 19); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 19); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 19); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 19); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 19); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 13)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 19); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 19); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 13; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 19); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 19); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 19); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 18)) << 1; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 19); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 19); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 11)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 19); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 19); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 15; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 19); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 19); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 19); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 3; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 19); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 19); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 10; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 19); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 19); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 17; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 19); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 19); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 19); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 5; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 19); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 19); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 7)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 19); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 19); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<20>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 20; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 20); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 20); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 20); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 4; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 20); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 20); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 8; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 20); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 20); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 12; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 20); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 20); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 4)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 20); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 20); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<21>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 21); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 21); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 21); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 21); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 21); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 19)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 21); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 21); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 3; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 21); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 21); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 17)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 21); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 21); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 5; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 21); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 21); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 21); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 21); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 7; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 21); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 21); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 21); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 21); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 9; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 21); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 21); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 11)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 21); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 21); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 11; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 21); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 21); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 9)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 21); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 21); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 13; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 21); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 21); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 7)) << 14; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 21); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 21); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 15; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 21); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 21); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 5)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 21); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 21); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 17; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 21); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 21); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 18; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 21); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 21); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 19; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 21); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 21); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 1)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 21); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 21); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<22>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 22); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 22); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 22); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 22); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 18; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 22); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 22); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 22); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 22); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 14; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 22); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 22); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 22); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 22); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 10; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 22); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 22); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 22); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 22); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 6; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 22); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 22); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 22); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 22); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 2; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 22); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 22; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 22); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 22); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 20; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 22); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 22); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 18; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 22); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 22); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 6)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 22); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 22); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 8)) << 14; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 22); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 22); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 10)) << 12; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 22); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 22); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 10; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 22); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 22); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 14)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 22); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 22); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 6; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 22); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 22); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 18)) << 4; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 22); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 22); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 2; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 22); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<23>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 23); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 23); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 5)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 23); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 23); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 10)) << 13; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 23); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 23); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 15)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 23); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 23); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 20)) << 3; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 23); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 21; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 23); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 23); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 7)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 23); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 23); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 11; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 23); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 23); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 17)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 23); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 23); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 1; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 23); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 19; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 23); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 23); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 9)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 23); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 23); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 9; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 23); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 23); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 19)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 23); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 1)) << 22; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 23); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 23); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 17; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 23); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 23); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 11)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 23); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 23); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 7; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 23); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 23); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 21)) << 2; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 23); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 3)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 23); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 23); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 15; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 23); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 23); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 13)) << 10; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 23); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 23); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 5; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 23); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<24>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 24; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 24); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 24); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 8)) << 16; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 24); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 24); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 8; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 24); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<25>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 25); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 25); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 11)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 25); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 25); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 3; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 25); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 17; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 25); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 25); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 19)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 25); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 5)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 25); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 25); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 9; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 25); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 23; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 25); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 25); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 25); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 25); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 1; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 25); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 15; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 25); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 25); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 21)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 25); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 7)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 25); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 25); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 7; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 25); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 21; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 25); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 25); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 15)) << 10; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 25); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 1)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 25); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 25); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 13; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 25); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 25); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 23)) << 2; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 25); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 25); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 25); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 5; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 25); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 19; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 25); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 25); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 17)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 25); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 3)) << 22; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 25); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 25); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 11; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 25); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<26>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 26); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 26); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 14)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 26); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 2)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 26); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 26); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 16)) << 10; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 26); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 22; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 26); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 26); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 18)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 26); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 26); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 26); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 6; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 26); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 18; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 26); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 26); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 26); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 26); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 26); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 2; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 26); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 14; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 26); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 26; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 26); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 26); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 14)) << 12; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 26); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 2)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 26); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 26); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 10; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 26); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 22; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 26); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 26); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 26); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 6)) << 20; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 26); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 26); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 6; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 26); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 8)) << 18; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 26); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 26); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 22)) << 4; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 26); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 26); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 26); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 2; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 26); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 14; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 26); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<27>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 27); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 27); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 17)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 27); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 27); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 27); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 3; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 27); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 13; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 27); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 23; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 27); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 27); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 21)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 27); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 27); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 27); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 27); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 9; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 27); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 19; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 27); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 27); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 25)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 27); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 15)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 27); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 5)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 27); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 27); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 5; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 27); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 15; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 27); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 25; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 27); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 27); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 19)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 27); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 9)) << 18; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 27); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 27); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 1; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 27); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 11; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 27); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 21; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 27); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 27); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 23)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 27); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 14; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 27); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 3)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 27); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 27); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 7; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 27); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 17; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 27); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<28>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 28; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 28); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 28); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 8; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 28); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 12)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 28); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 24; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 28); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 28); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 4; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 28); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 12; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 28); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 20; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 28); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<29>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 29); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 29); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 23)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 29); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 17)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 29); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 11)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 29); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 5)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 29); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 29); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 28)) << 1; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 29); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 22)) << 7; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 29); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 16)) << 13; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 29); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 19; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 29); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 25; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 29); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 29); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 27)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 29); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 21)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 29); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 15)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 29); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 9)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 29); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 3)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 29); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 29); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 3; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 29); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 9; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 29); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 15; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 29); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 21; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 29); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 27; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 29); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 29); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 25)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 29); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 19)) << 10; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 29); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 13)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 29); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 7)) << 22; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 29); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 1)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 29); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 29); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 5; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 29); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 18)) << 11; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 29); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 17; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 29); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 23; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 29); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<30>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 30); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 30); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 26)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 30); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 30); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 30); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 30); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 30); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 30); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 30); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 30); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 2; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 30); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 6; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 30); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 10; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 30); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 14; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 30); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 18; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 30); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 22; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 30); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 26; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 30); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 30; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 30); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 30); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 4; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 30); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 22)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 30); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 12; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 30); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 14)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 30); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 10)) << 20; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 30); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 30); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 2)) << 28; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 30); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 30); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 2; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 30); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 6; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 30); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 10; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 30); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 14; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 30); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 18; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 30); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 22; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 30); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 26; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 30); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<31>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 31); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 31); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 29)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 31); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 27)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 31); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 25)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 31); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 23)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 31); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 21)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 31); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 19)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 31); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 17)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 31); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 15)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 31); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 13)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 31); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 11)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 31); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 9)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 31); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 7)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 31); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 5)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 31); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 3)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 31); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 1)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 31); - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 31); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 1; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 31); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 3; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 31); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 5; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 31); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 7; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 31); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 9; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 31); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 11; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 31); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 13; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 31); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 15; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 31); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 17; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 31); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 19; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 31); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 21; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 31); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 23; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 31); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 25; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 31); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 27; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 31); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 2)) << 29; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 31); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<32>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 0)) << 32; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 32); - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<33>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 33); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 2)) << 31; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 33); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 29; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 33); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 6)) << 27; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 33); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 25; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 33); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 23; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 33); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 21; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 33); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 19; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 33); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 17; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 33); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 15; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 33); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 13; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 33); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 22)) << 11; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 33); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 9; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 33); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 26)) << 7; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 33); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 5; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 33); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 3; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 33); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 1; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 1)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 33); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 30; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 33); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 5)) << 28; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 33); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 7)) << 26; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 33); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 9)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 33); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 11)) << 22; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 33); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 20; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 33); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 15)) << 18; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 33); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 17)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 33); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 19)) << 14; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 33); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 21)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 33); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 23)) << 10; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 33); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 25)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 33); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 27)) << 6; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 33); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 29)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 33); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 31)) << 2; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<34>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 34); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 4)) << 30; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 34); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 8)) << 26; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 34); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 22; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 34); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 16)) << 18; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 34); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 20)) << 14; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 34); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 24)) << 10; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 34); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 28)) << 6; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 34); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 32)) << 2; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 2)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 34); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 28; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 34); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 34); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 20; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 34); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 34); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 12; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 34); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 34); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 4; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 0)) << 34; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 34); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 4)) << 30; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 34); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 26; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 34); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 12)) << 22; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 34); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 16)) << 18; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 34); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 14; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 34); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 24)) << 10; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 34); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 28)) << 6; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 34); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 2; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 2)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 34); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 6)) << 28; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 34); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 10)) << 24; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 34); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 14)) << 20; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 34); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 34); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 22)) << 12; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 34); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 26)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 34); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 4; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<35>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 35); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 6)) << 29; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 35); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 12)) << 23; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 35); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 17; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 35); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 11; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 35); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 30)) << 5; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 34; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 35); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 7)) << 28; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 35); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 13)) << 22; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 35); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 19)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 35); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 25)) << 10; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 35); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 31)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 33; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 35); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 27; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 35); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 14)) << 21; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 35); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 15; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 35); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 9; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 35); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 3; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 35); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 26; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 35); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 15)) << 20; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 35); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 21)) << 14; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 35); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 27)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 35); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 33)) << 2; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 31; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 35); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 10)) << 25; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 35); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 19; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 35); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 22)) << 13; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 35); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 28)) << 7; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 35); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 1; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 5)) << 30; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 35); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 11)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 35); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 17)) << 18; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 35); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 23)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 35); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 6; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<36>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 36; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 36); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 28; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 36); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 20; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 36); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 24)) << 12; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 36); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 4; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 36); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 12)) << 24; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 36); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 20)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 36); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 8; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<37>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 37); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 10)) << 27; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 37); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 20)) << 17; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 37); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 30)) << 7; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 3)) << 34; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 37); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 13)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 37); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 23)) << 14; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 37); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 33)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 6)) << 31; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 37); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 16)) << 21; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 37); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 26)) << 11; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 37); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 36)) << 1; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 9)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 37); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 19)) << 18; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 37); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 29)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 2)) << 35; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 37); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 12)) << 25; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 37); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 22)) << 15; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 37); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 32)) << 5; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 5)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 37); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 15)) << 22; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 37); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 25)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 37); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 35)) << 2; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 8)) << 29; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 37); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 18)) << 19; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 37); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 28)) << 9; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 1)) << 36; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 37); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 11)) << 26; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 37); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 21)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 37); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 31)) << 6; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 4)) << 33; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 37); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 14)) << 23; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 37); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 24)) << 13; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 37); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 34)) << 3; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 7)) << 30; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 37); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 17)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 37); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 27)) << 10; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<38>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 38); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 12)) << 26; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 38); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 24)) << 14; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 38); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 36)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 10)) << 28; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 38); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 22)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 38); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 34)) << 4; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 30; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 38); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 20)) << 18; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 38); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 32)) << 6; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 6)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 38); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 18)) << 20; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 38); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 34; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 38); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 22; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 38); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 28)) << 10; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 36; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 38); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 38); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 12; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 0)) << 38; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 38); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 12)) << 26; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 38); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 14; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 38); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 36)) << 2; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 28; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 38); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 22)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 38); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 34)) << 4; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 8)) << 30; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 38); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 20)) << 18; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 38); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 6; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 6)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 38); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 20; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 38); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 30)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 34; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 38); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 16)) << 22; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 38); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 28)) << 10; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 2)) << 36; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 38); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 14)) << 24; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 38); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 12; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<39>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 39); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 14)) << 25; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 39); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 28)) << 11; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 3)) << 36; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 39); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 17)) << 22; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 39); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 31)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 6)) << 33; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 39); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 19; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 39); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 34)) << 5; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 30; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 39); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 23)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 39); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 37)) << 2; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 27; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 39); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 26)) << 13; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 1)) << 38; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 39); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 15)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 39); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 29)) << 10; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 35; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 39); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 21; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 39); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 7; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 7)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 39); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 21)) << 18; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 39); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 35)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 29; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 39); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 15; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 39); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 1; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 13)) << 26; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 39); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 27)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 2)) << 37; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 39); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 23; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 39); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 30)) << 9; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 5)) << 34; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 39); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 19)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 39); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 33)) << 6; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 31; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 39); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 22)) << 17; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 39); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 3; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 11)) << 28; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 39); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 25)) << 14; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<40>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 0)) << 40; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 40); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 16)) << 24; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 40); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 8; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 8)) << 32; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 40); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 16; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<41>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 41); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 18)) << 23; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 41); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 36)) << 5; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 13)) << 28; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 41); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 31)) << 10; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 8)) << 33; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 41); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 26)) << 15; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 3)) << 38; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 41); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 21)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 41); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 39)) << 2; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 16)) << 25; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 41); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 34)) << 7; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 11)) << 30; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 41); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 29)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 6)) << 35; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 41); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 24)) << 17; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 1)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 41); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 19)) << 22; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 41); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 37)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 14)) << 27; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 41); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 32)) << 9; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 9)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 41); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 27)) << 14; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 4)) << 37; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 41); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 22)) << 19; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 41); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 40)) << 1; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 17)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 41); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 35)) << 6; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 12)) << 29; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 41); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 30)) << 11; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 7)) << 34; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 41); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 25)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 2)) << 39; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 41); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 20)) << 21; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 41); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 38)) << 3; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 15)) << 26; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 41); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 33)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 10)) << 31; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 41); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 28)) << 13; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 5)) << 36; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 41); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 23)) << 18; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<42>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 42); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 20)) << 22; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 42); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 40)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 18)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 42); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 38)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 26; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 42); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 36)) << 6; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 14)) << 28; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 42); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 34)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 30; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 42); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 10; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 42); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 12; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 34; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 42); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 14; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 36; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 42); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 38; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 42); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 18; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 40; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 42); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 20; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 42; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 42); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 22; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 42); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 2; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 18)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 42); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 4; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 26; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 42); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 6; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 14)) << 28; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 42); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 12)) << 30; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 42); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 10; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 10)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 42); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 12; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 34; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 42); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 14; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 6)) << 36; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 42); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 4)) << 38; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 42); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 18; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 2)) << 40; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 42); - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 20; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<43>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 43); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 22)) << 21; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 1)) << 42; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 43); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 23)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 2)) << 41; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 43); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 24)) << 19; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 3)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 43); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 25)) << 18; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 4)) << 39; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 43); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 26)) << 17; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 5)) << 38; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 43); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 27)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 6)) << 37; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 43); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 28)) << 15; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 7)) << 36; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 43); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 29)) << 14; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 8)) << 35; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 43); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 30)) << 13; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 9)) << 34; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 43); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 31)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 10)) << 33; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 43); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 32)) << 11; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 11)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 43); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 33)) << 10; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 12)) << 31; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 43); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 34)) << 9; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 13)) << 30; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 43); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 35)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 14)) << 29; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 43); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 36)) << 7; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 15)) << 28; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 43); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 37)) << 6; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 16)) << 27; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 43); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 38)) << 5; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 17)) << 26; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 43); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 39)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 18)) << 25; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 43); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 40)) << 3; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 19)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 43); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 41)) << 2; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 20)) << 23; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 43); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 42)) << 1; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 21)) << 22; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<44>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 0)) << 44; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 44); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 20; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 4)) << 40; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 44); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 28)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 8)) << 36; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 44); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 32)) << 12; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 12)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 44); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 36)) << 8; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 28; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 44); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 40)) << 4; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 20)) << 24; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<45>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 45); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 26)) << 19; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 7)) << 38; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 45); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 33)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 31; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 45); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 40)) << 5; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 21)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 43; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 45); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 17; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 9)) << 36; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 45); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 35)) << 10; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 29; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 45); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 42)) << 3; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 23)) << 22; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 41; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 45); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 15; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 11)) << 34; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 45); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 37)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 18)) << 27; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 45); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 1; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 25)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 39; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 45); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 13; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 45); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 39)) << 6; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 25; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 1)) << 44; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 45); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 27)) << 18; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 37; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 45); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 11; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 15)) << 30; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 45); - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 41)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 22)) << 23; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 3)) << 42; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 45); - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 10)) << 35; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 45); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 9; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 17)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 45); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 43)) << 2; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 21; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 5)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 45); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 31)) << 14; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 33; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 45); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 38)) << 7; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 19)) << 26; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<46>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 46); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 28)) << 18; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 10)) << 36; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 46); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 38)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 20)) << 26; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 2)) << 44; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 46); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 30)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 12)) << 34; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 46); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 40)) << 6; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 22)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 4)) << 42; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 46); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 32)) << 14; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 14)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 46); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 42)) << 4; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 24)) << 22; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 6)) << 40; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 46); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 34)) << 12; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 30; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 46); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 44)) << 2; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 20; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 38; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 46); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 36)) << 10; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 28; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 0)) << 46; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 46); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 28)) << 18; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 10)) << 36; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 46); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 38)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 20)) << 26; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 2)) << 44; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 46); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 30)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 12)) << 34; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 46); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 40)) << 6; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 22)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 4)) << 42; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 46); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 32)) << 14; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 14)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 46); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 42)) << 4; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 24)) << 22; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 6)) << 40; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 46); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 34)) << 12; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 16)) << 30; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 46); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 44)) << 2; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 26)) << 20; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 8)) << 38; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 46); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 36)) << 10; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 18)) << 28; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<47>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 47); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 30)) << 17; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 13)) << 34; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 47); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 43)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 26)) << 21; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 9)) << 38; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 47); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 39)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 22)) << 25; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 5)) << 42; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 47); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 35)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 18)) << 29; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 1)) << 46; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 47); - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 31)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 14)) << 33; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 47); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 44)) << 3; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 27)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 10)) << 37; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 47); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 40)) << 7; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 23)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 6)) << 41; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 47); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 36)) << 11; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 19)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 2)) << 45; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 47); - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 32)) << 15; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 15)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 47); - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 45)) << 2; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 28)) << 19; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 11)) << 36; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 47); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 41)) << 6; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 24)) << 23; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 7)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 47); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 37)) << 10; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 20)) << 27; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 3)) << 44; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 47); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 33)) << 14; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 16)) << 31; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 47); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 46)) << 1; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 29)) << 18; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 12)) << 35; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 47); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 42)) << 5; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 25)) << 22; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 8)) << 39; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 47); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 38)) << 9; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 21)) << 26; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 4)) << 43; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 47); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 34)) << 13; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 17)) << 30; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<48>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 0)) << 48; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 48); - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 32)) << 16; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 32; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<49>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 49); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 34)) << 15; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 19)) << 30; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 4)) << 45; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 49); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 38)) << 11; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 23)) << 26; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 41; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 49); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 42)) << 7; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 27)) << 22; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 12)) << 37; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 49); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 46)) << 3; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 31)) << 18; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 33; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 1)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 49); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 35)) << 14; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 20)) << 29; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 5)) << 44; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 49); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 39)) << 10; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 25; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 9)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 49); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 43)) << 6; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 28)) << 21; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 13)) << 36; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 49); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 47)) << 2; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 32)) << 17; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 17)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 2)) << 47; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 49); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 13; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 21)) << 28; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 6)) << 43; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 49); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 40)) << 9; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 25)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 10)) << 39; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 49); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 44)) << 5; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 20; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 14)) << 35; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 49); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 48)) << 1; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 33)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 18)) << 31; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 3)) << 46; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 49); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 37)) << 12; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 27; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 7)) << 42; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 49); - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 41)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 26)) << 23; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 11)) << 38; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 49); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 45)) << 4; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 30)) << 19; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 15)) << 34; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<50>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 50); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 36)) << 14; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 22)) << 28; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 8)) << 42; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 50); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 44)) << 6; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 30)) << 20; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 16)) << 34; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 2)) << 48; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 50); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 38)) << 12; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 26; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 10)) << 40; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 50); - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 46)) << 4; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 32)) << 18; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 18)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 46; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 50); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 40)) << 10; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 26)) << 24; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 38; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 50); - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 48)) << 2; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 34)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 30; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 44; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 50); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 42)) << 8; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 22; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 36; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 0)) << 50; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 50); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 36)) << 14; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 22)) << 28; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 42; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 50); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 44)) << 6; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 30)) << 20; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 16)) << 34; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 2)) << 48; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 50); - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 38)) << 12; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 26; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 10)) << 40; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 50); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 46)) << 4; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 18; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 18)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 4)) << 46; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 50); - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 40)) << 10; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 26)) << 24; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 38; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 50); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 48)) << 2; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 34)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 20)) << 30; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 6)) << 44; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 50); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 42)) << 8; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 28)) << 22; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 14)) << 36; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<51>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 51); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 38)) << 13; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 25)) << 26; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 12)) << 39; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 51); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 50)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 37)) << 14; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 24)) << 27; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 11)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 51); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 49)) << 2; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 36)) << 15; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 23)) << 28; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 10)) << 41; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 51); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 48)) << 3; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 35)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 29; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 42; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 51); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 47)) << 4; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 34)) << 17; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 21)) << 30; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 8)) << 43; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 51); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 46)) << 5; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 33)) << 18; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 20)) << 31; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 7)) << 44; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 51); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 45)) << 6; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 32)) << 19; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 19)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 6)) << 45; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 51); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 44)) << 7; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 31)) << 20; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 33; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 5)) << 46; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 51); - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 43)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 21; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 17)) << 34; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 4)) << 47; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 51); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 42)) << 9; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 29)) << 22; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 16)) << 35; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 3)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 51); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 41)) << 10; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 28)) << 23; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 15)) << 36; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 2)) << 49; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 51); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 40)) << 11; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 27)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 14)) << 37; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 1)) << 50; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 51); - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 39)) << 12; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 26)) << 25; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 13)) << 38; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<52>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 0)) << 52; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 52); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 40)) << 12; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 28)) << 24; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 16)) << 36; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 4)) << 48; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 52); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 44)) << 8; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 32)) << 20; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 20)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 8)) << 44; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 52); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 48)) << 4; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 36)) << 16; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 24)) << 28; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 12)) << 40; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<53>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 53); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 42)) << 11; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 31)) << 22; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 20)) << 33; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 9)) << 44; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 53); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 51)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 40)) << 13; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 29)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 18)) << 35; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 7)) << 46; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 53); - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 49)) << 4; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 38)) << 15; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 27)) << 26; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 16)) << 37; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 5)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 53); - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 47)) << 6; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 36)) << 17; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 25)) << 28; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 14)) << 39; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 3)) << 50; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 53); - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 45)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 34)) << 19; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 23)) << 30; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 12)) << 41; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 1)) << 52; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 53); - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 43)) << 10; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 32)) << 21; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 21)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 10)) << 43; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 53); - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 52)) << 1; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 41)) << 12; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 30)) << 23; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 19)) << 34; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 8)) << 45; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 53); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 50)) << 3; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 39)) << 14; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 28)) << 25; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 17)) << 36; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 6)) << 47; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 53); - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 48)) << 5; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 37)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 26)) << 27; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 15)) << 38; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 4)) << 49; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 53); - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 46)) << 7; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 35)) << 18; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 24)) << 29; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 13)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 2)) << 51; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 53); - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 44)) << 9; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 33)) << 20; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 22)) << 31; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 11)) << 42; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<54>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 54); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 44)) << 10; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 34)) << 20; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 24)) << 30; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 14)) << 40; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 4)) << 50; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 54); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 48)) << 6; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 38)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 26; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 18)) << 36; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 8)) << 46; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 54); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 52)) << 2; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 42)) << 12; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 32)) << 22; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 22)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 12)) << 42; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 52; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 54); - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 46)) << 8; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 36)) << 18; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 26)) << 28; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 16)) << 38; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 6)) << 48; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 54); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 50)) << 4; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 14; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 30)) << 24; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 34; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 44; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 0)) << 54; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 54); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 44)) << 10; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 20; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 24)) << 30; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 14)) << 40; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 50; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 54); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 48)) << 6; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 38)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 26; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 18)) << 36; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 8)) << 46; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 54); - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 52)) << 2; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 42)) << 12; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 32)) << 22; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 42; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 2)) << 52; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 54); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 46)) << 8; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 36)) << 18; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 26)) << 28; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 38; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 6)) << 48; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 54); - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 50)) << 4; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 40)) << 14; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 30)) << 24; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 20)) << 34; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 10)) << 44; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<55>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 55); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 46)) << 9; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 37)) << 18; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 28)) << 27; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 19)) << 36; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 10)) << 45; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 1)) << 54; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 55); - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 47)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 38)) << 17; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 29)) << 26; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 35; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 11)) << 44; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 2)) << 53; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 55); - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 48)) << 7; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 39)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 30)) << 25; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 21)) << 34; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 12)) << 43; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 3)) << 52; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 55); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 49)) << 6; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 15; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 31)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 22)) << 33; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 13)) << 42; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 4)) << 51; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 55); - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 50)) << 5; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 41)) << 14; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 32)) << 23; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 23)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 14)) << 41; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 5)) << 50; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 55); - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 51)) << 4; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 42)) << 13; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 33)) << 22; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 24)) << 31; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 15)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 6)) << 49; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 55); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 52)) << 3; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 43)) << 12; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 34)) << 21; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 25)) << 30; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 39; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 7)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 55); - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 53)) << 2; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 44)) << 11; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 35)) << 20; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 26)) << 29; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 17)) << 38; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 8)) << 47; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 55); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 54)) << 1; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 45)) << 10; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 36)) << 19; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 27)) << 28; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 18)) << 37; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 9)) << 46; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<56>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 0)) << 56; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 56); - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 48)) << 8; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 40)) << 16; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 32)) << 24; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 24)) << 32; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 16)) << 40; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 8)) << 48; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<57>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 57); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 50)) << 7; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 43)) << 14; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 36)) << 21; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 29)) << 28; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 22)) << 35; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 15)) << 42; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 8)) << 49; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 1)) << 56; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 57); - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 51)) << 6; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 44)) << 13; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 37)) << 20; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 30)) << 27; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 23)) << 34; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 16)) << 41; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 9)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 2)) << 55; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 57); - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 52)) << 5; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 45)) << 12; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 38)) << 19; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 31)) << 26; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 24)) << 33; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 17)) << 40; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 10)) << 47; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 3)) << 54; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 57); - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 53)) << 4; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 46)) << 11; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 39)) << 18; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 32)) << 25; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 25)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 18)) << 39; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 11)) << 46; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 4)) << 53; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 57); - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 54)) << 3; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 47)) << 10; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 40)) << 17; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 33)) << 24; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 31; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 19)) << 38; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 12)) << 45; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 5)) << 52; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 57); - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 55)) << 2; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 48)) << 9; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 41)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 34)) << 23; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 27)) << 30; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 20)) << 37; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 13)) << 44; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 6)) << 51; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 57); - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 56)) << 1; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 49)) << 8; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 42)) << 15; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 35)) << 22; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 28)) << 29; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 21)) << 36; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 14)) << 43; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 7)) << 50; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<58>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 58); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 52)) << 6; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 46)) << 12; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 40)) << 18; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 34)) << 24; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 28)) << 30; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 22)) << 36; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 16)) << 42; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 10)) << 48; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 4)) << 54; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 58); - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 56)) << 2; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 50)) << 8; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 44)) << 14; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 38)) << 20; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 32)) << 26; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 26)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 20)) << 38; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 14)) << 44; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 8)) << 50; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 2)) << 56; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 58); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 54)) << 4; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 48)) << 10; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 42)) << 16; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 36)) << 22; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 30)) << 28; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 24)) << 34; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 18)) << 40; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 46; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 52; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 0)) << 58; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 58); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 52)) << 6; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 46)) << 12; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 40)) << 18; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 34)) << 24; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 28)) << 30; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 22)) << 36; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 16)) << 42; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 10)) << 48; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 4)) << 54; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 58); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 56)) << 2; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 50)) << 8; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 44)) << 14; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 38)) << 20; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 32)) << 26; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 26)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 20)) << 38; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 14)) << 44; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 8)) << 50; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 2)) << 56; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 58); - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 54)) << 4; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 48)) << 10; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 42)) << 16; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 36)) << 22; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 30)) << 28; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 24)) << 34; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 18)) << 40; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 12)) << 46; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 6)) << 52; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<59>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 59); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 54)) << 5; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 49)) << 10; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 44)) << 15; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 39)) << 20; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 34)) << 25; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 29)) << 30; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 24)) << 35; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 19)) << 40; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 14)) << 45; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 9)) << 50; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 4)) << 55; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 59); - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 58)) << 1; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 53)) << 6; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 48)) << 11; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 43)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 38)) << 21; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 33)) << 26; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 28)) << 31; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 23)) << 36; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 18)) << 41; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 13)) << 46; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 8)) << 51; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 3)) << 56; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 59); - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 57)) << 2; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 52)) << 7; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 47)) << 12; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 42)) << 17; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 37)) << 22; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 32)) << 27; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 27)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 22)) << 37; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 17)) << 42; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 12)) << 47; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 7)) << 52; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 2)) << 57; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 59); - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 56)) << 3; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 51)) << 8; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 46)) << 13; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 41)) << 18; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 36)) << 23; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 31)) << 28; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 26)) << 33; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 21)) << 38; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 16)) << 43; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 11)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 6)) << 53; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 1)) << 58; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 59); - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 55)) << 4; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 50)) << 9; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 45)) << 14; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 40)) << 19; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 35)) << 24; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 30)) << 29; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 25)) << 34; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 20)) << 39; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 15)) << 44; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 10)) << 49; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 5)) << 54; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<60>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 0)) << 60; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 60); - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 56)) << 4; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 52)) << 8; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 48)) << 12; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 44)) << 16; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 40)) << 20; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 36)) << 24; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 32)) << 28; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 28)) << 32; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 24)) << 36; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 20)) << 40; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 16)) << 44; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 12)) << 48; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 8)) << 52; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 4)) << 56; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<61>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 61); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 58)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 55)) << 6; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 52)) << 9; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 49)) << 12; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 46)) << 15; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 43)) << 18; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 40)) << 21; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 37)) << 24; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 34)) << 27; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 31)) << 30; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 28)) << 33; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 25)) << 36; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 22)) << 39; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 19)) << 42; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 16)) << 45; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 13)) << 48; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 10)) << 51; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 7)) << 54; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 4)) << 57; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 1)) << 60; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 61); - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 59)) << 2; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 56)) << 5; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 53)) << 8; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 50)) << 11; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 47)) << 14; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 44)) << 17; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 41)) << 20; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 38)) << 23; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 35)) << 26; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 32)) << 29; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 29)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 26)) << 35; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 23)) << 38; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 20)) << 41; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 17)) << 44; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 14)) << 47; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 11)) << 50; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 8)) << 53; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 5)) << 56; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 2)) << 59; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 61); - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 60)) << 1; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 57)) << 4; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 54)) << 7; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 51)) << 10; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 48)) << 13; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 45)) << 16; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 42)) << 19; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 39)) << 22; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 36)) << 25; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 33)) << 28; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 30)) << 31; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 27)) << 34; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 24)) << 37; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 21)) << 40; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 18)) << 43; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 15)) << 46; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 12)) << 49; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 9)) << 52; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 6)) << 55; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 3)) << 58; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 61); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<62>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 62); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 60)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 58)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 56)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 54)) << 8; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 52)) << 10; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 50)) << 12; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 48)) << 14; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 46)) << 16; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 44)) << 18; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 42)) << 20; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 40)) << 22; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 38)) << 24; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 36)) << 26; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 34)) << 28; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 32)) << 30; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 30)) << 32; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 28)) << 34; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 26)) << 36; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 24)) << 38; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 22)) << 40; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 20)) << 42; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 18)) << 44; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 16)) << 46; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 14)) << 48; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 12)) << 50; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 10)) << 52; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 8)) << 54; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 6)) << 56; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 4)) << 58; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 2)) << 60; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 0)) << 62; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint64_t, 62); - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 60)) << 2; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 58)) << 4; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 56)) << 6; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 54)) << 8; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 52)) << 10; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 50)) << 12; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 48)) << 14; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 46)) << 16; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 44)) << 18; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 42)) << 20; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 40)) << 22; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 38)) << 24; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 36)) << 26; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 34)) << 28; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 32)) << 30; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 30)) << 32; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 28)) << 34; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 26)) << 36; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 24)) << 38; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 22)) << 40; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 20)) << 42; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 18)) << 44; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 16)) << 46; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 14)) << 48; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 12)) << 50; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 10)) << 52; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 8)) << 54; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 6)) << 56; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 4)) << 58; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 61]; - tmp |= (src & MASK(uint64_t, 2)) << 60; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<63>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - uint64_t src; - uint64_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint64_t, 63); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 63) & MASK(uint64_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint64_t, 62)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 62) & MASK(uint64_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint64_t, 61)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 61) & MASK(uint64_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint64_t, 60)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 60) & MASK(uint64_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint64_t, 59)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 59) & MASK(uint64_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint64_t, 58)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 58) & MASK(uint64_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint64_t, 57)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 57) & MASK(uint64_t, 7); - src = in[lane + LANE_COUNT * 7]; - tmp |= (src & MASK(uint64_t, 56)) << 7; - out[INDEX(7, lane)] = tmp + reference; - tmp = (src >> 56) & MASK(uint64_t, 8); - src = in[lane + LANE_COUNT * 8]; - tmp |= (src & MASK(uint64_t, 55)) << 8; - out[INDEX(8, lane)] = tmp + reference; - tmp = (src >> 55) & MASK(uint64_t, 9); - src = in[lane + LANE_COUNT * 9]; - tmp |= (src & MASK(uint64_t, 54)) << 9; - out[INDEX(9, lane)] = tmp + reference; - tmp = (src >> 54) & MASK(uint64_t, 10); - src = in[lane + LANE_COUNT * 10]; - tmp |= (src & MASK(uint64_t, 53)) << 10; - out[INDEX(10, lane)] = tmp + reference; - tmp = (src >> 53) & MASK(uint64_t, 11); - src = in[lane + LANE_COUNT * 11]; - tmp |= (src & MASK(uint64_t, 52)) << 11; - out[INDEX(11, lane)] = tmp + reference; - tmp = (src >> 52) & MASK(uint64_t, 12); - src = in[lane + LANE_COUNT * 12]; - tmp |= (src & MASK(uint64_t, 51)) << 12; - out[INDEX(12, lane)] = tmp + reference; - tmp = (src >> 51) & MASK(uint64_t, 13); - src = in[lane + LANE_COUNT * 13]; - tmp |= (src & MASK(uint64_t, 50)) << 13; - out[INDEX(13, lane)] = tmp + reference; - tmp = (src >> 50) & MASK(uint64_t, 14); - src = in[lane + LANE_COUNT * 14]; - tmp |= (src & MASK(uint64_t, 49)) << 14; - out[INDEX(14, lane)] = tmp + reference; - tmp = (src >> 49) & MASK(uint64_t, 15); - src = in[lane + LANE_COUNT * 15]; - tmp |= (src & MASK(uint64_t, 48)) << 15; - out[INDEX(15, lane)] = tmp + reference; - tmp = (src >> 48) & MASK(uint64_t, 16); - src = in[lane + LANE_COUNT * 16]; - tmp |= (src & MASK(uint64_t, 47)) << 16; - out[INDEX(16, lane)] = tmp + reference; - tmp = (src >> 47) & MASK(uint64_t, 17); - src = in[lane + LANE_COUNT * 17]; - tmp |= (src & MASK(uint64_t, 46)) << 17; - out[INDEX(17, lane)] = tmp + reference; - tmp = (src >> 46) & MASK(uint64_t, 18); - src = in[lane + LANE_COUNT * 18]; - tmp |= (src & MASK(uint64_t, 45)) << 18; - out[INDEX(18, lane)] = tmp + reference; - tmp = (src >> 45) & MASK(uint64_t, 19); - src = in[lane + LANE_COUNT * 19]; - tmp |= (src & MASK(uint64_t, 44)) << 19; - out[INDEX(19, lane)] = tmp + reference; - tmp = (src >> 44) & MASK(uint64_t, 20); - src = in[lane + LANE_COUNT * 20]; - tmp |= (src & MASK(uint64_t, 43)) << 20; - out[INDEX(20, lane)] = tmp + reference; - tmp = (src >> 43) & MASK(uint64_t, 21); - src = in[lane + LANE_COUNT * 21]; - tmp |= (src & MASK(uint64_t, 42)) << 21; - out[INDEX(21, lane)] = tmp + reference; - tmp = (src >> 42) & MASK(uint64_t, 22); - src = in[lane + LANE_COUNT * 22]; - tmp |= (src & MASK(uint64_t, 41)) << 22; - out[INDEX(22, lane)] = tmp + reference; - tmp = (src >> 41) & MASK(uint64_t, 23); - src = in[lane + LANE_COUNT * 23]; - tmp |= (src & MASK(uint64_t, 40)) << 23; - out[INDEX(23, lane)] = tmp + reference; - tmp = (src >> 40) & MASK(uint64_t, 24); - src = in[lane + LANE_COUNT * 24]; - tmp |= (src & MASK(uint64_t, 39)) << 24; - out[INDEX(24, lane)] = tmp + reference; - tmp = (src >> 39) & MASK(uint64_t, 25); - src = in[lane + LANE_COUNT * 25]; - tmp |= (src & MASK(uint64_t, 38)) << 25; - out[INDEX(25, lane)] = tmp + reference; - tmp = (src >> 38) & MASK(uint64_t, 26); - src = in[lane + LANE_COUNT * 26]; - tmp |= (src & MASK(uint64_t, 37)) << 26; - out[INDEX(26, lane)] = tmp + reference; - tmp = (src >> 37) & MASK(uint64_t, 27); - src = in[lane + LANE_COUNT * 27]; - tmp |= (src & MASK(uint64_t, 36)) << 27; - out[INDEX(27, lane)] = tmp + reference; - tmp = (src >> 36) & MASK(uint64_t, 28); - src = in[lane + LANE_COUNT * 28]; - tmp |= (src & MASK(uint64_t, 35)) << 28; - out[INDEX(28, lane)] = tmp + reference; - tmp = (src >> 35) & MASK(uint64_t, 29); - src = in[lane + LANE_COUNT * 29]; - tmp |= (src & MASK(uint64_t, 34)) << 29; - out[INDEX(29, lane)] = tmp + reference; - tmp = (src >> 34) & MASK(uint64_t, 30); - src = in[lane + LANE_COUNT * 30]; - tmp |= (src & MASK(uint64_t, 33)) << 30; - out[INDEX(30, lane)] = tmp + reference; - tmp = (src >> 33) & MASK(uint64_t, 31); - src = in[lane + LANE_COUNT * 31]; - tmp |= (src & MASK(uint64_t, 32)) << 31; - out[INDEX(31, lane)] = tmp + reference; - tmp = (src >> 32) & MASK(uint64_t, 32); - src = in[lane + LANE_COUNT * 32]; - tmp |= (src & MASK(uint64_t, 31)) << 32; - out[INDEX(32, lane)] = tmp + reference; - tmp = (src >> 31) & MASK(uint64_t, 33); - src = in[lane + LANE_COUNT * 33]; - tmp |= (src & MASK(uint64_t, 30)) << 33; - out[INDEX(33, lane)] = tmp + reference; - tmp = (src >> 30) & MASK(uint64_t, 34); - src = in[lane + LANE_COUNT * 34]; - tmp |= (src & MASK(uint64_t, 29)) << 34; - out[INDEX(34, lane)] = tmp + reference; - tmp = (src >> 29) & MASK(uint64_t, 35); - src = in[lane + LANE_COUNT * 35]; - tmp |= (src & MASK(uint64_t, 28)) << 35; - out[INDEX(35, lane)] = tmp + reference; - tmp = (src >> 28) & MASK(uint64_t, 36); - src = in[lane + LANE_COUNT * 36]; - tmp |= (src & MASK(uint64_t, 27)) << 36; - out[INDEX(36, lane)] = tmp + reference; - tmp = (src >> 27) & MASK(uint64_t, 37); - src = in[lane + LANE_COUNT * 37]; - tmp |= (src & MASK(uint64_t, 26)) << 37; - out[INDEX(37, lane)] = tmp + reference; - tmp = (src >> 26) & MASK(uint64_t, 38); - src = in[lane + LANE_COUNT * 38]; - tmp |= (src & MASK(uint64_t, 25)) << 38; - out[INDEX(38, lane)] = tmp + reference; - tmp = (src >> 25) & MASK(uint64_t, 39); - src = in[lane + LANE_COUNT * 39]; - tmp |= (src & MASK(uint64_t, 24)) << 39; - out[INDEX(39, lane)] = tmp + reference; - tmp = (src >> 24) & MASK(uint64_t, 40); - src = in[lane + LANE_COUNT * 40]; - tmp |= (src & MASK(uint64_t, 23)) << 40; - out[INDEX(40, lane)] = tmp + reference; - tmp = (src >> 23) & MASK(uint64_t, 41); - src = in[lane + LANE_COUNT * 41]; - tmp |= (src & MASK(uint64_t, 22)) << 41; - out[INDEX(41, lane)] = tmp + reference; - tmp = (src >> 22) & MASK(uint64_t, 42); - src = in[lane + LANE_COUNT * 42]; - tmp |= (src & MASK(uint64_t, 21)) << 42; - out[INDEX(42, lane)] = tmp + reference; - tmp = (src >> 21) & MASK(uint64_t, 43); - src = in[lane + LANE_COUNT * 43]; - tmp |= (src & MASK(uint64_t, 20)) << 43; - out[INDEX(43, lane)] = tmp + reference; - tmp = (src >> 20) & MASK(uint64_t, 44); - src = in[lane + LANE_COUNT * 44]; - tmp |= (src & MASK(uint64_t, 19)) << 44; - out[INDEX(44, lane)] = tmp + reference; - tmp = (src >> 19) & MASK(uint64_t, 45); - src = in[lane + LANE_COUNT * 45]; - tmp |= (src & MASK(uint64_t, 18)) << 45; - out[INDEX(45, lane)] = tmp + reference; - tmp = (src >> 18) & MASK(uint64_t, 46); - src = in[lane + LANE_COUNT * 46]; - tmp |= (src & MASK(uint64_t, 17)) << 46; - out[INDEX(46, lane)] = tmp + reference; - tmp = (src >> 17) & MASK(uint64_t, 47); - src = in[lane + LANE_COUNT * 47]; - tmp |= (src & MASK(uint64_t, 16)) << 47; - out[INDEX(47, lane)] = tmp + reference; - tmp = (src >> 16) & MASK(uint64_t, 48); - src = in[lane + LANE_COUNT * 48]; - tmp |= (src & MASK(uint64_t, 15)) << 48; - out[INDEX(48, lane)] = tmp + reference; - tmp = (src >> 15) & MASK(uint64_t, 49); - src = in[lane + LANE_COUNT * 49]; - tmp |= (src & MASK(uint64_t, 14)) << 49; - out[INDEX(49, lane)] = tmp + reference; - tmp = (src >> 14) & MASK(uint64_t, 50); - src = in[lane + LANE_COUNT * 50]; - tmp |= (src & MASK(uint64_t, 13)) << 50; - out[INDEX(50, lane)] = tmp + reference; - tmp = (src >> 13) & MASK(uint64_t, 51); - src = in[lane + LANE_COUNT * 51]; - tmp |= (src & MASK(uint64_t, 12)) << 51; - out[INDEX(51, lane)] = tmp + reference; - tmp = (src >> 12) & MASK(uint64_t, 52); - src = in[lane + LANE_COUNT * 52]; - tmp |= (src & MASK(uint64_t, 11)) << 52; - out[INDEX(52, lane)] = tmp + reference; - tmp = (src >> 11) & MASK(uint64_t, 53); - src = in[lane + LANE_COUNT * 53]; - tmp |= (src & MASK(uint64_t, 10)) << 53; - out[INDEX(53, lane)] = tmp + reference; - tmp = (src >> 10) & MASK(uint64_t, 54); - src = in[lane + LANE_COUNT * 54]; - tmp |= (src & MASK(uint64_t, 9)) << 54; - out[INDEX(54, lane)] = tmp + reference; - tmp = (src >> 9) & MASK(uint64_t, 55); - src = in[lane + LANE_COUNT * 55]; - tmp |= (src & MASK(uint64_t, 8)) << 55; - out[INDEX(55, lane)] = tmp + reference; - tmp = (src >> 8) & MASK(uint64_t, 56); - src = in[lane + LANE_COUNT * 56]; - tmp |= (src & MASK(uint64_t, 7)) << 56; - out[INDEX(56, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint64_t, 57); - src = in[lane + LANE_COUNT * 57]; - tmp |= (src & MASK(uint64_t, 6)) << 57; - out[INDEX(57, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint64_t, 58); - src = in[lane + LANE_COUNT * 58]; - tmp |= (src & MASK(uint64_t, 5)) << 58; - out[INDEX(58, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint64_t, 59); - src = in[lane + LANE_COUNT * 59]; - tmp |= (src & MASK(uint64_t, 4)) << 59; - out[INDEX(59, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint64_t, 60); - src = in[lane + LANE_COUNT * 60]; - tmp |= (src & MASK(uint64_t, 3)) << 60; - out[INDEX(60, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint64_t, 61); - src = in[lane + LANE_COUNT * 61]; - tmp |= (src & MASK(uint64_t, 2)) << 61; - out[INDEX(61, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint64_t, 62); - src = in[lane + LANE_COUNT * 62]; - tmp |= (src & MASK(uint64_t, 1)) << 62; - out[INDEX(62, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint64_t, 63); - out[INDEX(63, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_64_lane<64>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; - #pragma unroll - for (int row = 0; row < 64; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ inline void bit_unpack_64_lane( - const uint64_t *__restrict in, - uint64_t *__restrict out, - uint64_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_64_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_64_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_64_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_64_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_64_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_64_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_64_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_64_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_64_lane<8>(in, out, reference, lane); break; - case 9: _bit_unpack_64_lane<9>(in, out, reference, lane); break; - case 10: _bit_unpack_64_lane<10>(in, out, reference, lane); break; - case 11: _bit_unpack_64_lane<11>(in, out, reference, lane); break; - case 12: _bit_unpack_64_lane<12>(in, out, reference, lane); break; - case 13: _bit_unpack_64_lane<13>(in, out, reference, lane); break; - case 14: _bit_unpack_64_lane<14>(in, out, reference, lane); break; - case 15: _bit_unpack_64_lane<15>(in, out, reference, lane); break; - case 16: _bit_unpack_64_lane<16>(in, out, reference, lane); break; - case 17: _bit_unpack_64_lane<17>(in, out, reference, lane); break; - case 18: _bit_unpack_64_lane<18>(in, out, reference, lane); break; - case 19: _bit_unpack_64_lane<19>(in, out, reference, lane); break; - case 20: _bit_unpack_64_lane<20>(in, out, reference, lane); break; - case 21: _bit_unpack_64_lane<21>(in, out, reference, lane); break; - case 22: _bit_unpack_64_lane<22>(in, out, reference, lane); break; - case 23: _bit_unpack_64_lane<23>(in, out, reference, lane); break; - case 24: _bit_unpack_64_lane<24>(in, out, reference, lane); break; - case 25: _bit_unpack_64_lane<25>(in, out, reference, lane); break; - case 26: _bit_unpack_64_lane<26>(in, out, reference, lane); break; - case 27: _bit_unpack_64_lane<27>(in, out, reference, lane); break; - case 28: _bit_unpack_64_lane<28>(in, out, reference, lane); break; - case 29: _bit_unpack_64_lane<29>(in, out, reference, lane); break; - case 30: _bit_unpack_64_lane<30>(in, out, reference, lane); break; - case 31: _bit_unpack_64_lane<31>(in, out, reference, lane); break; - case 32: _bit_unpack_64_lane<32>(in, out, reference, lane); break; - case 33: _bit_unpack_64_lane<33>(in, out, reference, lane); break; - case 34: _bit_unpack_64_lane<34>(in, out, reference, lane); break; - case 35: _bit_unpack_64_lane<35>(in, out, reference, lane); break; - case 36: _bit_unpack_64_lane<36>(in, out, reference, lane); break; - case 37: _bit_unpack_64_lane<37>(in, out, reference, lane); break; - case 38: _bit_unpack_64_lane<38>(in, out, reference, lane); break; - case 39: _bit_unpack_64_lane<39>(in, out, reference, lane); break; - case 40: _bit_unpack_64_lane<40>(in, out, reference, lane); break; - case 41: _bit_unpack_64_lane<41>(in, out, reference, lane); break; - case 42: _bit_unpack_64_lane<42>(in, out, reference, lane); break; - case 43: _bit_unpack_64_lane<43>(in, out, reference, lane); break; - case 44: _bit_unpack_64_lane<44>(in, out, reference, lane); break; - case 45: _bit_unpack_64_lane<45>(in, out, reference, lane); break; - case 46: _bit_unpack_64_lane<46>(in, out, reference, lane); break; - case 47: _bit_unpack_64_lane<47>(in, out, reference, lane); break; - case 48: _bit_unpack_64_lane<48>(in, out, reference, lane); break; - case 49: _bit_unpack_64_lane<49>(in, out, reference, lane); break; - case 50: _bit_unpack_64_lane<50>(in, out, reference, lane); break; - case 51: _bit_unpack_64_lane<51>(in, out, reference, lane); break; - case 52: _bit_unpack_64_lane<52>(in, out, reference, lane); break; - case 53: _bit_unpack_64_lane<53>(in, out, reference, lane); break; - case 54: _bit_unpack_64_lane<54>(in, out, reference, lane); break; - case 55: _bit_unpack_64_lane<55>(in, out, reference, lane); break; - case 56: _bit_unpack_64_lane<56>(in, out, reference, lane); break; - case 57: _bit_unpack_64_lane<57>(in, out, reference, lane); break; - case 58: _bit_unpack_64_lane<58>(in, out, reference, lane); break; - case 59: _bit_unpack_64_lane<59>(in, out, reference, lane); break; - case 60: _bit_unpack_64_lane<60>(in, out, reference, lane); break; - case 61: _bit_unpack_64_lane<61>(in, out, reference, lane); break; - case 62: _bit_unpack_64_lane<62>(in, out, reference, lane); break; - case 63: _bit_unpack_64_lane<63>(in, out, reference, lane); break; - case 64: _bit_unpack_64_lane<64>(in, out, reference, lane); break; - } -} - template __device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint64_t shared_out[1024]; diff --git a/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh new file mode 100644 index 00000000000..0f20fee4e7d --- /dev/null +++ b/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh @@ -0,0 +1,12579 @@ +// AUTO-GENERATED. Do not edit by hand! +#pragma once + +#include +#include +#include +#include "fastlanes_common.cuh" + +template +__device__ void _bit_unpack_64_lane(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane); + +template <> +__device__ void _bit_unpack_64_lane<0>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + #pragma unroll + for (int row = 0; row < 64; row++) { + out[INDEX(row, lane)] = reference; + } +} + +template <> +__device__ void _bit_unpack_64_lane<1>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 1); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 1); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 1); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 1); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 1); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 1); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 1); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 1); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 1); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 1); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 1); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 1); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 1); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 1); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 1); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 1); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 1); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 1); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 1); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 1); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 1); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 1); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 1); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 1); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 1); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 1); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 1); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 1); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 1); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 1); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 1); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 1); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 1); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 1); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 1); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 1); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 1); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 1); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 1); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 1); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 1); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 1); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 1); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 1); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 1); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 1); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 1); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 1); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 1); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 1); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 1); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 1); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 1); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 1); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 1); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 1); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 1); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 1); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 1); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 1); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 1); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 1); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 1); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<2>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 2); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 2); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 2); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 2); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 2); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 2); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 2); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 2); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 2); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 2); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 2); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 2); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 2); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 2); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 2); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 2); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 2); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 2); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 2); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 2); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 2); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 2); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 2); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 2); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 2); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 2); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 2); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 2); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 2); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 2); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 2); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 0)) << 2; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 2); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 2); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 2); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 2); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 2); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 2); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 2); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 2); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 2); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 2); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 2); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 2); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 2); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 2); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 2); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 2); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 2); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 2); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 2); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 2); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 2); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 2); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 2); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 2); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 2); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 2); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 2); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 2); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 2); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 2); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 2); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<3>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 3); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 3); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 3); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 3); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 3); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 3); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 3); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 3); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 3); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 3); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 3); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 3); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 3); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 3); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 3); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 3); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 3); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 3); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 3); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 3); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 3); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 2)) << 1; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 3); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 3); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 3); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 3); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 3); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 3); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 3); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 3); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 3); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 3); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 3); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 3); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 3); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 3); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 3); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 3); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 3); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 3); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 3); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 3); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 1)) << 2; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 3); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 3); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 3); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 3); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 3); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 3); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 3); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 3); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 3); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 3); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 3); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 3); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 3); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 3); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 3); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 3); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 3); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 3); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 3); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 3); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<4>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 4); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 4); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 4); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 4); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 4); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 4); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 4); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 4); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 4); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 4); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 4); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 4); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 4); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 4); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 4); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 0)) << 4; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 4); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 4); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 4); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 4); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 4); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 4); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 4); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 4); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 4); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 4); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 4); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 4); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 4); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 4); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 4); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 0)) << 4; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 4); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 4); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 4); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 4); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 4); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 4); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 4); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 4); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 4); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 4); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 4); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 4); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 4); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 4); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 4); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 4; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 4); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 4); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 4); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 4); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 4); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 4); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 4); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 4); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 4); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 4); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 4); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 4); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 4); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 4); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 4); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<5>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 5); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 5); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 5); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 5); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 5); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 5); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 5); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 5); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 5); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 5); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 5); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 5); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 1)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 5); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 5); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 5); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 5); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 5); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 5); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 5); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 5); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 5); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 5); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 5); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 5); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 2)) << 3; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 5); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 5); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 5); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 5); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 5); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 5); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 5); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 5); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 5); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 5); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 5); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 5); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 3)) << 2; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 5); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 5); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 5); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 5); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 5); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 5); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 5); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 5); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 5); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 5); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 5); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 5); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 1; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 5); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 5); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 5); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 5); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 5); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 5); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 5); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 5); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 5); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 5); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 5); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<6>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 6); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 6); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 6); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 6); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 6); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 6); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 6); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 6); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 6); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 6); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 2)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 6); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 6); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 6); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 6); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 6); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 6); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 6); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 6); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 6); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 6); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 2; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 6); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 6); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 6); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 6); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 6); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 6); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 6); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 6); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 6); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 6; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 6); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 6); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 6); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 6); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 6); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 6); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 6); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 6); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 6); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 6); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 2)) << 4; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 6); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 6); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 6); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 6); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 6); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 6); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 6); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 6); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 6); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 6); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 2; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 6); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 6); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 6); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 6); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 6); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 6); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 6); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 6); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 6); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<7>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 7); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 7); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 7); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 7); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 7); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 7); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 7); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 7); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 7); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 6)) << 1; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 7); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 7); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 7); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 7); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 7); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 7); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 7); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 7); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 5)) << 2; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 7); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 7); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 7); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 7); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 7); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 7); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 7); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 7); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 4)) << 3; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 7); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 7); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 7); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 7); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 7); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 7); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 7); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 7); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 3)) << 4; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 7); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 7); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 7); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 7); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 7); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 7); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 7); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 7); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 2)) << 5; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 7); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 7); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 7); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 7); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 7); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 7); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 7); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 7); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 1)) << 6; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 7); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 7); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 7); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 7); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 7); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 7); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 7); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 7); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<8>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 8; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 8); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 8); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 8); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 8); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 8); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 8); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 8); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<9>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 9); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 9); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 9); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 9); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 9); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 9); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 9); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 8)) << 1; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 9); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 9); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 9); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 9); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 9); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 9); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 7)) << 2; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 9); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 9); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 9); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 9); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 9); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 9); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 6)) << 3; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 9); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 9); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 9); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 9); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 9); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 9); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 5)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 9); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 9); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 9); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 9); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 9); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 9); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 5; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 9); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 9); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 9); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 9); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 9); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 9); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 3)) << 6; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 9); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 9); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 9); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 9); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 9); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 9); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 7; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 9); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 9); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 9); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 9); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 9); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 9); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 1)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 9); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 9); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 9); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 9); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 9); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 9); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<10>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 10); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 10); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 10); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 10); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 10); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 10); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 6)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 10); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 10); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 10); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 10); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 10); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 2)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 10); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 10); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 10); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 10); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 10); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 10); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 8)) << 2; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 10); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 10); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 10); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 10); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 10); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 6; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 10); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 10); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 10); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 10); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 10); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 10; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 10); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 10); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 10); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 10); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 10); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 10); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 4; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 10); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 10); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 10); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 10); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 10); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 8; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 10); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 10); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 10); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 10); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 10); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 10); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 2; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 10); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 10); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 10); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 10); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 10); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 4)) << 6; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 10); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 10); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 10); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 10); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 10); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<11>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 11); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 11); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 11); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 11); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 11); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 2)) << 9; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 11); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 11); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 11); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 11); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 11); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 7; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 11); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 11); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 11); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 11); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 11); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 6)) << 5; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 11); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 11); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 11); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 11); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 11); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 3; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 11); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 11); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 11); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 11); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 11); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 1; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 11); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 11); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 11); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 11); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 1)) << 10; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 11); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 11); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 11); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 11); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 11); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 3)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 11); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 11); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 11); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 11); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 11); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 5)) << 6; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 11); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 11); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 11); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 11); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 11); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 7)) << 4; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 11); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 11); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 11); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 11); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 11); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 9)) << 2; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 11); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 11); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 11); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 11); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<12>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 12); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 12); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 12); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 12); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 12); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 8)) << 4; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 12); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 12); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 12); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 12); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 8; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 12); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 12); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 12); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 12); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 12; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 12); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 12); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 12); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 12); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 12); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 4; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 12); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 12); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 12); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 12); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 8; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 12); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 12); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 12); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 12); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 12; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 12); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 12); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 12); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 12); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 12); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 8)) << 4; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 12); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 12); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 12); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 12); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 4)) << 8; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 12); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 12); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 12); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 12); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 12; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 12); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 12); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 12); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 12); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 12); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 4; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 12); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 12); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 12); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 12); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 4)) << 8; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 12); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 12); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 12); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 12); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<13>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 13); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 13); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 13); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 13); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 1)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 13); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 13); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 13); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 13); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 2)) << 11; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 13); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 13); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 13); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 13); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 3)) << 10; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 13); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 13); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 13); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 13); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 9; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 13); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 13); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 13); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 13); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 5)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 13); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 13); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 13); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 13); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 7; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 13); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 13); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 13); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 13); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 7)) << 6; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 13); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 13); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 13); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 13); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 5; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 13); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 13); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 13); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 13); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 9)) << 4; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 13); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 13); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 13); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 13); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 10)) << 3; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 13); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 13); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 13); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 13); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 11)) << 2; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 13); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 13); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 13); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 13); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 1; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 13); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 13); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 13); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<14>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 14); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 14); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 14); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 14); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 6)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 14); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 14); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 14); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 14); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 12)) << 2; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 14); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 14); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 14); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 4)) << 10; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 14); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 14); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 14); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 14); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 10)) << 4; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 14); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 14); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 14); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 2)) << 12; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 14); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 14); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 14); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 14); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 8)) << 6; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 14); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 14); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 14); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 14; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 14); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 14); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 14); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 14); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 6)) << 8; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 14); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 14); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 14); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 14); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 12)) << 2; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 14); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 14); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 14); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 4)) << 10; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 14); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 14); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 14); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 14); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 4; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 14); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 14); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 14); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 2)) << 12; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 14); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 14); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 14); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 14); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 6; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 14); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 14); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 14); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<15>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 15); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 15); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 15); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 15); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 11)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 15); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 15); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 15); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 7)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 15); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 15); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 15); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 3)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 15); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 15); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 15); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 15); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 1; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 15); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 15); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 15); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 5; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 15); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 15); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 15); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 9; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 15); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 15); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 15); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 13; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 15); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 15); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 15); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 15); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 13)) << 2; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 15); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 15); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 15); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 9)) << 6; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 15); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 15); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 15); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 5)) << 10; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 15); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 15); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 15); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 1)) << 14; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 15); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 15); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 15); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 15); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 3; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 15); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 15); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 15); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 7; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 15); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 15); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 15); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 11; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 15); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 15); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 15); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<16>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 16; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 16); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 16); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 16); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<17>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 17); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 17); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 17); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 4)) << 13; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 17); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 17); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 17); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 8)) << 9; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 17); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 17); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 17); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 12)) << 5; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 17); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 17); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 17); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 16)) << 1; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 17); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 17); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 3)) << 14; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 17); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 17); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 17); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 7)) << 10; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 17); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 17); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 17); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 11)) << 6; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 17); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 17); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 17); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 15)) << 2; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 17); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 17); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 2)) << 15; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 17); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 17); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 17); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 6)) << 11; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 17); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 17); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 17); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 7; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 17); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 17); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 17); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 14)) << 3; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 17); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 17); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 1)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 17); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 17); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 17); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 5)) << 12; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 17); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 17); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 17); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 9)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 17); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 17); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 17); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 13)) << 4; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 17); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 17); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<18>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 18); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 18); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 18); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 8)) << 10; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 18); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 18); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 18); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 16)) << 2; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 18); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 18); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 6)) << 12; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 18); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 18); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 18); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 4; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 18); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 18); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 14; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 18); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 18); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 18); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 12)) << 6; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 18); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 18); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 18); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 18); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 18); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 10)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 18); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 18); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 18; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 18); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 18); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 18); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 10; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 18); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 18); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 18); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 2; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 18); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 18); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 6)) << 12; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 18); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 18); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 18); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 14)) << 4; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 18); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 18); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 14; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 18); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 18); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 18); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 12)) << 6; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 18); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 18); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 18); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 18); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 18); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 10)) << 8; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 18); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 18); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<19>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 19); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 19); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 19); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 12)) << 7; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 19); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 19); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 5)) << 14; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 19); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 19); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 19); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 17)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 19); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 19); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 10)) << 9; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 19); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 19); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 3)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 19); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 19); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 19); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 15)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 19); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 19); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 8)) << 11; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 19); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 19); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 1)) << 18; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 19); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 19); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 19); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 13)) << 6; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 19); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 19); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 6)) << 13; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 19); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 19); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 19); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 18)) << 1; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 19); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 19); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 11)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 19); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 19); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 4)) << 15; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 19); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 19); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 19); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 16)) << 3; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 19); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 19); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 9)) << 10; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 19); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 19); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 17; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 19); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 19); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 19); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 14)) << 5; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 19); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 19); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 7)) << 12; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 19); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 19); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<20>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 20); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 20); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 20); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 16)) << 4; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 20); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 20); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 12)) << 8; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 20); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 20); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 8)) << 12; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 20); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 20); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 16; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 20); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 20); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 20; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 20); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 20); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 20); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 16)) << 4; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 20); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 20); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 12)) << 8; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 20); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 20); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 12; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 20); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 20); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 4)) << 16; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 20); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 20); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 0)) << 20; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 20); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 20); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 20); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 4; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 20); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 20); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 8; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 20); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 20); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 12; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 20); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 20); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 16; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 20); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 20); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 20; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 20); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 20); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 20); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 16)) << 4; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 20); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 20); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 8; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 20); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 20); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 8)) << 12; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 20); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 20); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 4)) << 16; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 20); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 20); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<21>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 21); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 21); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 21); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 20)) << 1; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 21); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 21); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 19)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 21); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 21); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 18)) << 3; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 21); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 21); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 17)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 21); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 21); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 5; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 21); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 21); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 15)) << 6; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 21); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 21); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 14)) << 7; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 21); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 21); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 13)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 21); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 21); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 12)) << 9; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 21); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 21); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 11)) << 10; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 21); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 21); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 11; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 21); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 21); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 9)) << 12; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 21); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 21); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 13; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 21); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 21); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 7)) << 14; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 21); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 21); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 6)) << 15; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 21); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 21); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 5)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 21); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 21); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 17; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 21); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 21); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 3)) << 18; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 21); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 21); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 2)) << 19; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 21); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 21); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 1)) << 20; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 21); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 21); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<22>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 22); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 22); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 2)) << 20; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 22); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 22); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 18; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 22); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 22); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 6)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 22); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 22); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 14; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 22); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 22); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 12; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 22); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 22); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 12)) << 10; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 22); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 22); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 14)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 22); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 22); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 16)) << 6; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 22); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 22); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 18)) << 4; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 22); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 22); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 2; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 22); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 0)) << 22; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 22); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 22); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 2)) << 20; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 22); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 22); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 4)) << 18; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 22); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 22); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 6)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 22); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 22); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 8)) << 14; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 22); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 22); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 10)) << 12; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 22); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 22); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 10; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 22); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 22); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 14)) << 8; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 22); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 22); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 16)) << 6; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 22); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 22); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 18)) << 4; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 22); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 22); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 20)) << 2; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 22); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<23>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 23); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 23); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 5)) << 18; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 23); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 23); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 10)) << 13; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 23); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 23); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 15)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 23); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 23); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 20)) << 3; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 23); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 2)) << 21; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 23); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 23); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 7)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 23); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 23); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 12)) << 11; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 23); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 23); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 17)) << 6; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 23); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 23); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 22)) << 1; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 23); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 4)) << 19; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 23); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 23); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 9)) << 14; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 23); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 23); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 14)) << 9; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 23); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 23); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 19)) << 4; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 23); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 1)) << 22; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 23); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 23); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 6)) << 17; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 23); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 23); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 11)) << 12; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 23); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 23); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 16)) << 7; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 23); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 23); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 21)) << 2; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 23); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 3)) << 20; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 23); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 23); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 8)) << 15; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 23); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 23); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 13)) << 10; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 23); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 23); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 18)) << 5; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 23); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<24>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 24; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 24); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 24); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 8)) << 16; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 24); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 24); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 16)) << 8; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 24); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<25>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 25); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 25); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 11)) << 14; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 25); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 25); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 22)) << 3; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 25); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 8)) << 17; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 25); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 25); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 19)) << 6; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 25); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 5)) << 20; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 25); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 25); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 16)) << 9; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 25); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 23; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 25); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 25); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 13)) << 12; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 25); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 25); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 24)) << 1; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 25); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 10)) << 15; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 25); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 25); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 21)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 25); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 7)) << 18; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 25); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 25); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 18)) << 7; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 25); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 21; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 25); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 25); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 15)) << 10; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 25); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 1)) << 24; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 25); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 25); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 13; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 25); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 25); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 23)) << 2; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 25); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 9)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 25); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 25); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 20)) << 5; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 25); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 19; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 25); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 25); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 17)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 25); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 3)) << 22; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 25); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 25); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 14)) << 11; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 25); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<26>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 26); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 26); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 14)) << 12; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 26); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 2)) << 24; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 26); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 26); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 16)) << 10; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 26); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 22; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 26); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 26); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 18)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 26); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 20; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 26); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 26); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 20)) << 6; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 26); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 18; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 26); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 26); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 22)) << 4; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 26); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 10)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 26); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 26); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 24)) << 2; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 26); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 14; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 26); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 0)) << 26; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 26); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 26); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 14)) << 12; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 26); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 2)) << 24; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 26); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 26); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 16)) << 10; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 26); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 22; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 26); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 26); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 18)) << 8; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 26); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 6)) << 20; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 26); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 26); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 20)) << 6; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 26); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 8)) << 18; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 26); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 26); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 22)) << 4; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 26); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 10)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 26); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 26); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 2; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 26); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 12)) << 14; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 26); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<27>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 27); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 27); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 17)) << 10; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 27); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 7)) << 20; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 27); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 27); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 24)) << 3; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 27); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 13; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 27); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 23; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 27); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 27); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 21)) << 6; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 27); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 11)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 27); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 1)) << 26; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 27); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 27); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 18)) << 9; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 27); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 19; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 27); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 27); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 25)) << 2; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 27); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 15)) << 12; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 27); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 5)) << 22; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 27); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 27); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 22)) << 5; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 27); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 12)) << 15; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 27); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 25; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 27); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 27); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 19)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 27); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 9)) << 18; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 27); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 27); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 26)) << 1; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 27); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 16)) << 11; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 27); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 21; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 27); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 27); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 23)) << 4; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 27); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 13)) << 14; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 27); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 3)) << 24; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 27); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 27); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 7; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 27); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 10)) << 17; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 27); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<28>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 28); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 28); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 20)) << 8; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 28); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 12)) << 16; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 28); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 4)) << 24; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 28); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 28); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 24)) << 4; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 28); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 12; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 28); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 8)) << 20; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 28); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 28; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 28); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 28); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 20)) << 8; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 28); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 12)) << 16; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 28); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 4)) << 24; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 28); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 28); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 24)) << 4; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 28); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 16)) << 12; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 28); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 20; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 28); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 0)) << 28; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 28); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 28); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 20)) << 8; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 28); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 12)) << 16; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 28); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 24; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 28); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 28); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 24)) << 4; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 28); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 16)) << 12; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 28); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 8)) << 20; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 28); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 28; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 28); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 28); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 20)) << 8; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 28); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 12)) << 16; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 28); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 4)) << 24; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 28); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 28); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 24)) << 4; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 28); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 12; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 28); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 8)) << 20; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 28); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<29>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 29); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 29); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 23)) << 6; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 29); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 17)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 29); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 11)) << 18; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 29); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 5)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 29); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 29); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 28)) << 1; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 29); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 22)) << 7; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 29); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 16)) << 13; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 29); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 10)) << 19; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 29); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 4)) << 25; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 29); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 29); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 27)) << 2; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 29); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 21)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 29); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 15)) << 14; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 29); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 9)) << 20; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 29); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 3)) << 26; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 29); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 29); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 26)) << 3; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 29); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 20)) << 9; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 29); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 14)) << 15; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 29); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 8)) << 21; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 29); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 2)) << 27; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 29); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 29); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 25)) << 4; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 29); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 19)) << 10; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 29); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 13)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 29); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 7)) << 22; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 29); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 1)) << 28; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 29); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 29); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 24)) << 5; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 29); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 18)) << 11; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 29); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 12)) << 17; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 29); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 6)) << 23; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 29); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<30>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 30); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 30); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 26)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 30); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 22)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 30); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 18)) << 12; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 30); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 30); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 20; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 30); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 24; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 30); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 28; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 30); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 30); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 28)) << 2; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 30); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 24)) << 6; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 30); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 10; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 30); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 14; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 30); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 18; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 30); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 22; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 30); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 26; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 30); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 30; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 30); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 30); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 26)) << 4; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 30); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 22)) << 8; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 30); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 18)) << 12; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 30); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 14)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 30); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 10)) << 20; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 30); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 24; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 30); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 2)) << 28; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 30); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 30); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 28)) << 2; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 30); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 6; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 30); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 10; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 30); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 14; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 30); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 12)) << 18; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 30); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 22; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 30); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 4)) << 26; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 30); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<31>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 31); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 31); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 29)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 31); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 27)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 31); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 25)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 31); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 23)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 31); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 21)) << 10; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 31); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 19)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 31); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 17)) << 14; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 31); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 15)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 31); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 13)) << 18; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 31); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 11)) << 20; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 31); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 9)) << 22; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 31); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 7)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 31); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 5)) << 26; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 31); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 3)) << 28; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 31); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 1)) << 30; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 31); + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 31); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 30)) << 1; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 31); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 28)) << 3; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 31); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 26)) << 5; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 31); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 24)) << 7; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 31); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 22)) << 9; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 31); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 20)) << 11; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 31); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 18)) << 13; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 31); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 16)) << 15; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 31); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 14)) << 17; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 31); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 12)) << 19; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 31); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 10)) << 21; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 31); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 8)) << 23; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 31); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 6)) << 25; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 31); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 4)) << 27; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 31); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 2)) << 29; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 31); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<32>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 0)) << 32; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 32); + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<33>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 33); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 2)) << 31; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 33); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 29; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 33); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 6)) << 27; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 33); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 25; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 33); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 23; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 33); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 12)) << 21; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 33); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 14)) << 19; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 33); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 16)) << 17; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 33); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 18)) << 15; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 33); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 13; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 33); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 22)) << 11; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 33); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 24)) << 9; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 33); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 26)) << 7; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 33); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 28)) << 5; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 33); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 30)) << 3; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 33); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 32)) << 1; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 1)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 33); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 3)) << 30; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 33); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 5)) << 28; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 33); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 7)) << 26; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 33); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 9)) << 24; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 33); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 11)) << 22; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 33); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 13)) << 20; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 33); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 15)) << 18; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 33); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 17)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 33); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 19)) << 14; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 33); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 21)) << 12; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 33); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 23)) << 10; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 33); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 25)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 33); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 27)) << 6; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 33); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 29)) << 4; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 33); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 31)) << 2; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<34>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 34); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 4)) << 30; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 34); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 8)) << 26; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 34); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 12)) << 22; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 34); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 16)) << 18; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 34); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 20)) << 14; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 34); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 24)) << 10; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 34); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 28)) << 6; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 34); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 32)) << 2; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 2)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 34); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 6)) << 28; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 34); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 24; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 34); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 14)) << 20; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 34); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 18)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 34); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 22)) << 12; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 34); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 26)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 34); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 30)) << 4; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 0)) << 34; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 34); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 4)) << 30; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 34); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 8)) << 26; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 34); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 12)) << 22; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 34); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 16)) << 18; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 34); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 20)) << 14; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 34); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 24)) << 10; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 34); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 28)) << 6; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 34); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 32)) << 2; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 2)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 34); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 6)) << 28; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 34); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 10)) << 24; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 34); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 14)) << 20; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 34); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 18)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 34); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 22)) << 12; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 34); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 26)) << 8; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 34); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 30)) << 4; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<35>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 35); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 6)) << 29; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 35); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 12)) << 23; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 35); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 18)) << 17; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 35); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 24)) << 11; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 35); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 30)) << 5; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 1)) << 34; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 35); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 7)) << 28; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 35); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 13)) << 22; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 35); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 19)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 35); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 25)) << 10; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 35); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 31)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 2)) << 33; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 35); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 27; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 35); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 14)) << 21; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 35); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 20)) << 15; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 35); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 26)) << 9; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 35); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 32)) << 3; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 3)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 35); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 9)) << 26; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 35); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 15)) << 20; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 35); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 21)) << 14; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 35); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 27)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 35); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 33)) << 2; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 4)) << 31; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 35); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 10)) << 25; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 35); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 19; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 35); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 22)) << 13; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 35); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 28)) << 7; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 35); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 34)) << 1; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 5)) << 30; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 35); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 11)) << 24; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 35); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 17)) << 18; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 35); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 23)) << 12; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 35); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 29)) << 6; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<36>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 36); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 8)) << 28; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 36); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 16)) << 20; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 36); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 24)) << 12; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 36); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 32)) << 4; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 32; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 36); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 12)) << 24; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 36); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 20)) << 16; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 36); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 28)) << 8; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 36; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 36); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 28; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 36); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 20; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 36); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 24)) << 12; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 36); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 32)) << 4; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 32; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 36); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 12)) << 24; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 36); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 20)) << 16; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 36); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 28)) << 8; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 0)) << 36; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 36); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 8)) << 28; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 36); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 16)) << 20; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 36); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 24)) << 12; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 36); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 32)) << 4; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 4)) << 32; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 36); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 12)) << 24; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 36); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 16; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 36); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 28)) << 8; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 0)) << 36; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 36); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 28; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 36); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 16)) << 20; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 36); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 24)) << 12; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 36); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 32)) << 4; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 4)) << 32; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 36); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 12)) << 24; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 36); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 20)) << 16; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 36); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 28)) << 8; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<37>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 37); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 10)) << 27; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 37); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 20)) << 17; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 37); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 30)) << 7; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 3)) << 34; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 37); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 13)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 37); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 23)) << 14; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 37); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 33)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 6)) << 31; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 37); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 16)) << 21; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 37); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 26)) << 11; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 37); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 36)) << 1; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 9)) << 28; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 37); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 19)) << 18; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 37); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 29)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 2)) << 35; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 37); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 12)) << 25; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 37); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 22)) << 15; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 37); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 32)) << 5; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 5)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 37); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 15)) << 22; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 37); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 25)) << 12; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 37); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 35)) << 2; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 8)) << 29; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 37); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 18)) << 19; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 37); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 28)) << 9; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 1)) << 36; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 37); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 11)) << 26; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 37); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 21)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 37); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 31)) << 6; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 4)) << 33; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 37); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 14)) << 23; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 37); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 24)) << 13; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 37); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 34)) << 3; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 7)) << 30; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 37); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 17)) << 20; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 37); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 27)) << 10; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<38>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 38); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 12)) << 26; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 38); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 24)) << 14; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 38); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 36)) << 2; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 10)) << 28; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 38); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 22)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 38); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 34)) << 4; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 8)) << 30; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 38); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 20)) << 18; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 38); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 32)) << 6; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 6)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 38); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 18)) << 20; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 38); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 30)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 4)) << 34; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 38); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 16)) << 22; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 38); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 28)) << 10; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 36; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 38); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 14)) << 24; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 38); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 26)) << 12; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 0)) << 38; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 38); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 12)) << 26; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 38); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 24)) << 14; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 38); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 36)) << 2; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 10)) << 28; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 38); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 22)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 38); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 34)) << 4; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 8)) << 30; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 38); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 20)) << 18; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 38); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 32)) << 6; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 6)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 38); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 18)) << 20; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 38); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 30)) << 8; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 4)) << 34; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 38); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 16)) << 22; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 38); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 28)) << 10; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 2)) << 36; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 38); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 14)) << 24; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 38); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 26)) << 12; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<39>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 39); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 14)) << 25; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 39); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 28)) << 11; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 3)) << 36; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 39); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 17)) << 22; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 39); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 31)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 6)) << 33; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 39); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 20)) << 19; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 39); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 34)) << 5; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 9)) << 30; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 39); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 23)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 39); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 37)) << 2; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 27; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 39); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 26)) << 13; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 1)) << 38; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 39); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 15)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 39); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 29)) << 10; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 35; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 39); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 18)) << 21; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 39); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 32)) << 7; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 7)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 39); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 21)) << 18; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 39); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 35)) << 4; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 10)) << 29; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 39); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 15; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 39); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 38)) << 1; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 13)) << 26; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 39); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 27)) << 12; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 2)) << 37; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 39); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 16)) << 23; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 39); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 30)) << 9; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 5)) << 34; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 39); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 19)) << 20; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 39); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 33)) << 6; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 8)) << 31; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 39); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 22)) << 17; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 39); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 36)) << 3; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 11)) << 28; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 39); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 25)) << 14; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<40>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 0)) << 40; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 40); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 16)) << 24; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 40); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 32)) << 8; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 8)) << 32; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 40); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 16; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<41>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 41); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 18)) << 23; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 41); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 36)) << 5; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 13)) << 28; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 41); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 31)) << 10; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 8)) << 33; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 41); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 26)) << 15; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 3)) << 38; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 41); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 21)) << 20; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 41); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 39)) << 2; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 16)) << 25; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 41); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 34)) << 7; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 11)) << 30; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 41); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 29)) << 12; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 6)) << 35; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 41); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 24)) << 17; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 1)) << 40; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 41); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 19)) << 22; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 41); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 37)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 14)) << 27; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 41); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 32)) << 9; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 9)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 41); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 27)) << 14; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 4)) << 37; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 41); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 22)) << 19; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 41); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 40)) << 1; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 17)) << 24; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 41); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 35)) << 6; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 12)) << 29; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 41); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 30)) << 11; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 7)) << 34; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 41); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 25)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 2)) << 39; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 41); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 20)) << 21; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 41); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 38)) << 3; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 15)) << 26; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 41); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 33)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 10)) << 31; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 41); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 28)) << 13; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 5)) << 36; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 41); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 23)) << 18; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<42>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 42); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 20)) << 22; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 42); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 40)) << 2; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 18)) << 24; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 42); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 38)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 26; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 42); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 36)) << 6; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 14)) << 28; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 42); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 34)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 12)) << 30; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 42); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 32)) << 10; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 42); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 30)) << 12; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 34; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 42); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 28)) << 14; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 6)) << 36; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 42); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 26)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 38; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 42); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 24)) << 18; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 2)) << 40; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 42); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 22)) << 20; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 42; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 42); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 20)) << 22; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 42); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 40)) << 2; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 18)) << 24; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 42); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 38)) << 4; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 26; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 42); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 36)) << 6; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 14)) << 28; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 42); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 34)) << 8; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 12)) << 30; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 42); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 32)) << 10; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 10)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 42); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 30)) << 12; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 8)) << 34; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 42); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 28)) << 14; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 6)) << 36; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 42); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 26)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 4)) << 38; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 42); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 18; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 2)) << 40; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 42); + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 22)) << 20; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<43>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 43); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 22)) << 21; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 1)) << 42; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 43); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 23)) << 20; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 2)) << 41; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 43); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 24)) << 19; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 3)) << 40; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 43); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 25)) << 18; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 4)) << 39; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 43); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 26)) << 17; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 5)) << 38; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 43); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 27)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 6)) << 37; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 43); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 28)) << 15; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 7)) << 36; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 43); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 29)) << 14; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 8)) << 35; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 43); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 30)) << 13; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 9)) << 34; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 43); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 31)) << 12; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 10)) << 33; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 43); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 32)) << 11; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 11)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 43); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 33)) << 10; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 12)) << 31; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 43); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 34)) << 9; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 13)) << 30; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 43); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 35)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 14)) << 29; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 43); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 36)) << 7; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 15)) << 28; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 43); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 37)) << 6; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 16)) << 27; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 43); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 38)) << 5; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 17)) << 26; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 43); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 39)) << 4; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 18)) << 25; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 43); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 40)) << 3; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 19)) << 24; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 43); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 41)) << 2; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 20)) << 23; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 43); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 42)) << 1; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 21)) << 22; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<44>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 44); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 24)) << 20; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 4)) << 40; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 44); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 28)) << 16; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 8)) << 36; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 44); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 32)) << 12; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 12)) << 32; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 44); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 36)) << 8; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 16)) << 28; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 44); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 40)) << 4; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 24; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 0)) << 44; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 44); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 24)) << 20; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 4)) << 40; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 44); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 28)) << 16; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 8)) << 36; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 44); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 32)) << 12; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 32; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 44); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 36)) << 8; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 16)) << 28; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 44); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 40)) << 4; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 20)) << 24; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 0)) << 44; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 44); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 24)) << 20; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 4)) << 40; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 44); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 28)) << 16; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 8)) << 36; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 44); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 32)) << 12; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 12)) << 32; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 44); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 36)) << 8; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 16)) << 28; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 44); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 40)) << 4; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 20)) << 24; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 0)) << 44; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 44); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 24)) << 20; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 4)) << 40; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 44); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 28)) << 16; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 8)) << 36; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 44); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 32)) << 12; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 12)) << 32; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 44); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 36)) << 8; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 16)) << 28; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 44); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 40)) << 4; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 20)) << 24; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<45>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 45); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 26)) << 19; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 7)) << 38; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 45); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 33)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 31; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 45); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 40)) << 5; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 21)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 43; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 45); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 28)) << 17; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 9)) << 36; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 45); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 35)) << 10; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 29; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 45); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 42)) << 3; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 23)) << 22; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 41; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 45); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 30)) << 15; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 11)) << 34; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 45); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 37)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 18)) << 27; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 45); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 44)) << 1; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 25)) << 20; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 39; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 45); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 32)) << 13; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 13)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 45); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 39)) << 6; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 25; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 1)) << 44; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 45); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 27)) << 18; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 37; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 45); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 34)) << 11; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 15)) << 30; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 45); + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 41)) << 4; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 22)) << 23; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 3)) << 42; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 45); + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 29)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 10)) << 35; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 45); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 36)) << 9; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 17)) << 28; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 45); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 43)) << 2; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 21; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 5)) << 40; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 45); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 31)) << 14; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 12)) << 33; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 45); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 38)) << 7; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 19)) << 26; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<46>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 46); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 28)) << 18; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 10)) << 36; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 46); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 38)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 20)) << 26; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 2)) << 44; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 46); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 30)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 12)) << 34; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 46); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 40)) << 6; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 22)) << 24; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 4)) << 42; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 46); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 32)) << 14; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 14)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 46); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 42)) << 4; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 24)) << 22; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 6)) << 40; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 46); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 34)) << 12; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 16)) << 30; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 46); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 44)) << 2; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 26)) << 20; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 8)) << 38; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 46); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 36)) << 10; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 18)) << 28; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 0)) << 46; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 46); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 28)) << 18; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 10)) << 36; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 46); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 38)) << 8; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 20)) << 26; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 2)) << 44; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 46); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 30)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 12)) << 34; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 46); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 40)) << 6; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 22)) << 24; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 4)) << 42; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 46); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 32)) << 14; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 14)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 46); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 42)) << 4; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 24)) << 22; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 6)) << 40; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 46); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 34)) << 12; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 16)) << 30; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 46); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 44)) << 2; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 26)) << 20; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 8)) << 38; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 46); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 36)) << 10; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 18)) << 28; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<47>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 47); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 30)) << 17; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 13)) << 34; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 47); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 43)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 26)) << 21; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 9)) << 38; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 47); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 39)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 22)) << 25; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 5)) << 42; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 47); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 35)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 18)) << 29; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 1)) << 46; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 47); + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 31)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 14)) << 33; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 47); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 44)) << 3; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 27)) << 20; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 10)) << 37; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 47); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 40)) << 7; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 23)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 6)) << 41; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 47); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 36)) << 11; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 19)) << 28; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 2)) << 45; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 47); + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 32)) << 15; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 15)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 47); + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 45)) << 2; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 28)) << 19; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 11)) << 36; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 47); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 41)) << 6; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 24)) << 23; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 7)) << 40; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 47); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 37)) << 10; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 20)) << 27; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 3)) << 44; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 47); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 33)) << 14; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 16)) << 31; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 47); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 46)) << 1; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 29)) << 18; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 12)) << 35; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 47); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 42)) << 5; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 25)) << 22; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 8)) << 39; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 47); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 38)) << 9; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 21)) << 26; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 4)) << 43; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 47); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 34)) << 13; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 17)) << 30; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<48>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 0)) << 48; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 48); + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 32)) << 16; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 16)) << 32; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<49>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 49); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 34)) << 15; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 19)) << 30; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 4)) << 45; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 49); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 38)) << 11; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 23)) << 26; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 8)) << 41; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 49); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 42)) << 7; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 27)) << 22; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 12)) << 37; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 49); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 46)) << 3; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 31)) << 18; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 16)) << 33; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 1)) << 48; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 49); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 35)) << 14; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 20)) << 29; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 5)) << 44; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 49); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 39)) << 10; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 24)) << 25; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 9)) << 40; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 49); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 43)) << 6; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 28)) << 21; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 13)) << 36; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 49); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 47)) << 2; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 32)) << 17; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 17)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 2)) << 47; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 49); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 36)) << 13; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 21)) << 28; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 6)) << 43; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 49); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 40)) << 9; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 25)) << 24; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 10)) << 39; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 49); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 44)) << 5; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 29)) << 20; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 14)) << 35; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 49); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 48)) << 1; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 33)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 18)) << 31; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 3)) << 46; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 49); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 37)) << 12; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 22)) << 27; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 7)) << 42; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 49); + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 41)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 26)) << 23; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 11)) << 38; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 49); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 45)) << 4; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 30)) << 19; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 15)) << 34; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<50>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 50); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 36)) << 14; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 22)) << 28; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 8)) << 42; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 50); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 44)) << 6; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 30)) << 20; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 16)) << 34; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 2)) << 48; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 50); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 38)) << 12; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 24)) << 26; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 10)) << 40; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 50); + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 46)) << 4; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 32)) << 18; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 18)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 46; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 50); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 40)) << 10; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 26)) << 24; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 38; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 50); + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 48)) << 2; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 34)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 20)) << 30; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 44; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 50); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 42)) << 8; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 28)) << 22; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 14)) << 36; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 0)) << 50; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 50); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 36)) << 14; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 22)) << 28; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 42; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 50); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 44)) << 6; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 30)) << 20; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 16)) << 34; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 2)) << 48; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 50); + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 38)) << 12; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 24)) << 26; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 10)) << 40; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 50); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 46)) << 4; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 32)) << 18; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 18)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 4)) << 46; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 50); + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 40)) << 10; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 26)) << 24; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 12)) << 38; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 50); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 48)) << 2; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 34)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 20)) << 30; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 6)) << 44; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 50); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 42)) << 8; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 28)) << 22; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 14)) << 36; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<51>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 51); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 38)) << 13; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 25)) << 26; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 12)) << 39; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 51); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 50)) << 1; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 37)) << 14; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 24)) << 27; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 11)) << 40; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 51); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 49)) << 2; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 36)) << 15; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 23)) << 28; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 10)) << 41; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 51); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 48)) << 3; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 35)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 22)) << 29; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 9)) << 42; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 51); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 47)) << 4; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 34)) << 17; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 21)) << 30; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 8)) << 43; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 51); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 46)) << 5; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 33)) << 18; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 20)) << 31; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 7)) << 44; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 51); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 45)) << 6; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 32)) << 19; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 19)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 6)) << 45; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 51); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 44)) << 7; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 31)) << 20; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 18)) << 33; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 5)) << 46; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 51); + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 43)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 30)) << 21; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 17)) << 34; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 4)) << 47; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 51); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 42)) << 9; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 29)) << 22; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 16)) << 35; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 3)) << 48; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 51); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 41)) << 10; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 28)) << 23; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 15)) << 36; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 2)) << 49; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 51); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 40)) << 11; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 27)) << 24; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 14)) << 37; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 1)) << 50; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 51); + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 39)) << 12; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 26)) << 25; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 13)) << 38; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<52>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 52); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 40)) << 12; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 28)) << 24; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 16)) << 36; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 4)) << 48; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 52); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 44)) << 8; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 32)) << 20; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 20)) << 32; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 8)) << 44; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 52); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 48)) << 4; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 36)) << 16; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 24)) << 28; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 40; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 0)) << 52; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 52); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 40)) << 12; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 28)) << 24; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 16)) << 36; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 4)) << 48; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 52); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 44)) << 8; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 32)) << 20; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 20)) << 32; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 8)) << 44; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 52); + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 48)) << 4; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 36)) << 16; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 28; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 12)) << 40; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 0)) << 52; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 52); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 40)) << 12; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 28)) << 24; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 16)) << 36; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 4)) << 48; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 52); + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 44)) << 8; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 32)) << 20; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 20)) << 32; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 8)) << 44; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 52); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 48)) << 4; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 36)) << 16; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 24)) << 28; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 12)) << 40; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 0)) << 52; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 52); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 40)) << 12; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 28)) << 24; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 16)) << 36; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 4)) << 48; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 52); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 44)) << 8; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 32)) << 20; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 20)) << 32; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 8)) << 44; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 52); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 48)) << 4; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 36)) << 16; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 24)) << 28; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 12)) << 40; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<53>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 53); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 42)) << 11; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 31)) << 22; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 20)) << 33; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 9)) << 44; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 53); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 51)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 40)) << 13; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 29)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 18)) << 35; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 7)) << 46; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 53); + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 49)) << 4; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 38)) << 15; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 27)) << 26; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 16)) << 37; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 5)) << 48; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 53); + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 47)) << 6; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 36)) << 17; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 25)) << 28; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 14)) << 39; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 3)) << 50; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 53); + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 45)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 34)) << 19; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 23)) << 30; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 12)) << 41; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 1)) << 52; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 53); + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 43)) << 10; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 32)) << 21; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 21)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 10)) << 43; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 53); + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 52)) << 1; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 41)) << 12; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 30)) << 23; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 19)) << 34; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 8)) << 45; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 53); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 50)) << 3; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 39)) << 14; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 28)) << 25; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 17)) << 36; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 6)) << 47; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 53); + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 48)) << 5; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 37)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 26)) << 27; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 15)) << 38; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 4)) << 49; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 53); + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 46)) << 7; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 35)) << 18; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 24)) << 29; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 13)) << 40; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 2)) << 51; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 53); + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 44)) << 9; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 33)) << 20; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 22)) << 31; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 11)) << 42; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<54>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 54); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 44)) << 10; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 34)) << 20; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 24)) << 30; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 14)) << 40; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 4)) << 50; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 54); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 48)) << 6; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 38)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 28)) << 26; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 18)) << 36; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 8)) << 46; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 54); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 52)) << 2; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 42)) << 12; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 32)) << 22; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 22)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 12)) << 42; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 52; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 54); + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 46)) << 8; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 36)) << 18; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 26)) << 28; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 16)) << 38; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 6)) << 48; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 54); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 50)) << 4; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 40)) << 14; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 30)) << 24; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 34; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 10)) << 44; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 0)) << 54; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 54); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 44)) << 10; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 34)) << 20; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 24)) << 30; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 14)) << 40; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 4)) << 50; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 54); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 48)) << 6; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 38)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 28)) << 26; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 18)) << 36; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 8)) << 46; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 54); + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 52)) << 2; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 42)) << 12; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 32)) << 22; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 22)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 12)) << 42; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 2)) << 52; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 54); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 46)) << 8; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 36)) << 18; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 26)) << 28; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 16)) << 38; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 6)) << 48; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 54); + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 50)) << 4; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 40)) << 14; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 30)) << 24; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 20)) << 34; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 10)) << 44; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<55>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 55); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 46)) << 9; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 37)) << 18; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 28)) << 27; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 19)) << 36; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 10)) << 45; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 1)) << 54; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 55); + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 47)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 38)) << 17; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 29)) << 26; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 35; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 11)) << 44; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 2)) << 53; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 55); + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 48)) << 7; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 39)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 30)) << 25; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 21)) << 34; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 12)) << 43; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 3)) << 52; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 55); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 49)) << 6; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 40)) << 15; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 31)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 22)) << 33; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 13)) << 42; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 4)) << 51; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 55); + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 50)) << 5; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 41)) << 14; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 32)) << 23; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 23)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 14)) << 41; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 5)) << 50; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 55); + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 51)) << 4; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 42)) << 13; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 33)) << 22; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 24)) << 31; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 15)) << 40; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 6)) << 49; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 55); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 52)) << 3; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 43)) << 12; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 34)) << 21; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 25)) << 30; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 16)) << 39; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 7)) << 48; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 55); + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 53)) << 2; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 44)) << 11; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 35)) << 20; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 26)) << 29; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 17)) << 38; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 8)) << 47; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 55); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 54)) << 1; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 45)) << 10; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 36)) << 19; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 27)) << 28; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 18)) << 37; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 9)) << 46; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 55); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<56>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 0)) << 56; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 56); + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 48)) << 8; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 40)) << 16; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 32)) << 24; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 24)) << 32; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 16)) << 40; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 8)) << 48; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<57>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 57); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 50)) << 7; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 43)) << 14; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 36)) << 21; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 29)) << 28; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 22)) << 35; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 15)) << 42; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 8)) << 49; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 1)) << 56; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 57); + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 51)) << 6; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 44)) << 13; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 37)) << 20; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 30)) << 27; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 23)) << 34; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 16)) << 41; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 9)) << 48; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 55); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 2)) << 55; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 57); + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 52)) << 5; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 45)) << 12; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 38)) << 19; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 31)) << 26; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 24)) << 33; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 17)) << 40; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 10)) << 47; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 3)) << 54; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 57); + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 53)) << 4; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 46)) << 11; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 39)) << 18; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 32)) << 25; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 25)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 18)) << 39; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 11)) << 46; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 4)) << 53; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 57); + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 54)) << 3; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 47)) << 10; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 40)) << 17; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 33)) << 24; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 26)) << 31; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 19)) << 38; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 12)) << 45; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 5)) << 52; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 57); + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 55)) << 2; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 48)) << 9; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 41)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 34)) << 23; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 27)) << 30; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 20)) << 37; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 13)) << 44; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 6)) << 51; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 57); + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 56)) << 1; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 49)) << 8; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 42)) << 15; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 35)) << 22; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 28)) << 29; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 21)) << 36; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 14)) << 43; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 7)) << 50; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 57); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<58>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 58); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 52)) << 6; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 46)) << 12; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 40)) << 18; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 34)) << 24; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 28)) << 30; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 22)) << 36; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 16)) << 42; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 10)) << 48; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 4)) << 54; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 58); + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 56)) << 2; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 50)) << 8; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 44)) << 14; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 38)) << 20; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 32)) << 26; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 26)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 20)) << 38; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 14)) << 44; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 8)) << 50; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 2)) << 56; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 58); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 54)) << 4; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 48)) << 10; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 42)) << 16; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 36)) << 22; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 30)) << 28; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 24)) << 34; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 18)) << 40; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 12)) << 46; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 6)) << 52; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 0)) << 58; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 58); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 52)) << 6; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 46)) << 12; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 40)) << 18; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 34)) << 24; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 28)) << 30; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 22)) << 36; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 16)) << 42; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 10)) << 48; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 4)) << 54; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 58); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 56)) << 2; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 50)) << 8; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 44)) << 14; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 38)) << 20; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 32)) << 26; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 26)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 20)) << 38; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 14)) << 44; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 8)) << 50; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 2)) << 56; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 58); + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 54)) << 4; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 48)) << 10; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 42)) << 16; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 36)) << 22; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 30)) << 28; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 24)) << 34; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 18)) << 40; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 12)) << 46; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 6)) << 52; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<59>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 59); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 54)) << 5; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 49)) << 10; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 44)) << 15; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 39)) << 20; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 34)) << 25; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 29)) << 30; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 24)) << 35; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 19)) << 40; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 14)) << 45; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 9)) << 50; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 55); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 4)) << 55; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 59); + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 58)) << 1; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 53)) << 6; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 48)) << 11; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 43)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 38)) << 21; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 33)) << 26; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 28)) << 31; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 23)) << 36; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 18)) << 41; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 13)) << 46; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 8)) << 51; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 3)) << 56; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 59); + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 57)) << 2; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 52)) << 7; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 47)) << 12; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 42)) << 17; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 37)) << 22; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 32)) << 27; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 27)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 22)) << 37; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 17)) << 42; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 12)) << 47; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 7)) << 52; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 57); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 2)) << 57; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 59); + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 56)) << 3; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 51)) << 8; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 46)) << 13; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 41)) << 18; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 36)) << 23; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 31)) << 28; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 26)) << 33; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 21)) << 38; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 16)) << 43; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 11)) << 48; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 6)) << 53; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 1)) << 58; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 59); + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 55)) << 4; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 50)) << 9; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 45)) << 14; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 40)) << 19; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 35)) << 24; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 30)) << 29; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 25)) << 34; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 20)) << 39; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 15)) << 44; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 10)) << 49; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 58]; + tmp |= (src & MASK(uint64_t, 5)) << 54; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 59); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<60>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 60); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 56)) << 4; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 52)) << 8; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 48)) << 12; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 44)) << 16; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 40)) << 20; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 36)) << 24; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 32)) << 28; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 28)) << 32; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 24)) << 36; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 20)) << 40; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 16)) << 44; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 12)) << 48; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 8)) << 52; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 4)) << 56; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 0)) << 60; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 60); + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 56)) << 4; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 52)) << 8; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 48)) << 12; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 44)) << 16; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 40)) << 20; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 36)) << 24; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 32)) << 28; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 28)) << 32; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 24)) << 36; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 20)) << 40; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 16)) << 44; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 12)) << 48; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 8)) << 52; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 4)) << 56; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 0)) << 60; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 60); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 56)) << 4; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 52)) << 8; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 48)) << 12; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 44)) << 16; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 40)) << 20; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 36)) << 24; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 32)) << 28; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 28)) << 32; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 36; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 20)) << 40; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 16)) << 44; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 12)) << 48; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 8)) << 52; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 4)) << 56; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 0)) << 60; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 60); + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 56)) << 4; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 52)) << 8; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 48)) << 12; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 44)) << 16; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 40)) << 20; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 36)) << 24; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 32)) << 28; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 28)) << 32; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 24)) << 36; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 20)) << 40; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 16)) << 44; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 12)) << 48; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 58]; + tmp |= (src & MASK(uint64_t, 8)) << 52; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 59]; + tmp |= (src & MASK(uint64_t, 4)) << 56; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<61>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 61); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 58)) << 3; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 55)) << 6; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 52)) << 9; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 49)) << 12; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 46)) << 15; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 43)) << 18; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 40)) << 21; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 37)) << 24; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 34)) << 27; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 31)) << 30; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 28)) << 33; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 25)) << 36; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 22)) << 39; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 19)) << 42; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 16)) << 45; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 13)) << 48; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 10)) << 51; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 7)) << 54; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 57); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 4)) << 57; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 1)) << 60; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 61); + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 59)) << 2; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 56)) << 5; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 53)) << 8; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 50)) << 11; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 47)) << 14; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 44)) << 17; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 41)) << 20; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 38)) << 23; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 35)) << 26; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 32)) << 29; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 29)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 26)) << 35; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 23)) << 38; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 20)) << 41; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 17)) << 44; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 14)) << 47; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 11)) << 50; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 8)) << 53; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 5)) << 56; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 59); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 2)) << 59; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 61); + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 60)) << 1; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 57)) << 4; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 54)) << 7; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 51)) << 10; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 48)) << 13; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 45)) << 16; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 42)) << 19; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 39)) << 22; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 36)) << 25; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 33)) << 28; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 30)) << 31; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 27)) << 34; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 24)) << 37; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 21)) << 40; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 18)) << 43; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 15)) << 46; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 12)) << 49; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 58]; + tmp |= (src & MASK(uint64_t, 9)) << 52; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 55); + src = in[lane + LANE_COUNT * 59]; + tmp |= (src & MASK(uint64_t, 6)) << 55; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 60]; + tmp |= (src & MASK(uint64_t, 3)) << 58; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 61); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<62>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 62); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 60)) << 2; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 58)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 56)) << 6; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 54)) << 8; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 52)) << 10; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 50)) << 12; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 48)) << 14; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 46)) << 16; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 44)) << 18; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 42)) << 20; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 40)) << 22; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 38)) << 24; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 36)) << 26; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 34)) << 28; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 32)) << 30; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 30)) << 32; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 28)) << 34; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 26)) << 36; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 24)) << 38; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 22)) << 40; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 20)) << 42; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 18)) << 44; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 16)) << 46; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 14)) << 48; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 12)) << 50; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 10)) << 52; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 8)) << 54; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 6)) << 56; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 4)) << 58; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 2)) << 60; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 62); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 0)) << 62; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint64_t, 62); + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 60)) << 2; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 58)) << 4; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 56)) << 6; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 54)) << 8; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 52)) << 10; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 50)) << 12; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 48)) << 14; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 46)) << 16; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 44)) << 18; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 42)) << 20; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 40)) << 22; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 38)) << 24; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 36)) << 26; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 34)) << 28; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 32)) << 30; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 30)) << 32; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 28)) << 34; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 26)) << 36; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 24)) << 38; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 22)) << 40; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 20)) << 42; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 18)) << 44; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 16)) << 46; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 14)) << 48; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 12)) << 50; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 10)) << 52; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 58]; + tmp |= (src & MASK(uint64_t, 8)) << 54; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 59]; + tmp |= (src & MASK(uint64_t, 6)) << 56; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 60]; + tmp |= (src & MASK(uint64_t, 4)) << 58; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 61]; + tmp |= (src & MASK(uint64_t, 2)) << 60; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 62); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<63>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + uint64_t src; + uint64_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint64_t, 63); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 63) & MASK(uint64_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint64_t, 62)) << 1; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 62) & MASK(uint64_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint64_t, 61)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 61) & MASK(uint64_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint64_t, 60)) << 3; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 60) & MASK(uint64_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint64_t, 59)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 59) & MASK(uint64_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint64_t, 58)) << 5; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 58) & MASK(uint64_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint64_t, 57)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 57) & MASK(uint64_t, 7); + src = in[lane + LANE_COUNT * 7]; + tmp |= (src & MASK(uint64_t, 56)) << 7; + out[INDEX(7, lane)] = tmp + reference; + tmp = (src >> 56) & MASK(uint64_t, 8); + src = in[lane + LANE_COUNT * 8]; + tmp |= (src & MASK(uint64_t, 55)) << 8; + out[INDEX(8, lane)] = tmp + reference; + tmp = (src >> 55) & MASK(uint64_t, 9); + src = in[lane + LANE_COUNT * 9]; + tmp |= (src & MASK(uint64_t, 54)) << 9; + out[INDEX(9, lane)] = tmp + reference; + tmp = (src >> 54) & MASK(uint64_t, 10); + src = in[lane + LANE_COUNT * 10]; + tmp |= (src & MASK(uint64_t, 53)) << 10; + out[INDEX(10, lane)] = tmp + reference; + tmp = (src >> 53) & MASK(uint64_t, 11); + src = in[lane + LANE_COUNT * 11]; + tmp |= (src & MASK(uint64_t, 52)) << 11; + out[INDEX(11, lane)] = tmp + reference; + tmp = (src >> 52) & MASK(uint64_t, 12); + src = in[lane + LANE_COUNT * 12]; + tmp |= (src & MASK(uint64_t, 51)) << 12; + out[INDEX(12, lane)] = tmp + reference; + tmp = (src >> 51) & MASK(uint64_t, 13); + src = in[lane + LANE_COUNT * 13]; + tmp |= (src & MASK(uint64_t, 50)) << 13; + out[INDEX(13, lane)] = tmp + reference; + tmp = (src >> 50) & MASK(uint64_t, 14); + src = in[lane + LANE_COUNT * 14]; + tmp |= (src & MASK(uint64_t, 49)) << 14; + out[INDEX(14, lane)] = tmp + reference; + tmp = (src >> 49) & MASK(uint64_t, 15); + src = in[lane + LANE_COUNT * 15]; + tmp |= (src & MASK(uint64_t, 48)) << 15; + out[INDEX(15, lane)] = tmp + reference; + tmp = (src >> 48) & MASK(uint64_t, 16); + src = in[lane + LANE_COUNT * 16]; + tmp |= (src & MASK(uint64_t, 47)) << 16; + out[INDEX(16, lane)] = tmp + reference; + tmp = (src >> 47) & MASK(uint64_t, 17); + src = in[lane + LANE_COUNT * 17]; + tmp |= (src & MASK(uint64_t, 46)) << 17; + out[INDEX(17, lane)] = tmp + reference; + tmp = (src >> 46) & MASK(uint64_t, 18); + src = in[lane + LANE_COUNT * 18]; + tmp |= (src & MASK(uint64_t, 45)) << 18; + out[INDEX(18, lane)] = tmp + reference; + tmp = (src >> 45) & MASK(uint64_t, 19); + src = in[lane + LANE_COUNT * 19]; + tmp |= (src & MASK(uint64_t, 44)) << 19; + out[INDEX(19, lane)] = tmp + reference; + tmp = (src >> 44) & MASK(uint64_t, 20); + src = in[lane + LANE_COUNT * 20]; + tmp |= (src & MASK(uint64_t, 43)) << 20; + out[INDEX(20, lane)] = tmp + reference; + tmp = (src >> 43) & MASK(uint64_t, 21); + src = in[lane + LANE_COUNT * 21]; + tmp |= (src & MASK(uint64_t, 42)) << 21; + out[INDEX(21, lane)] = tmp + reference; + tmp = (src >> 42) & MASK(uint64_t, 22); + src = in[lane + LANE_COUNT * 22]; + tmp |= (src & MASK(uint64_t, 41)) << 22; + out[INDEX(22, lane)] = tmp + reference; + tmp = (src >> 41) & MASK(uint64_t, 23); + src = in[lane + LANE_COUNT * 23]; + tmp |= (src & MASK(uint64_t, 40)) << 23; + out[INDEX(23, lane)] = tmp + reference; + tmp = (src >> 40) & MASK(uint64_t, 24); + src = in[lane + LANE_COUNT * 24]; + tmp |= (src & MASK(uint64_t, 39)) << 24; + out[INDEX(24, lane)] = tmp + reference; + tmp = (src >> 39) & MASK(uint64_t, 25); + src = in[lane + LANE_COUNT * 25]; + tmp |= (src & MASK(uint64_t, 38)) << 25; + out[INDEX(25, lane)] = tmp + reference; + tmp = (src >> 38) & MASK(uint64_t, 26); + src = in[lane + LANE_COUNT * 26]; + tmp |= (src & MASK(uint64_t, 37)) << 26; + out[INDEX(26, lane)] = tmp + reference; + tmp = (src >> 37) & MASK(uint64_t, 27); + src = in[lane + LANE_COUNT * 27]; + tmp |= (src & MASK(uint64_t, 36)) << 27; + out[INDEX(27, lane)] = tmp + reference; + tmp = (src >> 36) & MASK(uint64_t, 28); + src = in[lane + LANE_COUNT * 28]; + tmp |= (src & MASK(uint64_t, 35)) << 28; + out[INDEX(28, lane)] = tmp + reference; + tmp = (src >> 35) & MASK(uint64_t, 29); + src = in[lane + LANE_COUNT * 29]; + tmp |= (src & MASK(uint64_t, 34)) << 29; + out[INDEX(29, lane)] = tmp + reference; + tmp = (src >> 34) & MASK(uint64_t, 30); + src = in[lane + LANE_COUNT * 30]; + tmp |= (src & MASK(uint64_t, 33)) << 30; + out[INDEX(30, lane)] = tmp + reference; + tmp = (src >> 33) & MASK(uint64_t, 31); + src = in[lane + LANE_COUNT * 31]; + tmp |= (src & MASK(uint64_t, 32)) << 31; + out[INDEX(31, lane)] = tmp + reference; + tmp = (src >> 32) & MASK(uint64_t, 32); + src = in[lane + LANE_COUNT * 32]; + tmp |= (src & MASK(uint64_t, 31)) << 32; + out[INDEX(32, lane)] = tmp + reference; + tmp = (src >> 31) & MASK(uint64_t, 33); + src = in[lane + LANE_COUNT * 33]; + tmp |= (src & MASK(uint64_t, 30)) << 33; + out[INDEX(33, lane)] = tmp + reference; + tmp = (src >> 30) & MASK(uint64_t, 34); + src = in[lane + LANE_COUNT * 34]; + tmp |= (src & MASK(uint64_t, 29)) << 34; + out[INDEX(34, lane)] = tmp + reference; + tmp = (src >> 29) & MASK(uint64_t, 35); + src = in[lane + LANE_COUNT * 35]; + tmp |= (src & MASK(uint64_t, 28)) << 35; + out[INDEX(35, lane)] = tmp + reference; + tmp = (src >> 28) & MASK(uint64_t, 36); + src = in[lane + LANE_COUNT * 36]; + tmp |= (src & MASK(uint64_t, 27)) << 36; + out[INDEX(36, lane)] = tmp + reference; + tmp = (src >> 27) & MASK(uint64_t, 37); + src = in[lane + LANE_COUNT * 37]; + tmp |= (src & MASK(uint64_t, 26)) << 37; + out[INDEX(37, lane)] = tmp + reference; + tmp = (src >> 26) & MASK(uint64_t, 38); + src = in[lane + LANE_COUNT * 38]; + tmp |= (src & MASK(uint64_t, 25)) << 38; + out[INDEX(38, lane)] = tmp + reference; + tmp = (src >> 25) & MASK(uint64_t, 39); + src = in[lane + LANE_COUNT * 39]; + tmp |= (src & MASK(uint64_t, 24)) << 39; + out[INDEX(39, lane)] = tmp + reference; + tmp = (src >> 24) & MASK(uint64_t, 40); + src = in[lane + LANE_COUNT * 40]; + tmp |= (src & MASK(uint64_t, 23)) << 40; + out[INDEX(40, lane)] = tmp + reference; + tmp = (src >> 23) & MASK(uint64_t, 41); + src = in[lane + LANE_COUNT * 41]; + tmp |= (src & MASK(uint64_t, 22)) << 41; + out[INDEX(41, lane)] = tmp + reference; + tmp = (src >> 22) & MASK(uint64_t, 42); + src = in[lane + LANE_COUNT * 42]; + tmp |= (src & MASK(uint64_t, 21)) << 42; + out[INDEX(42, lane)] = tmp + reference; + tmp = (src >> 21) & MASK(uint64_t, 43); + src = in[lane + LANE_COUNT * 43]; + tmp |= (src & MASK(uint64_t, 20)) << 43; + out[INDEX(43, lane)] = tmp + reference; + tmp = (src >> 20) & MASK(uint64_t, 44); + src = in[lane + LANE_COUNT * 44]; + tmp |= (src & MASK(uint64_t, 19)) << 44; + out[INDEX(44, lane)] = tmp + reference; + tmp = (src >> 19) & MASK(uint64_t, 45); + src = in[lane + LANE_COUNT * 45]; + tmp |= (src & MASK(uint64_t, 18)) << 45; + out[INDEX(45, lane)] = tmp + reference; + tmp = (src >> 18) & MASK(uint64_t, 46); + src = in[lane + LANE_COUNT * 46]; + tmp |= (src & MASK(uint64_t, 17)) << 46; + out[INDEX(46, lane)] = tmp + reference; + tmp = (src >> 17) & MASK(uint64_t, 47); + src = in[lane + LANE_COUNT * 47]; + tmp |= (src & MASK(uint64_t, 16)) << 47; + out[INDEX(47, lane)] = tmp + reference; + tmp = (src >> 16) & MASK(uint64_t, 48); + src = in[lane + LANE_COUNT * 48]; + tmp |= (src & MASK(uint64_t, 15)) << 48; + out[INDEX(48, lane)] = tmp + reference; + tmp = (src >> 15) & MASK(uint64_t, 49); + src = in[lane + LANE_COUNT * 49]; + tmp |= (src & MASK(uint64_t, 14)) << 49; + out[INDEX(49, lane)] = tmp + reference; + tmp = (src >> 14) & MASK(uint64_t, 50); + src = in[lane + LANE_COUNT * 50]; + tmp |= (src & MASK(uint64_t, 13)) << 50; + out[INDEX(50, lane)] = tmp + reference; + tmp = (src >> 13) & MASK(uint64_t, 51); + src = in[lane + LANE_COUNT * 51]; + tmp |= (src & MASK(uint64_t, 12)) << 51; + out[INDEX(51, lane)] = tmp + reference; + tmp = (src >> 12) & MASK(uint64_t, 52); + src = in[lane + LANE_COUNT * 52]; + tmp |= (src & MASK(uint64_t, 11)) << 52; + out[INDEX(52, lane)] = tmp + reference; + tmp = (src >> 11) & MASK(uint64_t, 53); + src = in[lane + LANE_COUNT * 53]; + tmp |= (src & MASK(uint64_t, 10)) << 53; + out[INDEX(53, lane)] = tmp + reference; + tmp = (src >> 10) & MASK(uint64_t, 54); + src = in[lane + LANE_COUNT * 54]; + tmp |= (src & MASK(uint64_t, 9)) << 54; + out[INDEX(54, lane)] = tmp + reference; + tmp = (src >> 9) & MASK(uint64_t, 55); + src = in[lane + LANE_COUNT * 55]; + tmp |= (src & MASK(uint64_t, 8)) << 55; + out[INDEX(55, lane)] = tmp + reference; + tmp = (src >> 8) & MASK(uint64_t, 56); + src = in[lane + LANE_COUNT * 56]; + tmp |= (src & MASK(uint64_t, 7)) << 56; + out[INDEX(56, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint64_t, 57); + src = in[lane + LANE_COUNT * 57]; + tmp |= (src & MASK(uint64_t, 6)) << 57; + out[INDEX(57, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint64_t, 58); + src = in[lane + LANE_COUNT * 58]; + tmp |= (src & MASK(uint64_t, 5)) << 58; + out[INDEX(58, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint64_t, 59); + src = in[lane + LANE_COUNT * 59]; + tmp |= (src & MASK(uint64_t, 4)) << 59; + out[INDEX(59, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint64_t, 60); + src = in[lane + LANE_COUNT * 60]; + tmp |= (src & MASK(uint64_t, 3)) << 60; + out[INDEX(60, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint64_t, 61); + src = in[lane + LANE_COUNT * 61]; + tmp |= (src & MASK(uint64_t, 2)) << 61; + out[INDEX(61, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint64_t, 62); + src = in[lane + LANE_COUNT * 62]; + tmp |= (src & MASK(uint64_t, 1)) << 62; + out[INDEX(62, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint64_t, 63); + out[INDEX(63, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_64_lane<64>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 16; + #pragma unroll + for (int row = 0; row < 64; row++) { + out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; + } +} + +/// Runtime dispatch to the optimized lane decoder for the given bit width. +__device__ __noinline__ void bit_unpack_64_lane( + const uint64_t *__restrict in, + uint64_t *__restrict out, + uint64_t reference, + unsigned int lane, + uint32_t bit_width +) { + switch (bit_width) { + case 0: _bit_unpack_64_lane<0>(in, out, reference, lane); break; + case 1: _bit_unpack_64_lane<1>(in, out, reference, lane); break; + case 2: _bit_unpack_64_lane<2>(in, out, reference, lane); break; + case 3: _bit_unpack_64_lane<3>(in, out, reference, lane); break; + case 4: _bit_unpack_64_lane<4>(in, out, reference, lane); break; + case 5: _bit_unpack_64_lane<5>(in, out, reference, lane); break; + case 6: _bit_unpack_64_lane<6>(in, out, reference, lane); break; + case 7: _bit_unpack_64_lane<7>(in, out, reference, lane); break; + case 8: _bit_unpack_64_lane<8>(in, out, reference, lane); break; + case 9: _bit_unpack_64_lane<9>(in, out, reference, lane); break; + case 10: _bit_unpack_64_lane<10>(in, out, reference, lane); break; + case 11: _bit_unpack_64_lane<11>(in, out, reference, lane); break; + case 12: _bit_unpack_64_lane<12>(in, out, reference, lane); break; + case 13: _bit_unpack_64_lane<13>(in, out, reference, lane); break; + case 14: _bit_unpack_64_lane<14>(in, out, reference, lane); break; + case 15: _bit_unpack_64_lane<15>(in, out, reference, lane); break; + case 16: _bit_unpack_64_lane<16>(in, out, reference, lane); break; + case 17: _bit_unpack_64_lane<17>(in, out, reference, lane); break; + case 18: _bit_unpack_64_lane<18>(in, out, reference, lane); break; + case 19: _bit_unpack_64_lane<19>(in, out, reference, lane); break; + case 20: _bit_unpack_64_lane<20>(in, out, reference, lane); break; + case 21: _bit_unpack_64_lane<21>(in, out, reference, lane); break; + case 22: _bit_unpack_64_lane<22>(in, out, reference, lane); break; + case 23: _bit_unpack_64_lane<23>(in, out, reference, lane); break; + case 24: _bit_unpack_64_lane<24>(in, out, reference, lane); break; + case 25: _bit_unpack_64_lane<25>(in, out, reference, lane); break; + case 26: _bit_unpack_64_lane<26>(in, out, reference, lane); break; + case 27: _bit_unpack_64_lane<27>(in, out, reference, lane); break; + case 28: _bit_unpack_64_lane<28>(in, out, reference, lane); break; + case 29: _bit_unpack_64_lane<29>(in, out, reference, lane); break; + case 30: _bit_unpack_64_lane<30>(in, out, reference, lane); break; + case 31: _bit_unpack_64_lane<31>(in, out, reference, lane); break; + case 32: _bit_unpack_64_lane<32>(in, out, reference, lane); break; + case 33: _bit_unpack_64_lane<33>(in, out, reference, lane); break; + case 34: _bit_unpack_64_lane<34>(in, out, reference, lane); break; + case 35: _bit_unpack_64_lane<35>(in, out, reference, lane); break; + case 36: _bit_unpack_64_lane<36>(in, out, reference, lane); break; + case 37: _bit_unpack_64_lane<37>(in, out, reference, lane); break; + case 38: _bit_unpack_64_lane<38>(in, out, reference, lane); break; + case 39: _bit_unpack_64_lane<39>(in, out, reference, lane); break; + case 40: _bit_unpack_64_lane<40>(in, out, reference, lane); break; + case 41: _bit_unpack_64_lane<41>(in, out, reference, lane); break; + case 42: _bit_unpack_64_lane<42>(in, out, reference, lane); break; + case 43: _bit_unpack_64_lane<43>(in, out, reference, lane); break; + case 44: _bit_unpack_64_lane<44>(in, out, reference, lane); break; + case 45: _bit_unpack_64_lane<45>(in, out, reference, lane); break; + case 46: _bit_unpack_64_lane<46>(in, out, reference, lane); break; + case 47: _bit_unpack_64_lane<47>(in, out, reference, lane); break; + case 48: _bit_unpack_64_lane<48>(in, out, reference, lane); break; + case 49: _bit_unpack_64_lane<49>(in, out, reference, lane); break; + case 50: _bit_unpack_64_lane<50>(in, out, reference, lane); break; + case 51: _bit_unpack_64_lane<51>(in, out, reference, lane); break; + case 52: _bit_unpack_64_lane<52>(in, out, reference, lane); break; + case 53: _bit_unpack_64_lane<53>(in, out, reference, lane); break; + case 54: _bit_unpack_64_lane<54>(in, out, reference, lane); break; + case 55: _bit_unpack_64_lane<55>(in, out, reference, lane); break; + case 56: _bit_unpack_64_lane<56>(in, out, reference, lane); break; + case 57: _bit_unpack_64_lane<57>(in, out, reference, lane); break; + case 58: _bit_unpack_64_lane<58>(in, out, reference, lane); break; + case 59: _bit_unpack_64_lane<59>(in, out, reference, lane); break; + case 60: _bit_unpack_64_lane<60>(in, out, reference, lane); break; + case 61: _bit_unpack_64_lane<61>(in, out, reference, lane); break; + case 62: _bit_unpack_64_lane<62>(in, out, reference, lane); break; + case 63: _bit_unpack_64_lane<63>(in, out, reference, lane); break; + case 64: _bit_unpack_64_lane<64>(in, out, reference, lane); break; + } +} + diff --git a/vortex-cuda/kernels/src/bit_unpack_8.cu b/vortex-cuda/kernels/src/bit_unpack_8.cu index 13ef84a2b2f..c933eda4df2 100644 --- a/vortex-cuda/kernels/src/bit_unpack_8.cu +++ b/vortex-cuda/kernels/src/bit_unpack_8.cu @@ -1,261 +1,7 @@ // AUTO-GENERATED. Do not edit by hand! -#include -#include -#include -#include "fastlanes_common.cuh" +#include "bit_unpack_8_lanes.cuh" #include "patches.cuh" -template -__device__ void _bit_unpack_8_lane(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane); - -template <> -__device__ void _bit_unpack_8_lane<0>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - #pragma unroll - for (int row = 0; row < 8; row++) { - out[INDEX(row, lane)] = reference; - } -} - -template <> -__device__ void _bit_unpack_8_lane<1>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 1); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 1); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 1); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 1); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 1); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 1); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 1); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<2>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 2); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 2); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 2); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 0)) << 2; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 2); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 2); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 2); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<3>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 3); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 3); - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 1)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 3); - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 3); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 2)) << 1; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 3); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<4>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 0)) << 4; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 4); - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<5>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 5); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 2)) << 3; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 5); - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 4)) << 1; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 1)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 5); - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 3)) << 2; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 5); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<6>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 6); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 4)) << 2; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 2)) << 4; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 0)) << 6; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 0) & MASK(uint8_t, 6); - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 4)) << 2; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint8_t, 2)) << 4; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<7>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - uint8_t src; - uint8_t tmp; - src = in[lane]; - tmp = (src >> 0) & MASK(uint8_t, 7); - out[INDEX(0, lane)] = tmp + reference; - tmp = (src >> 7) & MASK(uint8_t, 1); - src = in[lane + LANE_COUNT * 1]; - tmp |= (src & MASK(uint8_t, 6)) << 1; - out[INDEX(1, lane)] = tmp + reference; - tmp = (src >> 6) & MASK(uint8_t, 2); - src = in[lane + LANE_COUNT * 2]; - tmp |= (src & MASK(uint8_t, 5)) << 2; - out[INDEX(2, lane)] = tmp + reference; - tmp = (src >> 5) & MASK(uint8_t, 3); - src = in[lane + LANE_COUNT * 3]; - tmp |= (src & MASK(uint8_t, 4)) << 3; - out[INDEX(3, lane)] = tmp + reference; - tmp = (src >> 4) & MASK(uint8_t, 4); - src = in[lane + LANE_COUNT * 4]; - tmp |= (src & MASK(uint8_t, 3)) << 4; - out[INDEX(4, lane)] = tmp + reference; - tmp = (src >> 3) & MASK(uint8_t, 5); - src = in[lane + LANE_COUNT * 5]; - tmp |= (src & MASK(uint8_t, 2)) << 5; - out[INDEX(5, lane)] = tmp + reference; - tmp = (src >> 2) & MASK(uint8_t, 6); - src = in[lane + LANE_COUNT * 6]; - tmp |= (src & MASK(uint8_t, 1)) << 6; - out[INDEX(6, lane)] = tmp + reference; - tmp = (src >> 1) & MASK(uint8_t, 7); - out[INDEX(7, lane)] = tmp + reference; -} - -template <> -__device__ void _bit_unpack_8_lane<8>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; - #pragma unroll - for (int row = 0; row < 8; row++) { - out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; - } -} - -/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ inline void bit_unpack_8_lane( - const uint8_t *__restrict in, - uint8_t *__restrict out, - uint8_t reference, - unsigned int lane, - uint32_t bit_width -) { - switch (bit_width) { - case 0: _bit_unpack_8_lane<0>(in, out, reference, lane); break; - case 1: _bit_unpack_8_lane<1>(in, out, reference, lane); break; - case 2: _bit_unpack_8_lane<2>(in, out, reference, lane); break; - case 3: _bit_unpack_8_lane<3>(in, out, reference, lane); break; - case 4: _bit_unpack_8_lane<4>(in, out, reference, lane); break; - case 5: _bit_unpack_8_lane<5>(in, out, reference, lane); break; - case 6: _bit_unpack_8_lane<6>(in, out, reference, lane); break; - case 7: _bit_unpack_8_lane<7>(in, out, reference, lane); break; - case 8: _bit_unpack_8_lane<8>(in, out, reference, lane); break; - } -} - template __device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint8_t shared_out[1024]; diff --git a/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh new file mode 100644 index 00000000000..64fb8ce39a8 --- /dev/null +++ b/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh @@ -0,0 +1,259 @@ +// AUTO-GENERATED. Do not edit by hand! +#pragma once + +#include +#include +#include +#include "fastlanes_common.cuh" + +template +__device__ void _bit_unpack_8_lane(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane); + +template <> +__device__ void _bit_unpack_8_lane<0>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + #pragma unroll + for (int row = 0; row < 8; row++) { + out[INDEX(row, lane)] = reference; + } +} + +template <> +__device__ void _bit_unpack_8_lane<1>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 1); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint8_t, 1); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 1); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint8_t, 1); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 1); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint8_t, 1); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 1); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint8_t, 1); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<2>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 2); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 2); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 2); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 0)) << 2; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint8_t, 2); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 2); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 2); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<3>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 3); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint8_t, 3); + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 1)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint8_t, 3); + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 3); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint8_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint8_t, 2)) << 1; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 3); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint8_t, 3); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<4>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 4); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 0)) << 4; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint8_t, 4); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint8_t, 0)) << 4; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint8_t, 4); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint8_t, 0)) << 4; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint8_t, 4); + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<5>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 5); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint8_t, 3); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 2)) << 3; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 5); + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint8_t, 1); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint8_t, 4)) << 1; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint8_t, 1)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint8_t, 5); + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint8_t, 3)) << 2; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint8_t, 5); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<6>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 6); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 4)) << 2; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint8_t, 2)) << 4; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 6); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint8_t, 0)) << 6; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 0) & MASK(uint8_t, 6); + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint8_t, 4)) << 2; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint8_t, 2)) << 4; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 6); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<7>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + uint8_t src; + uint8_t tmp; + src = in[lane]; + tmp = (src >> 0) & MASK(uint8_t, 7); + out[INDEX(0, lane)] = tmp + reference; + tmp = (src >> 7) & MASK(uint8_t, 1); + src = in[lane + LANE_COUNT * 1]; + tmp |= (src & MASK(uint8_t, 6)) << 1; + out[INDEX(1, lane)] = tmp + reference; + tmp = (src >> 6) & MASK(uint8_t, 2); + src = in[lane + LANE_COUNT * 2]; + tmp |= (src & MASK(uint8_t, 5)) << 2; + out[INDEX(2, lane)] = tmp + reference; + tmp = (src >> 5) & MASK(uint8_t, 3); + src = in[lane + LANE_COUNT * 3]; + tmp |= (src & MASK(uint8_t, 4)) << 3; + out[INDEX(3, lane)] = tmp + reference; + tmp = (src >> 4) & MASK(uint8_t, 4); + src = in[lane + LANE_COUNT * 4]; + tmp |= (src & MASK(uint8_t, 3)) << 4; + out[INDEX(4, lane)] = tmp + reference; + tmp = (src >> 3) & MASK(uint8_t, 5); + src = in[lane + LANE_COUNT * 5]; + tmp |= (src & MASK(uint8_t, 2)) << 5; + out[INDEX(5, lane)] = tmp + reference; + tmp = (src >> 2) & MASK(uint8_t, 6); + src = in[lane + LANE_COUNT * 6]; + tmp |= (src & MASK(uint8_t, 1)) << 6; + out[INDEX(6, lane)] = tmp + reference; + tmp = (src >> 1) & MASK(uint8_t, 7); + out[INDEX(7, lane)] = tmp + reference; +} + +template <> +__device__ void _bit_unpack_8_lane<8>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { + unsigned int LANE_COUNT = 128; + #pragma unroll + for (int row = 0; row < 8; row++) { + out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; + } +} + +/// Runtime dispatch to the optimized lane decoder for the given bit width. +__device__ __noinline__ void bit_unpack_8_lane( + const uint8_t *__restrict in, + uint8_t *__restrict out, + uint8_t reference, + unsigned int lane, + uint32_t bit_width +) { + switch (bit_width) { + case 0: _bit_unpack_8_lane<0>(in, out, reference, lane); break; + case 1: _bit_unpack_8_lane<1>(in, out, reference, lane); break; + case 2: _bit_unpack_8_lane<2>(in, out, reference, lane); break; + case 3: _bit_unpack_8_lane<3>(in, out, reference, lane); break; + case 4: _bit_unpack_8_lane<4>(in, out, reference, lane); break; + case 5: _bit_unpack_8_lane<5>(in, out, reference, lane); break; + case 6: _bit_unpack_8_lane<6>(in, out, reference, lane); break; + case 7: _bit_unpack_8_lane<7>(in, out, reference, lane); break; + case 8: _bit_unpack_8_lane<8>(in, out, reference, lane); break; + } +} + diff --git a/vortex-cuda/src/bit_unpack_gen.rs b/vortex-cuda/src/bit_unpack_gen.rs index e4efa9bf2eb..e114f319de8 100644 --- a/vortex-cuda/src/bit_unpack_gen.rs +++ b/vortex-cuda/src/bit_unpack_gen.rs @@ -108,7 +108,7 @@ fn generate_lane_dispatch(output: &mut impl Write, bits: usize) -> io::Result<() write!( output, r#"/// Runtime dispatch to the optimized lane decoder for the given bit width. -__device__ inline void bit_unpack_{bits}_lane( +__device__ __noinline__ void bit_unpack_{bits}_lane( const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, @@ -195,23 +195,25 @@ fn generate_global_kernel( ) } -/// Generate CUDA lane decoders, dispatch function, and kernel wrappers for all bit widths. -pub fn generate_cuda_unpack( - output: &mut impl Write, - thread_count: usize, -) -> io::Result<()> { +/// Generate the lane-decoder header: template specializations + runtime dispatch. +/// +/// This produces a `.cuh` file that is included by `bit_unpack.cuh` (and +/// transitively by `dynamic_dispatch.cu`). It contains only `__device__` +/// functions — no `__global__` kernels — so that `dynamic_dispatch.cu` does +/// not pull in the 129 standalone bit-unpack kernel entry points. +pub fn generate_cuda_unpack_lanes(output: &mut impl Write) -> io::Result<()> { let bits = T::T; let lanes = T::LANES; - // File header + forward declaration for the lane-decoder template. write!( output, r#"// AUTO-GENERATED. Do not edit by hand! +#pragma once + #include #include #include #include "fastlanes_common.cuh" -#include "patches.cuh" template __device__ void _bit_unpack_{bits}_lane(const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, unsigned int lane); @@ -229,6 +231,29 @@ __device__ void _bit_unpack_{bits}_lane(const uint{bits}_t *__restrict in, uint{ generate_lane_dispatch(output, bits)?; writeln!(output)?; + Ok(()) +} + +/// Generate the standalone kernel file: `_device` template + `__global__` wrappers. +/// +/// This produces a `.cu` file that is compiled to PTX on its own. It includes +/// the corresponding `_lanes.cuh` header for the lane decoders. +pub fn generate_cuda_unpack_kernels( + output: &mut impl Write, + thread_count: usize, +) -> io::Result<()> { + let bits = T::T; + let lanes = T::LANES; + + write!( + output, + r#"// AUTO-GENERATED. Do not edit by hand! +#include "bit_unpack_{bits}_lanes.cuh" +#include "patches.cuh" + +"# + )?; + // Device kernel template (written once, instantiated per bit width). generate_device_kernel_template(output, bits, lanes, thread_count)?; writeln!(output)?; From 85f52d8f2aa4e438191210ba35174299b7d83833 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:25:58 -0400 Subject: [PATCH 048/250] add BtrBlocksCompressorBuilder::empty() (#7443) ## Summary This will be helpful for testing and benchmarking when we specifically do not want to run any compression. ## API Changes Adds the new `empty` method to the builder. ## Testing N/A Signed-off-by: Connor Tsui Co-authored-by: Claude --- vortex-btrblocks/public-api.lock | 2 ++ vortex-btrblocks/src/builder.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/vortex-btrblocks/public-api.lock b/vortex-btrblocks/public-api.lock index d6c2942e14d..cfb2f49b773 100644 --- a/vortex-btrblocks/public-api.lock +++ b/vortex-btrblocks/public-api.lock @@ -682,6 +682,8 @@ impl vortex_btrblocks::BtrBlocksCompressorBuilder pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::build(self) -> vortex_btrblocks::BtrBlocksCompressor +pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::empty() -> Self + pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::exclude_schemes(self, ids: impl core::iter::traits::collect::IntoIterator) -> Self pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::only_cuda_compatible(self) -> Self diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index da405b153e8..ab77f625764 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -97,6 +97,15 @@ impl Default for BtrBlocksCompressorBuilder { } impl BtrBlocksCompressorBuilder { + /// Creates a builder with no schemes registered. + /// + /// Useful when the caller wants explicit, scheme-by-scheme control over the compressor. + pub fn empty() -> Self { + Self { + schemes: Vec::new(), + } + } + /// Adds an external compression scheme not in [`ALL_SCHEMES`]. /// /// This allows encoding crates outside of `vortex-btrblocks` to register their own schemes @@ -188,3 +197,20 @@ impl BtrBlocksCompressorBuilder { BtrBlocksCompressor(CascadingCompressor::new(self.schemes)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_starts_with_no_schemes() { + let builder = BtrBlocksCompressorBuilder::empty(); + assert!(builder.schemes.is_empty()); + } + + #[test] + fn default_includes_all_schemes() { + let builder = BtrBlocksCompressorBuilder::default(); + assert_eq!(builder.schemes.len(), ALL_SCHEMES.len()); + } +} From 757bbfced8809ae98c5c9496e947be88a38e1e77 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 15 Apr 2026 15:36:53 +0100 Subject: [PATCH 049/250] Update locked rustls-webpki to address sec advisory (#7444) # Summary Addresses a couple of recent security advisories by updating the lockfile Signed-off-by: Adam Gutglick --- Cargo.lock | 4 ++-- deny.toml | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c925ab4d18c..9a31e702e77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8233,9 +8233,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", diff --git a/deny.toml b/deny.toml index 31f67e95f77..64814519f91 100644 --- a/deny.toml +++ b/deny.toml @@ -15,9 +15,6 @@ feature-depth = 1 ignore = [ # Paste is no longer maintained because its essentially "finished". "RUSTSEC-2024-0436", - # rustls-pemfile is a hard requirement from object_store crate. - # need to wait for them to release, see https://github.com/apache/arrow-rs-object-store/pull/565 - "RUSTSEC-2025-0134", ] [licenses] From afcdb9ab874dbd7bdf3cde17b8a0b0b46a94e960 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 15 Apr 2026 16:07:28 +0100 Subject: [PATCH 050/250] chore[gpu]: trim Criterion CUDA benchmarks (#7445) CUDA Criterion benchmarks are a bottleneck right now in the workflow when iterating on GPU kernels. We therefore trim away the least relevant benchmarks and reduce the number of iterations being measured. Note that the benchmark results are unchanged regardless of the trimming down the warmup and iteration count. Signed-off-by: Alexander Droste --- vortex-cuda/benches/bitpacked_cuda.rs | 19 +++++++++--------- vortex-cuda/benches/date_time_parts_cuda.rs | 17 +++++++++------- vortex-cuda/benches/dict_cuda.rs | 11 ++++++++-- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 21 ++++++++++---------- vortex-cuda/benches/filter_cuda.rs | 14 +++++++++---- vortex-cuda/benches/for_cuda.rs | 12 ++++++++--- vortex-cuda/benches/runend_cuda.rs | 19 ++++++++++-------- vortex-cuda/benches/throughput_cuda.rs | 12 ++++++++--- vortex-cuda/benches/transpose_patches.rs | 8 +++++++- vortex-cuda/benches/zstd_cuda.rs | 11 ++++++++-- 10 files changed, 94 insertions(+), 50 deletions(-) diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index 8337f5062b0..11c95295714 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -39,8 +39,7 @@ use crate::common::TimedLaunchStrategy; const N_ROWS: usize = 100_000_000; /// Patch frequencies to benchmark (as fractions) -const PATCH_FREQUENCIES: &[(f64, &str)] = - &[(0.001, "0.1%"), (0.01, "1%"), (0.05, "5%"), (0.10, "10%")]; +const PATCH_FREQUENCIES: &[(f64, &str)] = &[(0.01, "1%"), (0.10, "10%")]; /// Create a bit-packed array with the given bit width fn make_bitpacked_array(bit_width: u8, len: usize) -> BitPackedArray @@ -108,7 +107,6 @@ where T::Physical: DeviceRepr, { let mut group = c.benchmark_group(format!("bitunpack_cuda_{}", type_name)); - group.sample_size(10); let array = make_bitpacked_array::(bit_width, N_ROWS); let nbytes = N_ROWS * size_of::(); @@ -153,7 +151,6 @@ where T::Physical: DeviceRepr, { let mut group = c.benchmark_group(format!("bitunpack_cuda_patched_{}", type_name)); - group.sample_size(10); let nbytes = N_ROWS * size_of::(); group.throughput(Throughput::Bytes(nbytes as u64)); @@ -193,11 +190,15 @@ fn benchmark_bitunpack_with_patches(c: &mut Criterion) { benchmark_bitunpack_with_patches_typed::(c, "u64"); } -criterion::criterion_group!( - benches, - benchmark_bitunpack, - benchmark_bitunpack_with_patches -); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_bitunpack, benchmark_bitunpack_with_patches +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index 638daa77538..8a1e9a91641 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -51,13 +51,8 @@ fn make_datetimeparts_array(len: usize, time_unit: TimeUnit) -> DateTimePartsArr fn benchmark_datetimeparts(c: &mut Criterion) { let mut group = c.benchmark_group("datetimeparts_cuda"); - group.sample_size(10); - for (len, len_str) in [ - (1_000_000usize, "1M"), - (10_000_000usize, "10M"), - (100_000_000usize, "100M"), - ] { + for (len, len_str) in [(10_000_000usize, "10M"), (100_000_000usize, "100M")] { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); let (time_unit, unit_str) = (TimeUnit::Milliseconds, "ms"); @@ -90,7 +85,15 @@ fn benchmark_datetimeparts(c: &mut Criterion) { group.finish(); } -criterion::criterion_group!(benches, benchmark_datetimeparts); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_datetimeparts +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/dict_cuda.rs b/vortex-cuda/benches/dict_cuda.rs index 33eff05584d..fd6a1206773 100644 --- a/vortex-cuda/benches/dict_cuda.rs +++ b/vortex-cuda/benches/dict_cuda.rs @@ -73,7 +73,6 @@ where >::Error: std::fmt::Debug, { let mut group = c.benchmark_group("dict_cuda"); - group.sample_size(10); for (len, len_str) in BENCH_ARGS { // Throughput is based on output size (values read from dictionary) @@ -156,7 +155,15 @@ fn benchmark_dict(c: &mut Criterion) { ); } -criterion::criterion_group!(benches, benchmark_dict); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_dict +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 54ae6090f08..2ae0434b243 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -49,11 +49,7 @@ use vortex_cuda::dynamic_dispatch::MaterializedPlan; use vortex_cuda_macros::cuda_available; use vortex_cuda_macros::cuda_not_available; -const BENCH_ARGS: &[(usize, &str)] = &[ - (1_000_000, "1M"), - (10_000_000, "10M"), - (100_000_000, "100M"), -]; +const BENCH_ARGS: &[(usize, &str)] = &[(10_000_000, "10M"), (100_000_000, "100M")]; /// Launch the dynamic_dispatch kernel and return GPU-timed duration. /// @@ -177,7 +173,6 @@ impl BenchRunner { // --------------------------------------------------------------------------- fn bench_for_bitpacked(c: &mut Criterion) { let mut group = c.benchmark_group("for_bitpacked_6bw"); - group.sample_size(10); let bit_width: u8 = 6; let reference = 100_000u32; @@ -224,7 +219,6 @@ fn bench_for_bitpacked(c: &mut Criterion) { // --------------------------------------------------------------------------- fn bench_dict_bp_codes(c: &mut Criterion) { let mut group = c.benchmark_group("dict_256vals_bp8bw_codes"); - group.sample_size(10); let dict_size: usize = 256; let dict_bit_width: u8 = 8; @@ -269,7 +263,6 @@ fn bench_dict_bp_codes(c: &mut Criterion) { // --------------------------------------------------------------------------- fn bench_runend(c: &mut Criterion) { let mut group = c.benchmark_group("runend_100runs"); - group.sample_size(10); let num_runs: usize = 100; @@ -313,7 +306,6 @@ fn bench_runend(c: &mut Criterion) { // --------------------------------------------------------------------------- fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let mut group = c.benchmark_group("dict_64vals_bp6bw_codes_for_bp6bw_values"); - group.sample_size(10); let dict_size: usize = 64; let dict_bit_width: u8 = 6; @@ -367,7 +359,6 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { // --------------------------------------------------------------------------- fn bench_alp_for_bitpacked(c: &mut Criterion) { let mut group = c.benchmark_group("alp_for_bp_6bw_f32"); - group.sample_size(10); let exponents = Exponents { e: 2, f: 0 }; let bit_width: u8 = 6; @@ -428,7 +419,15 @@ fn benchmark_dynamic_dispatch(c: &mut Criterion) { bench_alp_for_bitpacked(c); } -criterion::criterion_group!(benches, benchmark_dynamic_dispatch); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_dynamic_dispatch +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/filter_cuda.rs b/vortex-cuda/benches/filter_cuda.rs index a08fe6797ba..b7647bfd814 100644 --- a/vortex-cuda/benches/filter_cuda.rs +++ b/vortex-cuda/benches/filter_cuda.rs @@ -32,7 +32,7 @@ use vortex_cuda::CudaSession; use vortex_cuda_macros::cuda_available; use vortex_cuda_macros::cuda_not_available; -const BENCH_SIZES: &[(usize, &str)] = &[(1_000_000, "1M"), (10_000_000, "10M")]; +const BENCH_SIZES: &[(usize, &str)] = &[(10_000_000, "10M")]; const SELECTIVITIES: &[(f64, &str)] = &[(0.1, "10%"), (0.5, "50%"), (0.9, "90%")]; /// Creates input data of the given length. @@ -142,7 +142,6 @@ where + 'static, { let mut group = c.benchmark_group(format!("filter_cuda_{type_name}")); - group.sample_size(10); for (len, len_label) in BENCH_SIZES { for (selectivity, sel_label) in SELECTIVITIES { @@ -229,10 +228,17 @@ where fn benchmark_filter(c: &mut Criterion) { benchmark_filter_type::(c, "i32"); benchmark_filter_type::(c, "i64"); - benchmark_filter_type::(c, "f64"); } -criterion::criterion_group!(benches, benchmark_filter); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_filter +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index df59481e6c0..0c28045173c 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -71,7 +71,6 @@ where Scalar: From, { let mut group = c.benchmark_group("for_cuda"); - group.sample_size(10); for &(len, len_str) in BENCH_ARGS { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); @@ -110,7 +109,6 @@ where Scalar: From, { let mut group = c.benchmark_group("ffor_cuda"); - group.sample_size(10); for &(len, len_str) in BENCH_ARGS { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); @@ -159,7 +157,15 @@ fn benchmark_ffor(c: &mut Criterion) { benchmark_ffor_typed::(c, "u64"); } -criterion::criterion_group!(benches, benchmark_for, benchmark_ffor); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_for, benchmark_ffor +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/runend_cuda.rs b/vortex-cuda/benches/runend_cuda.rs index 973a69ee914..bb9f6e4f108 100644 --- a/vortex-cuda/benches/runend_cuda.rs +++ b/vortex-cuda/benches/runend_cuda.rs @@ -64,16 +64,11 @@ where T: NativePType + DeviceRepr + From, { let mut group = c.benchmark_group("runend_cuda"); - group.sample_size(10); - for (len, len_str) in [ - (1_000_000usize, "1M"), - (10_000_000usize, "10M"), - (100_000_000usize, "100M"), - ] { + for (len, len_str) in [(10_000_000usize, "10M"), (100_000_000usize, "100M")] { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); - for run_len in [10, 100, 1000, 10000, 100000] { + for run_len in [10, 1000, 100000] { let runend_array = make_runend_array_typed::(len, run_len); group.bench_with_input( @@ -114,7 +109,15 @@ fn benchmark_runend(c: &mut Criterion) { benchmark_runend_typed::(c, "i32"); } -criterion::criterion_group!(benches, benchmark_runend); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_runend +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/throughput_cuda.rs b/vortex-cuda/benches/throughput_cuda.rs index dc9eb82a4f4..6a332fa6a4e 100644 --- a/vortex-cuda/benches/throughput_cuda.rs +++ b/vortex-cuda/benches/throughput_cuda.rs @@ -94,8 +94,6 @@ fn transfer_mix_timed( fn benchmark_transfer_throughput(c: &mut Criterion) { let mut group = c.benchmark_group("transfer_throughput_cuda"); - group.sample_size(10); - for &(input_bytes, output_bytes, label) in MIXES { let total_mem_bytes = input_bytes * 2 + output_bytes; group.throughput(Throughput::Bytes(total_mem_bytes as u64)); @@ -122,7 +120,15 @@ fn benchmark_transfer_throughput(c: &mut Criterion) { group.finish(); } -criterion::criterion_group!(benches, benchmark_transfer_throughput); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_transfer_throughput +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/transpose_patches.rs b/vortex-cuda/benches/transpose_patches.rs index fee0edba43a..d713a304a07 100644 --- a/vortex-cuda/benches/transpose_patches.rs +++ b/vortex-cuda/benches/transpose_patches.rs @@ -66,7 +66,13 @@ fn benchmark_transpose(c: &mut Criterion) { ); } -criterion::criterion_group!(benches, benchmark_transpose); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .warm_up_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_transpose +} #[cuda_available] criterion::criterion_main!(benches); diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index 8862f18915b..089d32a1fc9 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -118,7 +118,6 @@ async fn execute_zstd_kernel( /// Benchmark ZSTD CUDA decompression kernel performance fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut group = c.benchmark_group("ZSTD_cuda"); - group.sample_size(10); for (num_strings, label) in BENCH_ARGS { let (zstd_array, uncompressed_size) = @@ -162,7 +161,15 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { group.finish(); } -criterion::criterion_group!(benches, benchmark_zstd_cuda_decompress); +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_zstd_cuda_decompress +} #[cuda_available] criterion::criterion_main!(benches); From 89de477a0e3196d792de0b1936faf68e41068a43 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 15 Apr 2026 16:24:34 +0100 Subject: [PATCH 051/250] Add spark value writers for Date/Timestamp/TimestampNTZ and Struct types (#7424) Add value writers for all teh types that we can convert to spark. Previously we have been able to convert the type but failed at writing values Signed-off-by: Robert Kruszewski --- java/testfiles/Cargo.lock | 22 ++--- .../dev/vortex/api/expressions/IsNull.java | 3 +- .../spark/write/SparkToArrowSchema.java | 2 + .../vortex/spark/write/VortexDataWriter.java | 31 ++++++- .../spark/VortexDataSourceWriteTest.java | 85 +++++++++++++++++++ 5 files changed, 127 insertions(+), 16 deletions(-) diff --git a/java/testfiles/Cargo.lock b/java/testfiles/Cargo.lock index 190904c651f..b6618bc3fdc 100644 --- a/java/testfiles/Cargo.lock +++ b/java/testfiles/Cargo.lock @@ -913,13 +913,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.0", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] @@ -1636,9 +1634,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", @@ -2065,7 +2063,6 @@ dependencies = [ "enum-iterator", "flatbuffers", "futures", - "getrandom 0.4.2", "half", "humansize", "inventory", @@ -2085,6 +2082,7 @@ dependencies = [ "termtree", "tracing", "uuid", + "vortex-array-macros", "vortex-buffer", "vortex-error", "vortex-flatbuffers", @@ -2094,11 +2092,19 @@ dependencies = [ "vortex-utils", ] +[[package]] +name = "vortex-array-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vortex-btrblocks" version = "0.1.0" dependencies = [ - "getrandom 0.4.2", "itertools 0.14.0", "num-traits", "pco", @@ -2225,7 +2231,6 @@ dependencies = [ "bytes", "flatbuffers", "futures", - "getrandom 0.4.2", "itertools 0.14.0", "kanal", "moka", @@ -2233,7 +2238,6 @@ dependencies = [ "parking_lot", "pin-project-lite", "tracing", - "uuid", "vortex-alp", "vortex-array", "vortex-btrblocks", @@ -2292,7 +2296,6 @@ dependencies = [ "bytes", "custom-labels", "futures", - "getrandom 0.4.2", "glob", "handle", "kanal", @@ -2378,7 +2381,6 @@ dependencies = [ name = "vortex-metrics" version = "0.1.0" dependencies = [ - "getrandom 0.4.2", "parking_lot", "sketches-ddsketch", ] diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java index 834cc6d5494..b000fe88f2c 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java @@ -35,8 +35,7 @@ public static IsNull parse(byte[] metadata, List children) { "IsNull expression must have exactly one child, found: " + children.size()); } if (metadata.length > 0) { - throw new IllegalArgumentException( - "IsNull expression must not have metadata, found: " + metadata.length); + throw new IllegalArgumentException("IsNull expression must not have metadata, found: " + metadata.length); } return new IsNull(children.get(0)); } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java index ca6a90384b1..87488c61c75 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java @@ -100,6 +100,8 @@ private static ArrowType convertType(DataType sparkType) { return new ArrowType.Date(DateUnit.DAY); } else if (sparkType instanceof TimestampType) { return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); + } else if (sparkType instanceof TimestampNTZType) { + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } else if (sparkType instanceof DecimalType) { DecimalType decimal = (DecimalType) sparkType; return new ArrowType.Decimal(decimal.precision(), decimal.scale(), 128); diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java index ed1c986b9c1..df66c4ca254 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java @@ -10,8 +10,10 @@ import dev.vortex.relocated.org.apache.arrow.memory.BufferAllocator; import dev.vortex.relocated.org.apache.arrow.memory.RootAllocator; import dev.vortex.relocated.org.apache.arrow.vector.*; +import dev.vortex.relocated.org.apache.arrow.vector.FieldVector; import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; import dev.vortex.relocated.org.apache.arrow.vector.complex.ListVector; +import dev.vortex.relocated.org.apache.arrow.vector.complex.StructVector; import dev.vortex.spark.SparkTypes; import java.io.IOException; import java.nio.file.Files; @@ -205,16 +207,23 @@ private void populateVector( if (bytes != null) { ((VarBinaryVector) vector).setSafe(rowIndex, bytes); } - } else if (dataType instanceof DecimalType) { - DecimalType decType = (DecimalType) dataType; + } else if (dataType instanceof DateType) { + ((DateDayVector) vector).setSafe(rowIndex, row.getInt(fieldIndex)); + } else if (dataType instanceof TimestampType) { + ((TimeStampMicroTZVector) vector).setSafe(rowIndex, row.getLong(fieldIndex)); + } else if (dataType instanceof TimestampNTZType) { + ((TimeStampMicroVector) vector).setSafe(rowIndex, row.getLong(fieldIndex)); + } else if (dataType instanceof DecimalType decType) { if (decType.precision() <= 38) { // Use Decimal type from InternalRow java.math.BigDecimal decimal = row.getDecimal(fieldIndex, decType.precision(), decType.scale()) .toJavaBigDecimal(); ((DecimalVector) vector).setSafe(rowIndex, decimal); } - } else if (dataType instanceof ArrayType) { - ArrayType arrayType = (ArrayType) dataType; + } else if (dataType instanceof StructType structType) { + populateStructVector( + (StructVector) vector, structType, row.getStruct(fieldIndex, structType.fields().length), rowIndex); + } else if (dataType instanceof ArrayType arrayType) { ArrayData data = row.getArray(fieldIndex); ListVector listVector = ((ListVector) vector); int writtenElements = listVector.getElementEndIndex(listVector.getLastSet()); @@ -229,6 +238,20 @@ private void populateVector( } } + private void populateStructVector(StructVector vector, StructType dataType, InternalRow row, int rowIndex) { + vector.setIndexDefined(rowIndex); + + StructField[] fields = dataType.fields(); + for (int fieldIndex = 0; fieldIndex < fields.length; fieldIndex++) { + FieldVector childVector = (FieldVector) vector.getVectorById(fieldIndex); + if (row.isNullAt(fieldIndex)) { + childVector.setNull(rowIndex); + continue; + } + populateVector(childVector, fields[fieldIndex].dataType(), row, fieldIndex, rowIndex); + } + } + /** * Commits the write operation and returns a commit message. *

diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java index 6c5686fa93c..80764949bc0 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java @@ -3,6 +3,7 @@ package dev.vortex.spark; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -324,6 +325,73 @@ public void testSpecialCharactersAndNulls() throws IOException { assertEquals("special!@#$%^&*()", specialRows.first().getString(1)); } + @Test + @DisplayName("Write and read date, timestamp, and nested struct columns") + public void testWriteAndReadTemporalAndStructColumns() throws IOException { + Dataset originalDf = spark.range(0, 2) + .selectExpr( + "cast(id as int) as id", + "CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END AS event_date", + "CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) " + + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END AS event_ts", + "named_struct(" + + "'event_date', CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END, " + + "'event_ts', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) " + + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, " + + "'label', CASE WHEN id = 0 THEN 'alpha' ELSE 'beta' END" + + ") AS payload"); + + Path outputPath = tempDir.resolve("temporal_struct_output"); + originalDf + .write() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .mode(SaveMode.Overwrite) + .save(); + + Dataset readDf = spark.read() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .load(); + + List expectedRows = List.of( + "{\"id\":0,\"event_date\":\"2024-01-02\",\"event_ts\":\"2024-01-02 03:04:05.123456\"," + + "\"payload_event_date\":\"2024-01-02\",\"payload_event_ts\":\"2024-01-02 03:04:05.123456\"," + + "\"payload_label\":\"alpha\"}", + "{\"id\":1,\"event_date\":\"2024-02-03\",\"event_ts\":\"2024-02-03 04:05:06.654321\"," + + "\"payload_event_date\":\"2024-02-03\",\"payload_event_ts\":\"2024-02-03 04:05:06.654321\"," + + "\"payload_label\":\"beta\"}"); + + assertEquals(DataTypes.DateType, readDf.schema().fields()[1].dataType()); + assertEquals(DataTypes.TimestampType, readDf.schema().fields()[2].dataType()); + assertTrue(readDf.schema().fields()[3].dataType() instanceof StructType); + assertEquals(expectedRows, projectTemporalAndStructRows(readDf)); + } + + @Test + @DisplayName("Write TimestampNTZ columns and nested structs") + public void testWriteTimestampNtzColumns() throws IOException { + Dataset timestampNtzDf = spark.range(0, 2) + .selectExpr( + "cast(id as int) as id", + "CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) " + + "ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz", + "named_struct(" + + "'event_ntz', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) " + + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END" + + ") AS payload"); + + Path outputPath = tempDir.resolve("timestamp_ntz_output"); + assertDoesNotThrow(() -> timestampNtzDf + .write() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .mode(SaveMode.Overwrite) + .save()); + + assertTrue(!findVortexFiles(outputPath).isEmpty(), "TimestampNTZ write should create Vortex files"); + } + /** * Creates a test DataFrame with monotonically increasing integers * and their string representations. @@ -337,6 +405,23 @@ private Dataset createTestDataFrame(int numRows) { "array('Alpha', 'Bravo', 'Charlie') AS elements"); } + private List projectTemporalAndStructRows(Dataset df) { + return df + .orderBy("id") + .selectExpr("to_json(named_struct(" + + "'id', id, " + + "'event_date', cast(event_date as string), " + + "'event_ts', date_format(event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), " + + "'payload_event_date', cast(payload.event_date as string), " + + "'payload_event_ts', date_format(payload.event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), " + + "'payload_label', payload.label" + + ")) as json") + .collectAsList() + .stream() + .map(row -> row.getString(0)) + .collect(Collectors.toList()); + } + /** * Finds all Vortex files in the given directory. */ From 0e0401c98c8123dbd220f906bca18ab94412dfc3 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Wed, 15 Apr 2026 12:05:18 -0400 Subject: [PATCH 052/250] Gate scalar fn array plugin for tensor exprs (#7447) ## Summary This should have been a part of https://github.com/vortex-data/vortex/pull/7437 ## Testing N/A Signed-off-by: Connor Tsui --- vortex-tensor/public-api.lock | 2 ++ vortex-tensor/src/lib.rs | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 01ff0a6f477..99dc8f0edbd 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -588,4 +588,6 @@ pub fn vortex_tensor::vector_search::build_similarity_search_tree vortex_error::VortexResult +pub const vortex_tensor::SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV: &str + pub fn vortex_tensor::initialize(session: &vortex_session::VortexSession) diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index d2fb02a6c7f..68269407473 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -31,13 +31,19 @@ pub mod vector_search; mod utils; +/// Environment variable that gates registration of the tensor scalar-fn array plugins (the array +/// encodings that let [`CosineSimilarity`], [`InnerProduct`], [`L2Denorm`], [`L2Norm`], and +/// [`SorfTransform`] persist in a Vortex file). When unset, only the scalar functions themselves +/// are registered; readers of files containing serialized tensor scalar-fn arrays will fail to +/// deserialize. Opt-in by setting the variable to any non-empty value. +pub const SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV: &str = "VX_SCALAR_FN_ARRAY_TENSOR_PLUGIN"; + /// Initialize the Vortex tensor library with a Vortex session. pub fn initialize(session: &VortexSession) { session.dtypes().register(Vector); session.dtypes().register(FixedShapeTensor); let session_fns = session.scalar_fns(); - let session_arrays = session.arrays(); session_fns.register(CosineSimilarity); session_fns.register(InnerProduct); @@ -45,11 +51,19 @@ pub fn initialize(session: &VortexSession) { session_fns.register(L2Norm); session_fns.register(SorfTransform); - session_arrays.register(ScalarFnArrayPlugin::new(CosineSimilarity)); - session_arrays.register(ScalarFnArrayPlugin::new(InnerProduct)); - session_arrays.register(ScalarFnArrayPlugin::new(L2Denorm)); - session_arrays.register(ScalarFnArrayPlugin::new(L2Norm)); - session_arrays.register(ScalarFnArrayPlugin::new(SorfTransform)); + // Registering the scalar-fn array plugins lets the tensor scalar fns be serialized as array + // encodings inside Vortex files. Gate this on an env var so applications that do not intend + // to persist these encodings do not pay the registry cost or widen their stable-encoding + // surface unintentionally. + if std::env::var_os(SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV).is_some_and(|v| !v.is_empty()) { + let session_arrays = session.arrays(); + + session_arrays.register(ScalarFnArrayPlugin::new(CosineSimilarity)); + session_arrays.register(ScalarFnArrayPlugin::new(InnerProduct)); + session_arrays.register(ScalarFnArrayPlugin::new(L2Denorm)); + session_arrays.register(ScalarFnArrayPlugin::new(L2Norm)); + session_arrays.register(ScalarFnArrayPlugin::new(SorfTransform)); + } } #[cfg(test)] From 940630360de420549045df2e19ff3ef9723b8ac4 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 15 Apr 2026 18:33:06 +0100 Subject: [PATCH 053/250] chore(ci): workflow cleanup and fixes (#7448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Audit and cleanup of `.github/` CI workflows — fixes bugs, removes dead code, deduplicates config. ### Changes (one commit each) 1. Fix fuzz corpus restore path mismatch in `run-fuzzer.yml` — restore appended `-${{ inputs.extra_features }}` but persist did not, so previously-saved corpora were never restored 2. Fix stale PR message saying "30 days" when `days-before-pr-stale` is 14 3. Remove dead `Install Sweep` / `Timestamp Cache` steps from `setup-rust` action — they gate on undeclared `inputs.timestamp` so never run 4. Use `--bin` instead of `--package` in `bench-pr.yml` build step to match `bench.yml` 5. Right-size job timeouts: `bench.yml` commit-metadata 120m → 10m, `docs.yml` deploy 120m → 10m, `stale.yml` add missing timeout 6. Deduplicate SQL benchmark matrix — move single source of truth into `sql-benchmarks.yml` default input, derive lance targets from `mode` at runtime, sync matrix with develop (adds `local_dir` to statpopgen, drops stale `scale_factor`/`iterations` from fineweb/S3 entries, adds `vortex-compact` to clickbench) 7. Set `CARGO_TERM_COLOR: always` in `ci.yml` — `auto` disables color without a TTY, all other workflows already use `always` 8. Hoist R2 env vars to job-level in `run-fuzzer.yml`, `minimize_fuzz_corpus_workflow.yml`, `fuzz-coverage.yml` — removes 5 duplicate env blocks before and after for post merge, and on pr: ``` post-merge — before vs after: ┌──────────────┬─────────────────────┬────────────────────┬───────────────────┬──────────────────┐ │ Entry │ scale_factor before │ scale_factor after │ iterations before │ iterations after │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ clickbench │ - │ - │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-nvme │ 1.0 │ 1.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-s3 │ 1.0 │ 1.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-nvme-10 │ 10.0 │ 10.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-s3-10 │ 10.0 │ 10.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpcds │ 1.0 │ 1.0 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ statpopgen │ 100 │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ fineweb │ - │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ fineweb-s3 │ - │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ polarsignals │ 1 │ 1 │ - │ - │ └──────────────┴─────────────────────┴────────────────────┴───────────────────┴──────────────────┘ PR — before vs after: ┌──────────────┬─────────────────────┬────────────────────┬───────────────────┬──────────────────┐ │ Entry │ scale_factor before │ scale_factor after │ iterations before │ iterations after │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ clickbench │ - │ - │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-nvme │ 1.0 │ 1.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-s3 │ 1.0 │ 1.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-nvme-10 │ 10.0 │ 10.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpch-s3-10 │ 10.0 │ 10.0 │ - │ 10 │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ tpcds │ 1.0 │ 1.0 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ statpopgen │ 100 │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ fineweb │ 100 │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ fineweb-s3 │ 100 │ 100 │ - │ - │ ├──────────────┼─────────────────────┼────────────────────┼───────────────────┼──────────────────┤ │ polarsignals │ 1 │ 1 │ - │ - │ ``` --------- Signed-off-by: Alexander Droste --- .github/actions/setup-rust/action.yml | 10 --- .github/workflows/bench-pr.yml | 79 ------------------ .github/workflows/bench.yml | 82 +------------------ .github/workflows/ci.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/fuzz-coverage.yml | 10 +-- .../minimize_fuzz_corpus_workflow.yml | 15 ++-- .github/workflows/run-fuzzer.yml | 16 ++-- .github/workflows/sql-benchmarks.yml | 38 +++++++-- .github/workflows/stale.yml | 3 +- 10 files changed, 49 insertions(+), 208 deletions(-) diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 5fea9acaa53..2af54b1c20f 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -83,13 +83,3 @@ runs: - name: Install Protoc (for lance-encoding build step) if: runner.os != 'Windows' uses: ./.github/actions/setup-protoc - - - name: Install Sweep - shell: bash - if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }} - run: cargo install cargo-sweep - - - name: Timestamp Cache - shell: bash - if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }} - run: cargo sweep --stamp diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index d3de66765dc..34610171419 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -137,82 +137,3 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_matrix: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb", - "extra_data_formats": "vortex-compact" - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "1.0" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "1.0" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "10.0" - }, - { - "id": "tpch-s3-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on S3", - "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "10.0" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "targets": "datafusion:vortex", - "scale_factor": "1" - }, - ] diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index abc80c1d93e..645faf9fb37 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -14,7 +14,7 @@ permissions: jobs: commit-metadata: runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Setup AWS CLI @@ -118,83 +118,3 @@ jobs: secrets: inherit with: mode: "develop" - benchmark_matrix: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "build_lance": true - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "1.0", - "build_lance": true - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "1.0" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "10.0", - "build_lance": true - }, - { - "id": "tpch-s3-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on S3", - "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "10.0" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "targets": "datafusion:vortex", - "scale_factor": "1" - }, - ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 290612d099b..5126ed3e392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ permissions: issues: write # audit-check creates issues env: - CARGO_TERM_COLOR: auto + CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 NIGHTLY_TOOLCHAIN: nightly-2026-02-05 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6a7b6c47f4f..307526d92f3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -66,7 +66,7 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 10 needs: build steps: # Note, since we provide the job with a CloudFlare scoped API token, we run it in a separate job that doesn't diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index ee4ed865b4b..3aab125d671 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -11,6 +11,11 @@ env: jobs: coverage: name: "Coverage: ${{ matrix.fuzz_target }}" + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" strategy: fail-fast: false matrix: @@ -56,11 +61,6 @@ jobs: - name: Download corpus from R2 shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ matrix.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ matrix.fuzz_target }}" diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index 784d1e059d1..47475c8e8f9 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -34,6 +34,11 @@ env: jobs: minimize: name: "Minimize ${{ inputs.fuzz_target }}" + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=arm64-medium/disk=large/tag={1}-minimize', github.run_id, inputs.fuzz_target) @@ -64,11 +69,6 @@ jobs: - name: Restore corpus shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" @@ -98,11 +98,6 @@ jobs: - name: Persist corpus shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index 82f0dd0ace4..de1f1e0bf3b 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -61,6 +61,10 @@ jobs: name: "Run ${{ inputs.fuzz_name || inputs.fuzz_target }}" env: FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" timeout-minutes: 230 # almost 4 hours runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -95,14 +99,9 @@ jobs: - name: Restore corpus shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" - CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}-${{ inputs.extra_features }}" + CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" # Try to download corpus if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then @@ -189,11 +188,6 @@ jobs: - name: Persist corpus shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index c12a5fe7bea..2abc1f9be70 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -14,7 +14,6 @@ on: required: false type: string description: "JSON string containing the matrix configuration" - # We do not include lance in the default configuration. default: | [ { @@ -28,7 +27,8 @@ on: "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "1.0" + "scale_factor": "1.0", + "iterations": "10" }, { "id": "tpch-s3", @@ -45,7 +45,8 @@ on: "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", - "scale_factor": "10.0" + "scale_factor": "10.0", + "iterations": "10" }, { "id": "tpch-s3-10", @@ -68,6 +69,7 @@ on: "id": "statpopgen", "subcommand": "statpopgen", "name": "Statistical and Population Genetics", + "local_dir": "vortex-bench/data/statpopgen", "targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", "scale_factor": "100" }, @@ -85,8 +87,7 @@ on: "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", - "scale_factor": "100", - "iterations": "10" + "scale_factor": "100" }, { "id": "polarsignals", @@ -135,13 +136,32 @@ jobs: - uses: ./.github/actions/system-info + - name: Resolve targets + id: resolve + shell: bash + run: | + targets="${{ matrix.targets }}" + # Non-PR modes include additional targets + if [ "${{ inputs.mode }}" != "pr" ]; then + case "${{ matrix.subcommand }}" in + clickbench) targets="$targets,datafusion:vortex-compact,duckdb:vortex-compact" ;; + esac + # Lance comparisons for local clickbench/tpch (not S3) + if [ -z "${{ matrix.remote_storage }}" ]; then + case "${{ matrix.subcommand }}" in + clickbench|tpch) targets="$targets,datafusion:lance" ;; + esac + fi + fi + echo "targets=$targets" >> $GITHUB_OUTPUT + - name: Build binaries shell: bash env: RUSTFLAGS: "-C target-cpu=native" run: | packages="--bin data-gen --bin datafusion-bench --bin duckdb-bench" - if [ "${{ matrix.build_lance }}" = "true" ]; then + if [ "${{ inputs.mode }}" != "pr" ]; then packages="$packages --bin lance-bench" fi cargo build $packages --profile release_debug --features unstable_encodings @@ -152,7 +172,7 @@ jobs: RUST_BACKTRACE: full run: | # Extract all unique formats from targets (e.g., "datafusion:parquet,duckdb:vortex" -> "parquet,vortex") - all_formats=$(echo "${{ matrix.targets }}" | tr ',' '\n' | sed 's/^[^:]*://' | sort -u | tr '\n' ',' | sed 's/,$//') + all_formats=$(echo "${{ steps.resolve.outputs.targets }}" | tr ',' '\n' | sed 's/^[^:]*://' | sort -u | tr '\n' ',' | sed 's/,$//') # Append extra data formats if specified (for file size tracking without benchmarking) if [ -n "${{ matrix.extra_data_formats }}" ]; then @@ -206,7 +226,7 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ + bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ steps.resolve.outputs.targets }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }} @@ -221,7 +241,7 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ matrix.targets }}" \ + bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ steps.resolve.outputs.targets }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ --remote-storage "${{ matrix.remote_storage }}" \ --benchmark-id "${{ matrix.id }}" \ diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d24b6a1ec6b..04d44fec222 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,6 +7,7 @@ on: jobs: close-issues: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: pull-requests: write steps: @@ -17,7 +18,7 @@ jobs: # PR has 7 more days to become active, otherwise it will be closed. days-before-pr-close: 7 stale-pr-label: "stale" - stale-pr-message: "This PR has been marked as stale because it has been open for 30 days with no activity. Please comment or remove the stale label if you wish to keep it active, otherwise it will be closed in 7 days" + stale-pr-message: "This PR has been marked as stale because it has been open for 14 days with no activity. Please comment or remove the stale label if you wish to keep it active, otherwise it will be closed in 7 days" close-pr-message: "This PR was closed because it has been inactive for 7 days since being marked as stale." days-before-issue-stale: -1 days-before-issue-close: -1 From bff43dc81957fd05bdf479433f8860a7887d0ce4 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:23:21 -0400 Subject: [PATCH 054/250] Vector datasets catalog and downloader (#7446) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 We will want to add vector benchmarking soon (see https://github.com/vortex-data/vortex/pull/7399 for a draft). This adds a simple catalog for the vector datasets hosted by `https://assets.zilliz.com/benchmark` for [VectorDBBench](https://github.com/zilliztech/vectordbbench), which both describes the shape of the datasets (are things partitioned, randomly shuffled, are there neighbors lists for top k, etc). Also handles downloading everything. I had to verify that all of this stuff was correct by looking at the S3 buckets themselves: ```sh aws s3 ls s3://assets.zilliz.com/benchmark/ --region us-west-2 --no-sign-request ```

```sh for d in bioasq_large_10m bioasq_medium_1m cohere_large_10m cohere_medium_1m \ cohere_small_100k gist_medium_1m gist_small_100k glove_medium_1m \ glove_small_100k laion_large_100m \ openai_large_5m openai_medium_500k openai_small_50k \ sift_large_50m sift_medium_5m sift_small_500k; do echo "=== $d ===" aws s3 ls s3://assets.zilliz.com/benchmark/$d/ --region us-west-2 --no-sign-request done ```
And this script from the main repo helped too: https://github.com/zilliztech/VectorDBBench/blob/main/vectordb_bench/backend/dataset.py --- Things that are not implemented that I would like to add: - Is the dataset pre-normalized for cosine similarity? This is not so obvious to me without actually working with the datasets, so I will do this later. - Some datasets have scalar labels for all vectors that help mimic similarity + filter by some other column. Some of them also have neighbor lists for these specific filtered queries. So that is something we'll probably want to add in the future. ## Testing N/A Signed-off-by: Connor Tsui --- vortex-bench/src/lib.rs | 1 + vortex-bench/src/vector_dataset/catalog.rs | 405 ++++++++++++++++++++ vortex-bench/src/vector_dataset/download.rs | 225 +++++++++++ vortex-bench/src/vector_dataset/layout.rs | 185 +++++++++ vortex-bench/src/vector_dataset/mod.rs | 38 ++ vortex-bench/src/vector_dataset/paths.rs | 116 ++++++ 6 files changed, 970 insertions(+) create mode 100644 vortex-bench/src/vector_dataset/catalog.rs create mode 100644 vortex-bench/src/vector_dataset/download.rs create mode 100644 vortex-bench/src/vector_dataset/layout.rs create mode 100644 vortex-bench/src/vector_dataset/mod.rs create mode 100644 vortex-bench/src/vector_dataset/paths.rs diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs index af0d3fdef30..db9bd69c6ce 100644 --- a/vortex-bench/src/lib.rs +++ b/vortex-bench/src/lib.rs @@ -53,6 +53,7 @@ pub mod statpopgen; pub mod tpcds; pub mod tpch; pub mod utils; +pub mod vector_dataset; pub use benchmark::Benchmark; pub use benchmark::TableSpec; diff --git a/vortex-bench/src/vector_dataset/catalog.rs b/vortex-bench/src/vector_dataset/catalog.rs new file mode 100644 index 00000000000..5549add0c14 --- /dev/null +++ b/vortex-bench/src/vector_dataset/catalog.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The static catalog of hosted vector benchmark corpora. +//! +//! Every entry in [`ALL_VECTOR_DATASETS`] declares its dimensionality, row count, element type, +//! distance metric, the set of train-split layouts hosted on the upstream bucket, and whether +//! ground-truth `neighbors.parquet` and `scalar_labels` columns are available. +//! Most entries mirror VectorDBBench's dataset model; a few are extra prefixes that are present +//! on the same public bucket and use the same file layout. +//! +//! Higher-level code (downloaders, ingest pipelines, recall measurement) should be parameterized +//! over these descriptors rather than hardcoding per-dataset URLs. + +use std::num::NonZeroU32; + +use anyhow::Result; +use anyhow::bail; +use clap::ValueEnum; +use vortex::dtype::PType; + +use super::layout::LayoutSpec; +use super::layout::TrainLayout; +use super::layout::VectorMetric; + +const TEN_SHARDS: NonZeroU32 = NonZeroU32::new(10).unwrap(); +const FIFTY_SHARDS: NonZeroU32 = NonZeroU32::new(50).unwrap(); +const ONE_HUNDRED_SHARDS: NonZeroU32 = NonZeroU32::new(100).unwrap(); + +/// Every [`VectorDataset`] variant in catalog order. Useful for CLI help and metadata-consistency +/// tests. +pub const ALL_VECTOR_DATASETS: &[VectorDataset] = &[ + VectorDataset::CohereSmall100k, + VectorDataset::CohereMedium1m, + VectorDataset::CohereLarge10m, + VectorDataset::OpenaiSmall50k, + VectorDataset::OpenaiMedium500k, + VectorDataset::OpenaiLarge5m, + VectorDataset::BioasqMedium1m, + VectorDataset::BioasqLarge10m, + VectorDataset::GloveSmall100k, + VectorDataset::GloveMedium1m, + VectorDataset::GistSmall100k, + VectorDataset::GistMedium1m, + VectorDataset::SiftSmall500k, + VectorDataset::SiftMedium5m, + VectorDataset::SiftLarge50m, + VectorDataset::LaionLarge100m, +]; + +/// The publicly hosted vector benchmark datasets. +/// +/// Variants are named ``, kebab-cased on the CLI (e.g. `cohere-large-10m`). +/// +/// The static metadata for each variant (dimensionality, row count, hosted layouts, etc.) is +/// exposed via the inherent methods below; the full table is reachable via [`ALL_VECTOR_DATASETS`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ValueEnum)] +#[clap(rename_all = "kebab-case")] +pub enum VectorDataset { + /// Cohere wiki-22-12, 100K × 768 f32, cosine. Single + SingleShuffled. + CohereSmall100k, + /// Cohere wiki-22-12, 1M × 768 f32, cosine. Single + SingleShuffled. + CohereMedium1m, + /// Cohere wiki-22-12, 10M × 768 f32, cosine. Partitioned + PartitionedShuffled (10 shards). + CohereLarge10m, + + /// OpenAI embeddings on C4, 50K × 1536 f64, cosine. Single + SingleShuffled. + OpenaiSmall50k, + /// OpenAI embeddings on C4, 500K × 1536 f64, cosine. Single + SingleShuffled. + OpenaiMedium500k, + /// OpenAI embeddings on C4, 5M × 1536 f64, cosine. Partitioned + PartitionedShuffled (10 + /// shards). + OpenaiLarge5m, + + /// Bioasq biomedical, 1M × 1024 f32, cosine. SingleShuffled only. + BioasqMedium1m, + /// Bioasq biomedical, 10M × 1024 f32, cosine. PartitionedShuffled only (10 shards). + BioasqLarge10m, + + /// GloVe word vectors, 100K × 200 f32, cosine. Single only. No neighbors / labels. + GloveSmall100k, + /// GloVe word vectors, 1M × 200 f32, cosine. Single only. No neighbors / labels. + GloveMedium1m, + + /// GIST image features, 100K × 960 f32, L2. Single only. No neighbors / labels. + GistSmall100k, + /// GIST image features, 1M × 960 f32, L2. Single only. No neighbors / labels. + GistMedium1m, + + /// SIFT image features, 500K × 128 f32, L2. Single only. No neighbors / labels. + SiftSmall500k, + /// SIFT image features, 5M × 128 f32, L2. Single only. No neighbors / labels. + SiftMedium5m, + /// SIFT image features, 50M × 128 f32, L2. Partitioned only (50 shards). No labels. + SiftLarge50m, + + /// LAION image embeddings, 100M × 768 f32, L2. Partitioned only (100 shards). + /// Has `neighbors.parquet` and `scalar_labels.parquet`. + LaionLarge100m, +} + +impl VectorDataset { + /// Stable kebab-cased label used in CLI args, file paths, and metric names. + pub fn name(&self) -> &'static str { + match self { + VectorDataset::CohereSmall100k => "cohere-small-100k", + VectorDataset::CohereMedium1m => "cohere-medium-1m", + VectorDataset::CohereLarge10m => "cohere-large-10m", + VectorDataset::OpenaiSmall50k => "openai-small-50k", + VectorDataset::OpenaiMedium500k => "openai-medium-500k", + VectorDataset::OpenaiLarge5m => "openai-large-5m", + VectorDataset::BioasqMedium1m => "bioasq-medium-1m", + VectorDataset::BioasqLarge10m => "bioasq-large-10m", + VectorDataset::GloveSmall100k => "glove-small-100k", + VectorDataset::GloveMedium1m => "glove-medium-1m", + VectorDataset::GistSmall100k => "gist-small-100k", + VectorDataset::GistMedium1m => "gist-medium-1m", + VectorDataset::SiftSmall500k => "sift-small-500k", + VectorDataset::SiftMedium5m => "sift-medium-5m", + VectorDataset::SiftLarge50m => "sift-large-50m", + VectorDataset::LaionLarge100m => "laion-large-100m", + } + } + + /// The directory name on `assets.zilliz.com/benchmark//`. Snake-cased to + /// match the upstream bucket's existing naming convention. + pub fn s3_prefix(&self) -> &'static str { + match self { + VectorDataset::CohereSmall100k => "cohere_small_100k", + VectorDataset::CohereMedium1m => "cohere_medium_1m", + VectorDataset::CohereLarge10m => "cohere_large_10m", + VectorDataset::OpenaiSmall50k => "openai_small_50k", + VectorDataset::OpenaiMedium500k => "openai_medium_500k", + VectorDataset::OpenaiLarge5m => "openai_large_5m", + VectorDataset::BioasqMedium1m => "bioasq_medium_1m", + VectorDataset::BioasqLarge10m => "bioasq_large_10m", + VectorDataset::GloveSmall100k => "glove_small_100k", + VectorDataset::GloveMedium1m => "glove_medium_1m", + VectorDataset::GistSmall100k => "gist_small_100k", + VectorDataset::GistMedium1m => "gist_medium_1m", + VectorDataset::SiftSmall500k => "sift_small_500k", + VectorDataset::SiftMedium5m => "sift_medium_5m", + VectorDataset::SiftLarge50m => "sift_large_50m", + VectorDataset::LaionLarge100m => "laion_large_100m", + } + } + + /// Vector dimensionality. + pub fn dim(&self) -> u32 { + match self { + VectorDataset::CohereSmall100k + | VectorDataset::CohereMedium1m + | VectorDataset::CohereLarge10m + | VectorDataset::LaionLarge100m => 768, + VectorDataset::OpenaiSmall50k + | VectorDataset::OpenaiMedium500k + | VectorDataset::OpenaiLarge5m => 1536, + VectorDataset::BioasqMedium1m | VectorDataset::BioasqLarge10m => 1024, + VectorDataset::GloveSmall100k | VectorDataset::GloveMedium1m => 200, + VectorDataset::GistSmall100k | VectorDataset::GistMedium1m => 960, + VectorDataset::SiftSmall500k + | VectorDataset::SiftMedium5m + | VectorDataset::SiftLarge50m => 128, + } + } + + /// Number of rows in the train split (sum across shards if partitioned). + pub fn num_rows(&self) -> u64 { + match self { + VectorDataset::CohereSmall100k => 100_000, + VectorDataset::CohereMedium1m => 1_000_000, + VectorDataset::CohereLarge10m => 10_000_000, + VectorDataset::OpenaiSmall50k => 50_000, + VectorDataset::OpenaiMedium500k => 500_000, + VectorDataset::OpenaiLarge5m => 5_000_000, + VectorDataset::BioasqMedium1m => 1_000_000, + VectorDataset::BioasqLarge10m => 10_000_000, + VectorDataset::GloveSmall100k => 100_000, + VectorDataset::GloveMedium1m => 1_000_000, + VectorDataset::GistSmall100k => 100_000, + VectorDataset::GistMedium1m => 1_000_000, + VectorDataset::SiftSmall500k => 500_000, + VectorDataset::SiftMedium5m => 5_000_000, + VectorDataset::SiftLarge50m => 50_000_000, + VectorDataset::LaionLarge100m => 100_000_000, + } + } + + /// Element scalar type as stored in the upstream parquet. The benchmark always casts to + /// `f32` after ingest (TurboQuant + the handrolled baseline operate in f32), so this is + /// only consulted by the parquet readers. + pub fn element_ptype(&self) -> PType { + match self { + VectorDataset::OpenaiSmall50k + | VectorDataset::OpenaiMedium500k + | VectorDataset::OpenaiLarge5m => PType::F64, + _ => PType::F32, + } + } + + /// Distance metric the upstream dataset was curated for. + pub fn metric(&self) -> VectorMetric { + match self { + VectorDataset::GistSmall100k + | VectorDataset::GistMedium1m + | VectorDataset::SiftSmall500k + | VectorDataset::SiftMedium5m + | VectorDataset::SiftLarge50m + | VectorDataset::LaionLarge100m => VectorMetric::L2, + _ => VectorMetric::Cosine, + } + } + + /// The set of train-split layouts hosted on the upstream bucket for this dataset. + /// + /// Always non-empty since the catalog never declares a dataset with zero layouts. + pub fn layouts(&self) -> Vec { + match self { + VectorDataset::CohereSmall100k + | VectorDataset::CohereMedium1m + | VectorDataset::OpenaiSmall50k + | VectorDataset::OpenaiMedium500k => { + vec![LayoutSpec::single(), LayoutSpec::single_shuffled()] + } + VectorDataset::CohereLarge10m | VectorDataset::OpenaiLarge5m => vec![ + LayoutSpec::partitioned(TEN_SHARDS), + LayoutSpec::partitioned_shuffled(TEN_SHARDS), + ], + VectorDataset::BioasqMedium1m => vec![LayoutSpec::single_shuffled()], + VectorDataset::BioasqLarge10m => { + vec![LayoutSpec::partitioned_shuffled(TEN_SHARDS)] + } + VectorDataset::GloveSmall100k + | VectorDataset::GloveMedium1m + | VectorDataset::GistSmall100k + | VectorDataset::GistMedium1m + | VectorDataset::SiftSmall500k + | VectorDataset::SiftMedium5m => vec![LayoutSpec::single()], + VectorDataset::SiftLarge50m => vec![LayoutSpec::partitioned(FIFTY_SHARDS)], + VectorDataset::LaionLarge100m => vec![LayoutSpec::partitioned(ONE_HUNDRED_SHARDS)], + } + } + + /// Whether `neighbors.parquet` (top-K ground truth for recall) is hosted. + pub fn has_neighbors(&self) -> bool { + match self { + VectorDataset::CohereSmall100k + | VectorDataset::CohereMedium1m + | VectorDataset::CohereLarge10m + | VectorDataset::OpenaiSmall50k + | VectorDataset::OpenaiMedium500k + | VectorDataset::OpenaiLarge5m + | VectorDataset::BioasqMedium1m + | VectorDataset::BioasqLarge10m + | VectorDataset::LaionLarge100m => true, + VectorDataset::GloveSmall100k + | VectorDataset::GloveMedium1m + | VectorDataset::GistSmall100k + | VectorDataset::GistMedium1m + | VectorDataset::SiftSmall500k + | VectorDataset::SiftMedium5m + | VectorDataset::SiftLarge50m => false, + } + } + + /// Whether the train split carries a `scalar_labels` column (for filtered-search + /// benchmarks). The benchmark does not exercise filtered search yet, but the ingest + /// pipeline copies the column through when present so future filtered-recall work does + /// not require re-ingest. + pub fn has_scalar_labels(&self) -> bool { + matches!( + self, + VectorDataset::CohereSmall100k + | VectorDataset::CohereMedium1m + | VectorDataset::CohereLarge10m + | VectorDataset::OpenaiSmall50k + | VectorDataset::OpenaiMedium500k + | VectorDataset::OpenaiLarge5m + | VectorDataset::BioasqMedium1m + | VectorDataset::BioasqLarge10m + | VectorDataset::LaionLarge100m + ) + } + + /// Validate that `layout` is one of the layouts hosted for this dataset and return its + /// [`LayoutSpec`]. + /// + /// Bails with an error message that lists the allowed values. + pub fn validate_layout(&self, layout: TrainLayout) -> Result { + let layouts = self.layouts(); + match layouts.iter().find(|spec| spec.layout() == layout) { + Some(spec) => Ok(*spec), + None => { + let allowed = layouts + .iter() + .map(|s| s.layout().label()) + .collect::>() + .join(", "); + bail!( + "dataset {} does not have layout '{}'; allowed layouts: [{}]", + self.name(), + layout, + allowed, + ); + } + } + } + + /// Pick the default layout for this dataset — the first entry in [`Self::layouts`]. + /// Stable across runs since the catalog table is statically ordered. + pub fn default_layout(&self) -> LayoutSpec { + self.layouts()[0] + } +} + +#[cfg(test)] +mod tests { + use vortex::utils::aliases::hash_set::HashSet; + + use super::*; + + #[test] + fn all_datasets_have_consistent_metadata() { + let mut seen: HashSet<&'static str> = HashSet::default(); + for &ds in ALL_VECTOR_DATASETS { + assert!(seen.insert(ds.name()), "duplicate name {}", ds.name()); + assert!(ds.dim() > 0); + assert!(ds.num_rows() > 0); + assert!(!ds.layouts().is_empty(), "{} has no layouts", ds.name()); + assert_eq!( + ds.s3_prefix().chars().filter(|c| *c == '_').count() + 1, + ds.name().split('-').count(), + "{}: s3_prefix '{}' shape disagrees with name '{}'", + ds.name(), + ds.s3_prefix(), + ds.name(), + ); + } + } + + #[test] + fn validate_layout_accepts_hosted_layout() { + let ds = VectorDataset::CohereSmall100k; + let spec = ds.validate_layout(TrainLayout::Single).unwrap(); + assert_eq!(spec.layout(), TrainLayout::Single); + assert_eq!(spec.num_files(), 1); + } + + #[test] + fn validate_layout_rejects_unhosted_layout() { + let ds = VectorDataset::SiftSmall500k; + let err = ds + .validate_layout(TrainLayout::SingleShuffled) + .unwrap_err() + .to_string(); + assert!( + err.contains("does not have layout 'single-shuffled'"), + "{err}" + ); + assert!(err.contains("[single]"), "{err}"); + } + + #[test] + fn partitioned_datasets_declare_shard_count() { + let layouts = VectorDataset::CohereLarge10m.layouts(); + assert_eq!(layouts.len(), 2); + for spec in layouts { + assert!(spec.layout().is_partitioned()); + assert_eq!(spec.num_files(), 10); + } + } + + #[test] + fn l2_datasets_match_upstream_metric() { + for ds in [ + VectorDataset::GistSmall100k, + VectorDataset::GistMedium1m, + VectorDataset::SiftSmall500k, + VectorDataset::SiftMedium5m, + VectorDataset::SiftLarge50m, + VectorDataset::LaionLarge100m, + ] { + assert_eq!(ds.metric(), VectorMetric::L2, "{} should use L2", ds.name()); + } + } + + #[test] + fn datasets_without_neighbors_skip_recall() { + for ds in [ + VectorDataset::GloveSmall100k, + VectorDataset::GloveMedium1m, + VectorDataset::GistSmall100k, + VectorDataset::GistMedium1m, + VectorDataset::SiftSmall500k, + VectorDataset::SiftMedium5m, + VectorDataset::SiftLarge50m, + ] { + assert!( + !ds.has_neighbors(), + "{} unexpectedly has neighbors", + ds.name() + ); + } + } +} diff --git a/vortex-bench/src/vector_dataset/download.rs b/vortex-bench/src/vector_dataset/download.rs new file mode 100644 index 00000000000..da4b43262b6 --- /dev/null +++ b/vortex-bench/src/vector_dataset/download.rs @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! URL builders and idempotent download driver for vector benchmark datasets. +//! +//! The upstream bucket is `https://assets.zilliz.com/benchmark//`. Within each +//! prefix the train split is named according to a four-way convention: +//! +//! - `Single`: `train.parquet` +//! - `SingleShuffled`: `shuffle_train.parquet` +//! - `Partitioned`: `train-NN-of-MM.parquet` +//! - `PartitionedShuffled`: `shuffle_train-NN-of-MM.parquet` +//! +//! `test.parquet` and (when present) `neighbors.parquet` live alongside the train files. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use bytes::Bytes; +use futures::StreamExt; +use indicatif::ProgressBar; +use indicatif::ProgressStyle; +use reqwest::Client; +use reqwest::IntoUrl; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; +use tokio::task::JoinSet; +use tracing::info; +use tracing::warn; + +use crate::datasets::data_downloads::download_data; +use crate::utils::file::idempotent_async; +use crate::vector_dataset::catalog::VectorDataset; +use crate::vector_dataset::layout::LayoutSpec; +use crate::vector_dataset::layout::TrainLayout; +use crate::vector_dataset::paths; + +/// Bucket root for all VectorDBBench datasets we mirror against. +const BENCHMARK_ROOT: &str = "https://assets.zilliz.com/benchmark"; + +/// All train-shard URLs for a `(dataset, layout)` pair. Length matches `layout.num_files()`. +pub fn train_urls(ds: VectorDataset, spec: LayoutSpec) -> Vec { + let prefix = format!("{BENCHMARK_ROOT}/{}", ds.s3_prefix()); + let layout = spec.layout(); + if layout.is_partitioned() { + let n = spec.num_files(); + (0..n) + .map(|i| format!("{prefix}/{}", partitioned_file_name(layout, i, n),)) + .collect() + } else { + let name = match layout { + TrainLayout::Single => "train.parquet", + TrainLayout::SingleShuffled => "shuffle_train.parquet", + _ => unreachable!("non-partitioned guard above"), + }; + vec![format!("{prefix}/{name}")] + } +} + +/// URL for `test.parquet`. +pub fn test_url(ds: VectorDataset) -> String { + format!("{BENCHMARK_ROOT}/{}/test.parquet", ds.s3_prefix()) +} + +/// URL for `neighbors.parquet`, or `None` when the dataset doesn't host one. +pub fn neighbors_url(ds: VectorDataset) -> Option { + ds.has_neighbors() + .then(|| format!("{BENCHMARK_ROOT}/{}/neighbors.parquet", ds.s3_prefix())) +} + +fn partitioned_file_name(layout: TrainLayout, shard_idx: u32, num_files: u32) -> String { + let prefix = match layout { + TrainLayout::Partitioned => "train", + TrainLayout::PartitionedShuffled => "shuffle_train", + _ => unreachable!("partitioned guard"), + }; + format!( + "{prefix}-{shard_idx:0width$}-of-{num_files:0width$}.parquet", + width = num_files_width(num_files), + ) +} + +fn num_files_width(num_files: u32) -> usize { + let digits = num_files.checked_ilog10().unwrap_or(0) as usize + 1; + digits.max(2) +} + +/// Local on-disk paths to the cached parquet files for a `(dataset, layout)` pair after +/// [`download`] returns successfully. +#[derive(Debug, Clone)] +pub struct DatasetPaths { + /// Per-shard train parquet paths in shard order. + pub train_files: Vec, + /// `test.parquet`. + pub test: PathBuf, + /// `neighbors.parquet` if the dataset hosts top-K ground truth. + pub neighbors: Option, +} + +/// Download every parquet file required to run a `(dataset, layout)` benchmark, returning local +/// on-disk paths. +/// +/// This has idempotent semantics, so files already present on disk are skipped, and re-runs only +/// pay for new files. +/// +/// Train shards download in parallel using a shared HTTP client; the small `test.parquet` and +/// `neighbors.parquet` files use the simple [`download_data`] helper. +pub async fn download(ds: VectorDataset, layout: TrainLayout) -> Result { + let spec = ds.validate_layout(layout)?; + let urls = train_urls(ds, spec); + let train_targets = paths::train_files(ds, layout, spec.num_files()); + debug_assert_eq!(urls.len(), train_targets.len()); + + let client = Client::builder() + .timeout(Duration::from_secs(60 * 60)) + .build() + .context("build reqwest client")?; + + let mut tasks: JoinSet> = JoinSet::new(); + for (url, target) in urls.into_iter().zip(train_targets.iter().cloned()) { + let client = client.clone(); + tasks.spawn(async move { + idempotent_async(target, |tmp| async move { + info!("downloading {}", url); + if spec.layout().is_partitioned() { + download_with_retry(&client, &url, &tmp).await?; + } else { + download_with_progress(&client, &url, &tmp).await?; + } + Ok(()) + }) + .await?; + Ok(()) + }); + } + while let Some(joined) = tasks.join_next().await { + joined.context("train download task panicked")??; + } + + let test = download_data(paths::test_path(ds, layout), &test_url(ds)) + .await + .with_context(|| format!("download test.parquet for {}", ds.name()))?; + + let neighbors = if let Some(url) = neighbors_url(ds) { + Some( + download_data(paths::neighbors_path(ds, layout), &url) + .await + .with_context(|| format!("download neighbors.parquet for {}", ds.name()))?, + ) + } else { + None + }; + + Ok(DatasetPaths { + train_files: train_targets, + test, + neighbors, + }) +} + +/// Stream a large file to disk with a byte-progress bar. +async fn download_with_progress(client: &Client, url: &str, output: &PathBuf) -> Result<()> { + let response = client + .get(url) + .send() + .await + .with_context(|| format!("GET {url}"))? + .error_for_status()?; + let total = response.content_length().unwrap_or(0); + + let progress = ProgressBar::new(total); + progress.set_style( + ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} ({bytes_per_sec})", + ) + .expect("valid template"), + ); + + let mut file = File::create(output).await?; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + file.write_all(&chunk).await?; + progress.inc(chunk.len() as u64); + } + progress.finish_and_clear(); + file.flush().await?; + Ok(()) +} + +/// Buffer-the-whole-body download with simple exponential backoff. Used for partitioned +/// shards because we already have download concurrency at the shard granularity. +async fn download_with_retry(client: &Client, url: &str, output: &PathBuf) -> Result<()> { + let body = retry_get(client, url).await?; + let mut file = File::create(output).await?; + file.write_all(&body).await?; + file.flush().await?; + Ok(()) +} + +async fn retry_get(client: &Client, url: impl IntoUrl + Clone) -> Result { + const MAX_ATTEMPTS: u32 = 4; + let mut last_err: Option = None; + for attempt in 0..MAX_ATTEMPTS { + let outcome: Result = async { + let resp = client.get(url.clone()).send().await?.error_for_status()?; + Ok(resp.bytes().await?) + } + .await; + match outcome { + Ok(b) => return Ok(b), + Err(e) => last_err = Some(e), + } + let backoff = Duration::from_secs(1u64 << attempt); + warn!( + "download attempt {} failed; retrying in {:?}", + attempt + 1, + backoff + ); + tokio::time::sleep(backoff).await; + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("retry_get exhausted with no recorded error"))) +} diff --git a/vortex-bench/src/vector_dataset/layout.rs b/vortex-bench/src/vector_dataset/layout.rs new file mode 100644 index 00000000000..3356776d0c3 --- /dev/null +++ b/vortex-bench/src/vector_dataset/layout.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Train-split layout variants for vector benchmark datasets. +//! +//! VectorDBBench corpora are published under `assets.zilliz.com/benchmark//` in up to four +//! different shapes: +//! - a single train file +//! - a single shuffled-rows train file +//! - a partitioned train file split into N shards +//! - the same partitioned shape in shuffled-rows order. +//! +//! Not every dataset hosts every layout. See [`VectorDataset::layouts`] for the per-dataset list. +//! +//! [`VectorDataset::layouts`]: super::VectorDataset::layouts + +use std::fmt; +use std::num::NonZeroU32; + +use clap::ValueEnum; +use serde::Deserialize; +use serde::Serialize; + +/// Distance metric a dataset was curated for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VectorMetric { + /// `dot(a, b) / (||a|| * ||b||)`. + Cosine, + /// `sum((a - b)^2)`. + L2, + // TODO(connor): Do we even need this? + /// `dot(a, b)`. + InnerProduct, +} + +/// A specific train layout published for a dataset, plus the shard count when partitioned +/// (always `1` for `Single` / `SingleShuffled`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LayoutSpec { + layout: TrainLayout, + num_files: NonZeroU32, +} + +impl LayoutSpec { + /// Build a single-file layout spec. + pub const fn single() -> Self { + Self { + layout: TrainLayout::Single, + num_files: NonZeroU32::MIN, + } + } + + /// Build a shuffled single-file layout spec. + pub const fn single_shuffled() -> Self { + Self { + layout: TrainLayout::SingleShuffled, + num_files: NonZeroU32::MIN, + } + } + + /// Build a partitioned layout spec with the given shard count. + pub const fn partitioned(num_files: NonZeroU32) -> Self { + Self { + layout: TrainLayout::Partitioned, + num_files, + } + } + + /// Build a shuffled partitioned layout spec with the given shard count. + pub const fn partitioned_shuffled(num_files: NonZeroU32) -> Self { + Self { + layout: TrainLayout::PartitionedShuffled, + num_files, + } + } + + /// Which of the four published shapes this entry describes. + pub const fn layout(&self) -> TrainLayout { + self.layout + } + + /// Number of parquet shards on the bucket. `1` for the single-file layouts. + pub const fn num_files(&self) -> u32 { + self.num_files.get() + } +} + +/// One of the four published train-split shapes for a VectorDBBench corpus. +/// +/// `Single` and `SingleShuffled` are one-file layouts; `Partitioned` and `PartitionedShuffled` are +/// sharded into N files. The shuffled variants randomize the row order, which is useful when you +/// want the on-disk arrangement to be representative of a query workload rather than of the +/// upstream ingest order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TrainLayout { + /// One `train.parquet` file. Row order matches the upstream curation. + #[clap(name = "single")] + Single, + /// One `shuffle_train.parquet` file. Row order is randomized. + #[clap(name = "single-shuffled")] + SingleShuffled, + /// Multiple `train-NN-of-N.parquet` shards. Row order matches the upstream curation. + #[clap(name = "partitioned")] + Partitioned, + /// Multiple `shuffle_train-NN-of-N.parquet` shards. Row order is randomized. + #[clap(name = "partitioned-shuffled")] + PartitionedShuffled, +} + +impl TrainLayout { + /// Stable kebab-cased label used in CLI args, file paths, and metric names. + pub fn label(&self) -> &'static str { + match self { + TrainLayout::Single => "single", + TrainLayout::SingleShuffled => "single-shuffled", + TrainLayout::Partitioned => "partitioned", + TrainLayout::PartitionedShuffled => "partitioned-shuffled", + } + } + + /// Whether this layout is split across multiple parquet files. + pub fn is_partitioned(&self) -> bool { + matches!( + self, + TrainLayout::Partitioned | TrainLayout::PartitionedShuffled + ) + } +} + +impl fmt::Display for TrainLayout { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.label()) + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU32; + + use super::*; + + #[test] + fn label_round_trips_through_value_enum() { + for layout in [ + TrainLayout::Single, + TrainLayout::SingleShuffled, + TrainLayout::Partitioned, + TrainLayout::PartitionedShuffled, + ] { + let parsed = TrainLayout::from_str(layout.label(), true).unwrap(); + assert_eq!(parsed, layout); + } + } + + #[test] + fn is_partitioned_matches_variant() { + assert!(!TrainLayout::Single.is_partitioned()); + assert!(!TrainLayout::SingleShuffled.is_partitioned()); + assert!(TrainLayout::Partitioned.is_partitioned()); + assert!(TrainLayout::PartitionedShuffled.is_partitioned()); + } + + #[test] + fn layout_specs_encode_valid_shape_and_count() { + assert_eq!(LayoutSpec::single().layout(), TrainLayout::Single); + assert_eq!(LayoutSpec::single().num_files(), 1); + assert_eq!( + LayoutSpec::single_shuffled().layout(), + TrainLayout::SingleShuffled + ); + assert_eq!( + LayoutSpec::partitioned(NonZeroU32::new(10).unwrap()).layout(), + TrainLayout::Partitioned + ); + assert_eq!( + LayoutSpec::partitioned_shuffled(NonZeroU32::new(10).unwrap()).layout(), + TrainLayout::PartitionedShuffled + ); + assert_eq!( + LayoutSpec::partitioned_shuffled(NonZeroU32::new(10).unwrap()).num_files(), + 10 + ); + } +} diff --git a/vortex-bench/src/vector_dataset/mod.rs b/vortex-bench/src/vector_dataset/mod.rs new file mode 100644 index 00000000000..fe1e42a68d0 --- /dev/null +++ b/vortex-bench/src/vector_dataset/mod.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Public catalog of VectorDBBench corpora used by the vector-search benchmark. +//! +//! The catalog is intentionally separate from the [`crate::datasets::Dataset`] trait used by the +//! row-table benchmarks: the train split of a vector dataset is sometimes partitioned across many +//! parquet files, sometimes single-file, sometimes shuffled, sometimes not, and its `emb` column +//! has to be rewrapped into an `Extension` before it's useful to a cosine-similarity scan. +//! None of that fits the row-table `Dataset` contract. +//! +//! The four sub-modules split the catalog into roughly orthogonal concerns: +//! +//! - `catalog`: the static [`VectorDataset`] enum + per-dataset metadata. +//! - `layout`: [`TrainLayout`] / [`LayoutSpec`] (the four hosted train shapes) and +//! [`VectorMetric`]. +//! - `download`: URL builders and the idempotent download driver. +//! - `paths`: local filesystem layout (`/vector-search/...`). +//! +//! Higher-level callers (the bench crate's ingest + scan pipeline) compose these: +//! [`download::download`] returns a [`download::DatasetPaths`] handle that the ingest pass turns +//! into per-flavor `.vortex` files, after which the scan driver re-opens those files per iteration. + +mod catalog; +mod download; +mod layout; +mod paths; + +pub use catalog::ALL_VECTOR_DATASETS; +pub use catalog::VectorDataset; +pub use download::DatasetPaths; +pub use download::download; +pub use download::neighbors_url; +pub use download::test_url; +pub use download::train_urls; +pub use layout::LayoutSpec; +pub use layout::TrainLayout; +pub use layout::VectorMetric; diff --git a/vortex-bench/src/vector_dataset/paths.rs b/vortex-bench/src/vector_dataset/paths.rs new file mode 100644 index 00000000000..3fbf9452ff7 --- /dev/null +++ b/vortex-bench/src/vector_dataset/paths.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Local-filesystem layout for cached vector benchmark datasets. +//! +//! ```text +//! /vector-search/// +//! train<> single-file: train.parquet +//! partitioned: 00-of-N.parquet, 01-of-N.parquet, ... +//! test.parquet +//! neighbors.parquet only when ds.has_neighbors() +//! +//! + some more things +//! ``` +//! +//! This module exists purely to centralize the path-construction logic used by both the downloader +//! and the ingest pipeline. + +use std::path::PathBuf; + +use crate::utils::file::data_dir; +use crate::vector_dataset::VectorDataset; +use crate::vector_dataset::layout::TrainLayout; + +/// Top-level cache root: `/vector-search/`. +pub fn root() -> PathBuf { + data_dir().join("vector-search") +} + +/// Per-dataset directory: `///`. +pub fn dataset_dir(ds: VectorDataset, layout: TrainLayout) -> PathBuf { + root().join(ds.name()).join(layout.label()) +} + +/// Train-shard directory: `/train/`. +pub fn train_dir(ds: VectorDataset, layout: TrainLayout) -> PathBuf { + dataset_dir(ds, layout).join("train") +} + +/// File name for one train shard within [`train_dir`]. +/// +/// Single-file layouts produce `train.parquet`; partitioned layouts produce `NN-of-MM.parquet` so a +/// directory listing sorts shards in sequence order. +pub fn train_file_name(layout: TrainLayout, shard_idx: u32, num_files: u32) -> String { + if layout.is_partitioned() { + format!( + "{shard_idx:0width$}-of-{num_files:0width$}.parquet", + width = num_files_width(num_files), + ) + } else { + "train.parquet".to_owned() + } +} + +/// All train-shard paths for a dataset/layout pair, in shard order. +pub fn train_files(ds: VectorDataset, layout: TrainLayout, num_files: u32) -> Vec { + let dir = train_dir(ds, layout); + (0..num_files) + .map(|i| dir.join(train_file_name(layout, i, num_files))) + .collect() +} + +/// Path to the cached `test.parquet` for a dataset/layout pair. +pub fn test_path(ds: VectorDataset, layout: TrainLayout) -> PathBuf { + dataset_dir(ds, layout).join("test.parquet") +} + +/// Path to the cached `neighbors.parquet` for a dataset/layout pair. +pub fn neighbors_path(ds: VectorDataset, layout: TrainLayout) -> PathBuf { + dataset_dir(ds, layout).join("neighbors.parquet") +} + +/// Width used to zero-pad shard indices in partitioned filenames. `10` shards is 2 digits, `100` +/// shards is 3 digits. +fn num_files_width(num_files: u32) -> usize { + let digits = num_files.checked_ilog10().unwrap_or(0) as usize + 1; + digits.max(2) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single_layout_uses_train_parquet() { + assert_eq!(train_file_name(TrainLayout::Single, 0, 1), "train.parquet"); + } + + #[test] + fn partitioned_filename_zero_pads_to_match_total() { + assert_eq!( + train_file_name(TrainLayout::Partitioned, 0, 10), + "00-of-10.parquet" + ); + assert_eq!( + train_file_name(TrainLayout::Partitioned, 9, 10), + "09-of-10.parquet" + ); + assert_eq!( + train_file_name(TrainLayout::Partitioned, 99, 100), + "099-of-100.parquet" + ); + } + + #[test] + fn train_files_lists_all_shards_in_order() { + let files = train_files(VectorDataset::CohereLarge10m, TrainLayout::Partitioned, 10); + assert_eq!(files.len(), 10); + for (i, path) in files.iter().enumerate() { + assert!( + path.to_string_lossy() + .ends_with(&format!("{i:02}-of-10.parquet")) + ); + } + } +} From 0d143dab5a08d060ae0d82fa58c1d100147d156a Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 15 Apr 2026 20:16:35 +0100 Subject: [PATCH 055/250] fix(ci): restore vortex-compact file size tracking for PR clickbench (#7452) ## Summary - The matrix consolidation in #7448 dropped `extra_data_formats: "vortex-compact"` from the clickbench default matrix entry - This meant PR clickbench no longer generates vortex-compact data, so file size comparisons in PR comments lost vortex-compact coverage - Restores the field so `data-gen` produces vortex-compact files for file size tracking without adding it as a benchmark target on PRs Signed-off-by: Alexander Droste --- .github/workflows/sql-benchmarks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 2abc1f9be70..2492e3d35e4 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -20,7 +20,8 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", - "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb" + "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb", + "extra_data_formats": "vortex-compact" }, { "id": "tpch-nvme", From 3188e2491d2ee5aea0e0511fb761278ad825313f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 15 Apr 2026 20:19:11 +0100 Subject: [PATCH 056/250] expose execution context for validity::to_bool and remove validity mask from array vtable (#7432) Signed-off-by: Joe Isaacs --- encodings/alp/benches/alp_compress.rs | 12 +- encodings/alp/public-api.lock | 2 +- encodings/alp/src/alp/array.rs | 64 ++++++- encodings/alp/src/alp/compress.rs | 180 +++++++++++++++--- encodings/alp/src/alp/compute/between.rs | 9 +- encodings/alp/src/alp/compute/cast.rs | 27 ++- encodings/alp/src/alp/compute/compare.rs | 58 +++++- encodings/alp/src/alp/compute/filter.rs | 9 +- encodings/alp/src/alp/compute/mask.rs | 21 +- encodings/alp/src/alp/compute/mod.rs | 26 +-- encodings/alp/src/alp/compute/take.rs | 9 +- encodings/alp/src/alp/plugin.rs | 5 +- encodings/alp/src/alp_rd/array.rs | 5 +- encodings/bytebool/public-api.lock | 4 - encodings/bytebool/src/array.rs | 5 - .../fastlanes/benches/compute_between.rs | 9 +- encodings/fastlanes/public-api.lock | 4 - .../src/bitpacking/array/bitpack_compress.rs | 24 ++- .../bitpacking/array/bitpack_decompress.rs | 2 +- .../fastlanes/src/bitpacking/array/mod.rs | 5 - .../fastlanes/src/bitpacking/compute/take.rs | 8 +- .../fastlanes/src/for/array/for_decompress.rs | 2 +- encodings/fastlanes/src/rle/array/mod.rs | 9 +- encodings/fsst/src/array.rs | 9 +- encodings/pco/src/array.rs | 5 +- encodings/pco/src/tests.rs | 13 +- .../runend/benches/run_end_null_count.rs | 10 +- encodings/runend/src/compress.rs | 12 +- encodings/runend/src/decompress_bool.rs | 45 ++++- encodings/sequence/src/compute/take.rs | 2 +- encodings/sparse/src/canonical.rs | 29 ++- encodings/sparse/src/lib.rs | 45 ++++- encodings/zstd/src/array.rs | 10 +- encodings/zstd/src/test.rs | 13 +- fuzz/fuzz_targets/file_io.rs | 5 +- fuzz/src/array/cast.rs | 23 ++- fuzz/src/array/compare.rs | 14 +- fuzz/src/array/filter.rs | 7 +- fuzz/src/array/search_sorted.rs | 29 ++- fuzz/src/array/slice.rs | 7 +- fuzz/src/array/sort.rs | 38 +++- fuzz/src/array/take.rs | 7 +- vortex-array/public-api.lock | 76 ++------ .../src/aggregate_fn/fns/count/mod.rs | 4 +- .../src/aggregate_fn/fns/first/mod.rs | 4 +- .../src/aggregate_fn/fns/is_constant/mod.rs | 4 +- .../src/aggregate_fn/fns/is_sorted/bool.rs | 13 +- .../src/aggregate_fn/fns/is_sorted/decimal.rs | 21 +- .../src/aggregate_fn/fns/is_sorted/mod.rs | 8 +- .../aggregate_fn/fns/is_sorted/primitive.rs | 23 ++- vortex-array/src/aggregate_fn/fns/last/mod.rs | 4 +- .../src/aggregate_fn/fns/min_max/bool.rs | 12 +- .../src/aggregate_fn/fns/min_max/decimal.rs | 39 ++-- .../src/aggregate_fn/fns/min_max/mod.rs | 8 +- .../src/aggregate_fn/fns/min_max/primitive.rs | 37 ++-- .../src/aggregate_fn/fns/nan_count/mod.rs | 6 +- .../aggregate_fn/fns/nan_count/primitive.rs | 12 +- vortex-array/src/aggregate_fn/fns/sum/bool.rs | 9 +- .../src/aggregate_fn/fns/sum/decimal.rs | 9 +- vortex-array/src/aggregate_fn/fns/sum/mod.rs | 8 +- .../src/aggregate_fn/fns/sum/primitive.rs | 9 +- vortex-array/src/array/erased.rs | 20 +- vortex-array/src/array/typed.rs | 12 +- vortex-array/src/arrays/bool/array.rs | 12 +- vortex-array/src/arrays/bool/compute/take.rs | 2 +- vortex-array/src/arrays/bool/test_harness.rs | 20 +- .../src/arrays/chunked/compute/take.rs | 14 +- .../src/arrays/constant/compute/take.rs | 28 ++- .../src/arrays/constant/vtable/canonical.rs | 9 +- .../src/arrays/decimal/compute/cast.rs | 12 +- .../src/arrays/decimal/compute/fill_null.rs | 11 +- vortex-array/src/arrays/dict/array.rs | 47 ++++- vortex-array/src/arrays/filter/execute/mod.rs | 10 +- .../src/arrays/fixed_size_list/array.rs | 4 - .../arrays/fixed_size_list/compute/take.rs | 22 ++- vortex-array/src/arrays/list/array.rs | 4 - vortex-array/src/arrays/list/compute/take.rs | 14 +- vortex-array/src/arrays/listview/array.rs | 4 - vortex-array/src/arrays/masked/array.rs | 4 - vortex-array/src/arrays/masked/tests.rs | 10 +- .../src/arrays/masked/vtable/canonical.rs | 13 +- vortex-array/src/arrays/masked/vtable/mod.rs | 2 +- vortex-array/src/arrays/null/compute/mod.rs | 16 +- .../src/arrays/primitive/array/mod.rs | 4 - .../src/arrays/primitive/array/top_value.rs | 10 +- .../src/arrays/primitive/compute/cast.rs | 10 +- .../src/arrays/primitive/compute/fill_null.rs | 38 +++- vortex-array/src/arrays/varbin/array.rs | 5 - .../src/arrays/varbin/compute/compare.rs | 24 ++- .../src/arrays/varbin/compute/filter.rs | 5 +- .../src/arrays/varbin/compute/take.rs | 7 +- vortex-array/src/arrays/varbinview/array.rs | 5 - vortex-array/src/arrays/varbinview/compact.rs | 7 +- .../src/arrays/varbinview/compute/take.rs | 5 +- .../src/arrays/varbinview/compute/zip.rs | 4 +- vortex-array/src/arrow/executor/bool.rs | 14 +- vortex-array/src/arrow/executor/byte_view.rs | 10 +- vortex-array/src/arrow/executor/decimal.rs | 44 +++-- vortex-array/src/arrow/executor/primitive.rs | 8 +- vortex-array/src/arrow/executor/temporal.rs | 5 +- vortex-array/src/builders/bool.rs | 15 +- vortex-array/src/builders/decimal.rs | 15 +- vortex-array/src/builders/fixed_size_list.rs | 8 +- vortex-array/src/builders/list.rs | 8 +- vortex-array/src/builders/listview.rs | 11 +- vortex-array/src/builders/primitive.rs | 15 +- vortex-array/src/builders/struct_.rs | 20 +- vortex-array/src/builders/varbinview.rs | 25 ++- vortex-array/src/compute/conformance/cast.rs | 12 +- .../src/compute/conformance/consistency.rs | 4 +- vortex-array/src/compute/conformance/mask.rs | 5 +- vortex-array/src/mask.rs | 5 +- vortex-array/src/patches.rs | 30 ++- vortex-array/src/scalar_fn/fns/case_when.rs | 2 +- vortex-array/src/scalar_fn/fns/zip/mod.rs | 2 +- vortex-array/src/stats/array.rs | 6 +- vortex-array/src/test_harness.rs | 7 +- vortex-array/src/validity.rs | 63 +++--- vortex-array/src/variants.rs | 4 +- vortex-btrblocks/src/schemes/float.rs | 6 +- vortex-compressor/src/stats/bool.rs | 7 +- vortex-compressor/src/stats/float.rs | 7 +- vortex-compressor/src/stats/integer.rs | 7 +- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 9 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 8 +- vortex-cuda/src/kernel/encodings/alp.rs | 14 +- vortex-duckdb/src/convert/vector.rs | 42 +++- vortex-duckdb/src/exporter/decimal.rs | 6 +- vortex-duckdb/src/exporter/dict.rs | 9 +- vortex-ffi/src/array.rs | 4 +- vortex-jni/src/array.rs | 6 +- vortex-layout/src/layouts/dict/reader.rs | 9 +- vortex-layout/src/layouts/flat/writer.rs | 9 +- vortex-layout/src/layouts/table.rs | 7 +- vortex-python/src/arrays/mod.rs | 6 +- .../arrays/synthetic/encodings/alp.rs | 79 ++++++-- .../common_encoding_tree_throughput.rs | 9 +- vortex/benches/single_encoding_throughput.rs | 17 +- 138 files changed, 1643 insertions(+), 498 deletions(-) diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index c8745926845..8817daedbbc 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -66,9 +66,14 @@ fn compress_alp(bencher: Bencher, args: (usize, f64, let values = values.freeze(); let array = PrimitiveArray::new(values, validity); - bencher - .with_inputs(|| &array) - .bench_values(|array| alp_encode(array.as_view(), None).unwrap()) + bencher.with_inputs(|| &array).bench_values(|array| { + alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + }) } #[divan::bench(types = [f32, f64], args = BENCH_ARGS)] @@ -95,6 +100,7 @@ fn decompress_alp(bencher: Bencher, args: (usize, f64 alp_encode( PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()).as_view(), None, + &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(), LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index ff4535970be..a2fa68e3cfa 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -636,7 +636,7 @@ pub fn f64::to_bits(value: Self) -> Self::UINT pub fn f64::to_u16(bits: Self::UINT) -> u16 -pub fn vortex_alp::alp_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, exponents: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_alp::alp_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, exponents: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::alp_rd_decode(left_parts: vortex_buffer::buffer_mut::BufferMut, left_parts_dict: &[u16], right_bit_width: u8, right_parts: vortex_buffer::buffer_mut::BufferMut<::UINT>, left_parts_patches: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index be0db0f938e..8a86019dcf9 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -552,11 +552,11 @@ mod tests { #[case(2048)] #[case(2049)] fn test_execute_f32(#[case] size: usize) { + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter((0..size).map(|i| i as f32)); - let encoded = alp_encode(values.as_view(), None).unwrap(); + let encoded = alp_encode(values.as_view(), None, &mut ctx).unwrap(); let result_canonical = { - let mut ctx = SESSION.create_execution_ctx(); encoded .clone() .into_array() @@ -582,7 +582,12 @@ mod tests { #[case(2049)] fn test_execute_f64(#[case] size: usize) { let values = PrimitiveArray::from_iter((0..size).map(|i| i as f64)); - let encoded = alp_encode(values.as_view(), None).unwrap(); + let encoded = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -616,7 +621,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -650,7 +660,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -685,7 +700,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -721,7 +741,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -772,7 +797,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -783,7 +813,16 @@ mod tests { for idx in 0..slice_len { let expected_value = values[slice_start + idx]; - let result_valid = result_primitive.validity_mask().unwrap().value(idx); + let result_valid = result_primitive + .as_ref() + .validity() + .unwrap() + .to_mask( + result_primitive.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .value(idx); assert_eq!( result_valid, expected_value.is_some(), @@ -813,7 +852,12 @@ mod tests { let original = PrimitiveArray::from_iter(values); // First encode normally to get a properly formed ALPArray with patches. - let normally_encoded = alp_encode(original.as_view(), None).unwrap(); + let normally_encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!( normally_encoded.patches().is_some(), "Test requires patches to be present" diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index f67779996da..059c4b8821c 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -44,10 +45,11 @@ macro_rules! match_each_alp_float_ptype { pub fn alp_encode( parray: ArrayView<'_, Primitive>, exponents: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult { let (exponents, encoded, patches) = match parray.ptype() { - PType::F32 => alp_encode_components_typed::(parray, exponents)?, - PType::F64 => alp_encode_components_typed::(parray, exponents)?, + PType::F32 => alp_encode_components_typed::(parray, exponents, ctx)?, + PType::F64 => alp_encode_components_typed::(parray, exponents, ctx)?, _ => vortex_bail!("ALP can only encode f32 and f64"), }; @@ -62,6 +64,7 @@ pub fn alp_encode( fn alp_encode_components_typed( values: ArrayView<'_, Primitive>, exponents: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult<(Exponents, ArrayRef, Option)> where T: ALPFloat, @@ -73,7 +76,10 @@ where let encoded_array = PrimitiveArray::new(encoded, values.validity()?).into_array(); - let validity = values.array().validity_mask()?; + let validity = values + .array() + .validity()? + .to_mask(values.array().len(), ctx)?; // exceptional_positions may contain exceptions at invalid positions (which contain garbage // data). We remove null exceptions in order to keep the Patches small. let (valid_exceptional_positions, valid_exceptional_values): (Buffer, Buffer) = @@ -146,7 +152,12 @@ mod tests { #[test] fn test_compress() { let array = PrimitiveArray::new(buffer![1.234f32; 1025], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i32; 1025]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -160,7 +171,12 @@ mod tests { #[test] fn test_nullable_compress() { let array = PrimitiveArray::from_option_iter([None, Some(1.234f32), None]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter([None, Some(1234i32), None]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -177,7 +193,12 @@ mod tests { fn test_patched_compress() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i64, 2718, 1234, 4000]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -194,7 +215,12 @@ mod tests { fn test_compress_ignores_invalid_exceptional_values() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values, Validity::from_iter([true, true, false, true])); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter(buffer![Some(1234i64), Some(2718), None, Some(4000)]); @@ -216,7 +242,12 @@ mod tests { Some(4.0), None, ]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); assert_eq!(encoded.exponents(), Exponents { e: 16, f: 13 }); @@ -230,7 +261,12 @@ mod tests { #[test] fn roundtrips_close_fractional() { let original = PrimitiveArray::from_iter([195.26274f32, 195.27837, -48.815685]); - let alp_arr = alp_encode(original.as_view(), None).unwrap(); + let alp_arr = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert_arrays_eq!(alp_arr, original); } @@ -238,7 +274,12 @@ mod tests { fn roundtrips_all_null() { let original = PrimitiveArray::new(buffer![195.26274f64, PI, -48.815685], Validity::AllInvalid); - let alp_arr = alp_encode(original.as_view(), None).unwrap(); + let alp_arr = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decompressed = alp_arr.into_array().to_primitive(); assert_eq!( @@ -256,7 +297,12 @@ mod tests { buffer![0.0f32, -0.0, f32::NAN, f32::NEG_INFINITY, f32::INFINITY], Validity::NonNullable, ); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = encoded.as_array().to_primitive(); for idx in 0..original.len() { let decoded_val = decoded.as_slice::()[idx]; @@ -277,7 +323,12 @@ mod tests { values[1025] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -300,7 +351,12 @@ mod tests { values[2048] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -322,7 +378,12 @@ mod tests { values[0] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -345,7 +406,12 @@ mod tests { values[100] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -366,7 +432,12 @@ mod tests { // Create 1024 elements, encode, slice to first 512, then decode let values = vec![1.234f32; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -378,7 +449,12 @@ mod tests { fn test_slice_half_chunk_f64_roundtrip() { let values = vec![5.678f64; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -394,7 +470,12 @@ mod tests { values[600] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -414,7 +495,12 @@ mod tests { values[1023] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(1023..1025).unwrap(); @@ -430,7 +516,12 @@ mod tests { .collect::>(); let original = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); let decoded = sliced_alp.to_primitive(); @@ -443,7 +534,12 @@ mod tests { fn test_large_f32_array_uniform_values() { let size = 10_000; let array = PrimitiveArray::new(buffer![42.125f32; size], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -455,7 +551,12 @@ mod tests { fn test_large_f64_array_uniform_values() { let size = 50_000; let array = PrimitiveArray::new(buffer![123.456789f64; size], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -473,7 +574,12 @@ mod tests { values[4500] = f32::INFINITY; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -495,7 +601,12 @@ mod tests { values[7000] = 999.999999999; let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -525,7 +636,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -546,7 +662,12 @@ mod tests { let validity = Validity::from_iter((0..size).map(|i| !matches!(i, 500 | 2500))); let array = PrimitiveArray::new(Buffer::from(values), validity); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -577,7 +698,12 @@ mod tests { values[2900] = PI; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); // Slice ending mid-chunk-2 (element 2500 is inside chunk 2 = 2048..3072). diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index b9281140c1d..7f35f67d51c 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -95,6 +95,8 @@ where #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -123,7 +125,12 @@ mod tests { fn comparison_range() { let value = 0.0605_f32; let array = PrimitiveArray::from_iter([value; 1]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_between( diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index b663b861c79..b6ce03b9eee 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -60,7 +60,9 @@ impl CastReduce for ALP { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -78,7 +80,11 @@ mod tests { #[test] fn issue_5766_test_cast_alp_with_patches_to_nullable() -> VortexResult<()> { let values = buffer![1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!( alp.patches().is_some(), @@ -98,7 +104,11 @@ mod tests { #[test] fn test_cast_alp_f32_to_f64() -> VortexResult<()> { let values = buffer![1.5f32, 2.5, 3.5, 4.5].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let casted = alp .into_array() @@ -120,7 +130,11 @@ mod tests { #[test] fn test_cast_alp_to_int() -> VortexResult<()> { let values = buffer![1.0f32, 2.0, 3.0, 4.0].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let casted = alp .into_array() @@ -143,7 +157,12 @@ mod tests { #[case(buffer![42.42f64].into_array())] #[case(buffer![0.0f32, -1.5, 2.5, -3.5, 4.5].into_array())] fn test_cast_alp_conformance(#[case] array: vortex_array::ArrayRef) -> VortexResult<()> { - let alp = alp_encode(array.to_primitive().as_view(), None).vortex_expect("cannot fail"); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("cannot fail"); test_cast_conformance(&alp.into_array()); Ok(()) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index a6f475a3df2..ebda0a7797c 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -152,7 +152,9 @@ where mod tests { use rstest::rstest; use vortex_array::ArrayRef; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -183,7 +185,12 @@ mod tests { #[test] fn basic_comparison_test() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -206,7 +213,12 @@ mod tests { #[test] fn comparison_with_unencodable_value() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -229,7 +241,12 @@ mod tests { #[test] fn comparison_range() { let array = PrimitiveArray::from_iter([0.0605_f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -268,7 +285,12 @@ mod tests { #[test] fn comparison_zeroes() { let array = PrimitiveArray::from_iter([0.0_f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -310,7 +332,12 @@ mod tests { fn compare_with_patches() { let array = PrimitiveArray::from_iter([1.234f32, 1.5, 19.0, std::f32::consts::E, 1_000_000.9]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); // Not supported! @@ -324,7 +351,12 @@ mod tests { #[test] fn compare_to_null() { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let other = ConstantArray::new( Scalar::null(DType::Primitive(PType::F32, Nullability::Nullable)), @@ -347,7 +379,12 @@ mod tests { #[case(f32::NEG_INFINITY, true)] fn compare_to_non_finite_gt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Gt).unwrap(); let expected = BoolArray::from_iter([result; 10]); @@ -361,7 +398,12 @@ mod tests { #[case(f32::NEG_INFINITY, false)] fn compare_to_non_finite_lt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Lt).unwrap(); let expected = BoolArray::from_iter([result; 10]); diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 622a8d59b6b..c305ea5e99b 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -44,7 +44,9 @@ mod test { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::filter::test_filter_conformance; use vortex_buffer::buffer; @@ -61,7 +63,12 @@ mod test { 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 ].into_array())] fn test_filter_alp_conformance(#[case] array: ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_filter_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 137aa263f53..7b9757376d9 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -79,7 +79,12 @@ mod test { 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0 ].into_array())] fn test_mask_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_mask_conformance(&alp.into_array()); } @@ -91,7 +96,12 @@ mod test { .map(|i| if i % 4 == 3 { PI } else { 1.0 }) .collect(); let array = PrimitiveArray::from_iter(values); - let alp = alp_encode(array.as_view(), None).unwrap(); + let alp = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(alp.patches().is_some(), "expected patches"); test_mask_conformance(&alp.into_array()); } @@ -99,7 +109,12 @@ mod test { #[test] fn test_mask_alp_with_patches_casts_surviving_patch_values_to_nullable() { let values = PrimitiveArray::from_iter([1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456]); - let alp = alp_encode(values.as_view(), None).unwrap(); + let alp = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(alp.patches().is_some(), "expected patches"); let keep_mask = BoolArray::from_iter([false, true, true, true, true]).into_array(); diff --git a/encodings/alp/src/alp/compute/mod.rs b/encodings/alp/src/alp/compute/mod.rs index 14e118a870a..e5e5b657740 100644 --- a/encodings/alp/src/alp/compute/mod.rs +++ b/encodings/alp/src/alp/compute/mod.rs @@ -14,6 +14,8 @@ mod take; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -23,27 +25,27 @@ mod tests { #[rstest] // Basic float arrays - #[case::f32_array(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] - #[case::f64_array(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] + #[case::f32_array(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_array(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Nullable arrays - #[case::nullable_f32(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] - #[case::nullable_f64(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] + #[case::nullable_f32(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::nullable_f64(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Edge cases - #[case::single_element(alp_encode(PrimitiveArray::from_iter([42.42f64]).as_view(), None).unwrap())] + #[case::single_element(alp_encode(PrimitiveArray::from_iter([42.42f64]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Large arrays - #[case::large_f32(alp_encode(PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)).as_view(), None).unwrap())] + #[case::large_f32(alp_encode(PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Arrays with patterns - #[case::repeating_pattern(alp_encode(PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]).as_view(), None).unwrap())] - #[case::close_values(alp_encode(PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]).as_view(), None).unwrap())] + #[case::repeating_pattern(alp_encode(PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::close_values(alp_encode(PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_alp_consistency(#[case] array: ALPArray) { test_array_consistency(&array.into_array()); } #[rstest] - #[case::f32_basic(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] - #[case::f64_basic(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] - #[case::f32_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)).as_view(), None).unwrap())] - #[case::f64_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)).as_view(), None).unwrap())] + #[case::f32_basic(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_basic(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f32_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_alp_binary_numeric(#[case] array: ALPArray) { test_binary_numeric_array(array.into_array()); } diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index f3a7d23fbe4..3fcfa7c9f77 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -42,7 +42,9 @@ impl TakeExecute for ALP { mod test { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::take::test_take_conformance; use vortex_buffer::buffer; @@ -55,7 +57,12 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).into_array())] #[case(buffer![42.42f64].into_array())] fn test_take_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_take_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 2ac3ac85c54..25266bef821 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -93,6 +93,7 @@ mod tests { use vortex_array::ArrayPlugin; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PatchedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::PatchedArraySlotsExt; @@ -124,7 +125,7 @@ mod tests { .collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(parray.as_view(), None)?; + let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?; assert!( alp_encoded.patches().is_some(), @@ -173,7 +174,7 @@ mod tests { // Values that encode cleanly without patches. let values: Vec = (0..100).map(|i| i as f64).collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(parray.as_view(), None)?; + let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?; assert!( alp_encoded.patches().is_none(), diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index be5f5f9a918..0cdac5c4001 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -261,7 +261,10 @@ impl VTable for ALPRD { // Decode the left_parts using our builtin dictionary. let left_parts_dict = left_parts_dictionary; - let validity = left_parts.validity_mask()?; + let validity = left_parts + .as_ref() + .validity()? + .to_mask(left_parts.as_ref().len(), ctx)?; let decoded_array = if ptype == PType::F32 { PrimitiveArray::new( diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 514f8742a28..8edd17d2912 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -122,12 +122,8 @@ pub trait vortex_bytebool::ByteBoolArrayExt: vortex_array::array::typed::TypedAr pub fn vortex_bytebool::ByteBoolArrayExt::validity(&self) -> vortex_array::validity::Validity -pub fn vortex_bytebool::ByteBoolArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_bytebool::ByteBoolArrayExt for T pub fn T::validity(&self) -> vortex_array::validity::Validity -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub type vortex_bytebool::ByteBoolArray = vortex_array::array::typed::Array diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index e98c5bd656d..42cc48c094e 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -35,7 +35,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -190,10 +189,6 @@ pub trait ByteBoolArrayExt: TypedArrayRef { self.as_ref().dtype().nullability(), ) } - - fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.as_ref().len()) - } } impl> ByteBoolArrayExt for T {} diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index 5f232f61196..49f518a7949 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -11,7 +11,9 @@ use vortex_alp::ALPArraySlotsExt; use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_error::VortexExpect; @@ -49,7 +51,12 @@ fn generate_alp_bit_pack_primitive_array( .map(|_| T::from_usize(rng.random_range(0..10_000)).vortex_expect("")) .collect::(); - let alp = alp_encode(a.as_view(), None).vortex_expect(""); + let alp = alp_encode( + a.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect(""); let encoded = alp.encoded().to_primitive(); diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index cf7776cd872..0a84e217c76 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -578,8 +578,6 @@ pub fn vortex_fastlanes::BitPackedArrayExt::unpacked_chunks vortex_array::validity::Validity -pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_fastlanes::BitPackedArrayExt for T pub fn T::bit_width(&self) -> u8 @@ -596,8 +594,6 @@ pub fn T::unpacked_chunks(&self) -> pub fn T::validity(&self) -> vortex_array::validity::Validity -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub trait vortex_fastlanes::BitPackedArraySlotsExt: vortex_array::array::typed::TypedArrayRef pub fn vortex_fastlanes::BitPackedArraySlotsExt::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index b2b6ce8648c..7c7e1bc5d26 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -6,6 +6,8 @@ use itertools::Itertools; use num_traits::PrimInt; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -200,7 +202,10 @@ pub fn gather_patches( }; let array_len = parray.len(); - let validity_mask = parray.validity_mask()?; + let validity_mask = parray + .as_ref() + .validity()? + .to_mask(parray.len(), &mut LEGACY_SESSION.create_execution_ctx())?; let patches = if array_len < u8::MAX as usize { match_each_integer_ptype!(parray.ptype(), |T| { @@ -302,7 +307,14 @@ fn bit_width_histogram_typed( |v: T| (8 * size_of::()) - (PrimInt::leading_zeros(v) as usize); let mut bit_widths = vec![0usize; size_of::() * 8 + 1]; - match array.validity_mask().bit_buffer() { + match array + .validity()? + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .bit_buffer() + { AllOr::All => { // All values are valid. for v in array.as_slice::() { @@ -457,7 +469,13 @@ mod test { assert_eq!( (0..(1 << 4)).collect::>(), compressed - .validity_mask() + .as_ref() + .validity() + .unwrap() + .to_mask( + compressed.as_ref().len(), + &mut SESSION.create_execution_ctx() + ) .unwrap() .to_bit_buffer() .set_indices() diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 5ed8d6b89fa..465c467bcfb 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -58,7 +58,7 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: We later initialize the the uninitialized range of values with `copy_from_slice`. unsafe { // Append a dense null Mask. - uninit_range.append_mask(array.validity_mask()); + uninit_range.append_mask(array.validity()?.to_mask(array.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index ca86e78e148..dbca8c0e48b 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -317,11 +317,6 @@ pub trait BitPackedArrayExt: BitPackedArraySlotsExt { ) } - #[inline] - fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.as_ref().len()) - } - #[inline] fn packed_slice(&self) -> &[T] { BitPackedData::packed_slice::(self) diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 5a3177df6c1..5eb03415c88 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -280,7 +280,13 @@ mod test { taken_primitive, PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(4)]) ); - assert_eq!(taken_primitive.to_primitive().invalid_count().unwrap(), 1); + assert_eq!( + taken_primitive + .to_primitive() + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1 + ); } #[rstest] diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index e19ef8d0ff3..070bef28d32 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -109,7 +109,7 @@ pub(crate) fn fused_decompress< let mut uninit_range = builder.uninit_range(bp.len()); unsafe { // Append a dense null Mask. - uninit_range.append_mask(bp.validity_mask()); + uninit_range.append_mask(bp.validity()?.to_mask(bp.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index a44ae69ddbd..9857874d040 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -300,7 +300,14 @@ mod tests { .vortex_expect("RLEData is always valid"); let sliced_array = rle_array.slice(1..4).unwrap(); - let validity_mask = sliced_array.validity_mask().unwrap(); + let validity_mask = sliced_array + .validity() + .unwrap() + .to_mask( + sliced_array.len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let expected_mask = Validity::from_iter([false, true, false]) diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index d3a51a254e9..ed605f47818 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -288,7 +288,14 @@ impl VTable for FSST { let next_buffer_index = builder.completed_block_count() + u32::from(builder.in_progress()); let (buffers, views) = fsst_decode_views(array, next_buffer_index, ctx)?; - builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); + builder.push_buffer_and_adjusted_views( + &buffers, + &views, + array + .array() + .validity()? + .to_mask(array.array().len(), ctx)?, + ); Ok(()) } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 7c8e74c825c..904c6e4dc13 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -258,7 +258,10 @@ pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { } fn collect_valid(parray: ArrayView<'_, Primitive>) -> VortexResult { - let mask = parray.array().validity_mask()?; + let mask = parray.array().validity()?.to_mask( + parray.array().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(parray.array().filter(mask)?.to_primitive()) } diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index d694a0efdbe..b21bf121afc 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -164,12 +164,21 @@ fn test_validity_vtable() { Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); + let arr = compressed.as_array(); assert_eq!( - compressed.as_array().validity_mask().unwrap(), + arr.validity() + .unwrap() + .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(mask_bools) ); + let sliced = compressed.slice(1..4).unwrap(); assert_eq!( - compressed.slice(1..4).unwrap().validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, true, false]) ); } diff --git a/encodings/runend/benches/run_end_null_count.rs b/encodings/runend/benches/run_end_null_count.rs index f1473cc47aa..6fa318c7192 100644 --- a/encodings/runend/benches/run_end_null_count.rs +++ b/encodings/runend/benches/run_end_null_count.rs @@ -8,6 +8,8 @@ use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_buffer::Buffer; use vortex_runend::RunEnd; @@ -49,9 +51,11 @@ const BENCH_ARGS: &[(usize, usize, f64)] = &[ fn null_count_run_end(bencher: Bencher, (n, run_step, valid_density): (usize, usize, f64)) { let array = fixture(n, run_step, valid_density).into_array(); - bencher - .with_inputs(|| &array) - .bench_refs(|array| array.invalid_count().unwrap()); + bencher.with_inputs(|| &array).bench_refs(|array| { + array + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + }); } fn fixture(n: usize, run_step: usize, valid_density: f64) -> RunEndArray { diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index df86d6cc5fb..65739507fd9 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -5,7 +5,9 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; @@ -177,7 +179,10 @@ pub fn runend_decode_primitive( offset: usize, length: usize, ) -> VortexResult { - let validity_mask = values.validity_mask()?; + let validity_mask = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(match_each_native_ptype!(values.ptype(), |P| { match_each_unsigned_integer_ptype!(ends.ptype(), |E| { runend_decode_typed_primitive( @@ -277,7 +282,10 @@ pub fn runend_decode_varbinview( offset: usize, length: usize, ) -> VortexResult { - let validity_mask = values.validity_mask()?; + let validity_mask = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let views = values.views(); let (decoded_views, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| { diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index 6733f143a8c..69876b0a273 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -9,6 +9,8 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -36,7 +38,10 @@ pub fn runend_decode_bools( offset: usize, length: usize, ) -> VortexResult { - let validity = values.validity_mask()?; + let validity = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let values_buf = values.to_bit_buffer(); let nullability = values.dtype().nullability(); @@ -241,7 +246,9 @@ fn decode_nullable_sequential( #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -371,12 +378,42 @@ mod tests { // Check length and a few values assert_eq!(decoded.len(), 10000); // First run: valid true - assert!(decoded.validity_mask()?.value(0)); + assert!( + decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(0) + ); assert!(decoded.to_bit_buffer().value(0)); // Second run: null (validity false) - assert!(!decoded.validity_mask()?.value(2000)); + assert!( + !decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(2000) + ); // Third run: valid true - assert!(decoded.validity_mask()?.value(4000)); + assert!( + decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(4000) + ); assert!(decoded.to_bit_buffer().value(4000)); Ok(()) } diff --git a/encodings/sequence/src/compute/take.rs b/encodings/sequence/src/compute/take.rs index ab3a4ce5f97..7610b1dd038 100644 --- a/encodings/sequence/src/compute/take.rs +++ b/encodings/sequence/src/compute/take.rs @@ -77,7 +77,7 @@ impl TakeExecute for Sequence { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mask = indices.validity_mask()?; + let mask = indices.validity()?.to_mask(indices.len(), ctx)?; let indices = indices.clone().execute::(ctx)?; let result_nullability = array.dtype().nullability() | indices.dtype().nullability(); diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index c53fd8989e8..04ba70398f6 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -148,7 +148,10 @@ fn execute_sparse_lists( let n_filled = array.len() - resolved_patches.num_patches(); let total_canonical_values = values.elements().len() + fill_value.len() * n_filled; - let validity = Validity::from_mask(array.as_array().validity_mask()?, nullability); + let validity = { + let arr = array.as_array(); + Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { match_smallest_offset_type!(total_canonical_values, |O| { @@ -233,7 +236,10 @@ fn execute_sparse_fixed_size_list( .execute::(ctx)?; let fill_value = array.fill_scalar().as_list(); - let validity = Validity::from_mask(array.as_array().validity_mask()?, nullability); + let validity = { + let arr = array.as_array(); + Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { execute_sparse_fixed_size_list_inner::( @@ -420,10 +426,13 @@ fn execute_sparse_struct( unresolved_patches.offset(), unresolved_patches.indices(), &Validity::from_mask( - unresolved_patches - .values() - .validity_mask() - .vortex_expect("validity_mask"), + { + let v = unresolved_patches.values(); + v.validity() + .vortex_expect("validity_mask") + .to_mask(v.len(), ctx) + .vortex_expect("Failed to compute validity mask") + }, Nullability::Nullable, ), ctx, @@ -490,7 +499,13 @@ fn execute_varbin( let patches = array.resolved_patches()?; let indices = patches.indices().clone().execute::(ctx)?; let values = patches.values().clone().execute::(ctx)?; - let validity = Validity::from_mask(array.as_array().validity_mask()?, dtype.nullability()); + let validity = { + let arr = array.as_array(); + Validity::from_mask( + arr.validity()?.to_mask(arr.len(), ctx)?, + dtype.nullability(), + ) + }; let len = array.len(); Ok(match_each_integer_ptype!(indices.ptype(), |I| { diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 845d918a7a3..736f173e894 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -19,8 +19,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::Precision; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::buffer::BufferHandle; @@ -402,7 +404,9 @@ impl SparseData { fill_value.dtype() ) } - let mask = array.validity_mask()?; + let mask = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?; if mask.all_false() { // Array is constant NULL @@ -512,6 +516,8 @@ impl ValidityVTable for Sparse { mod test { use itertools::Itertools; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -592,7 +598,11 @@ mod test { pub fn validity_mask_sliced_null_fill() { let sliced = sparse_array(nullable_fill()).slice(2..7).unwrap(); assert_eq!( - sliced.validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, false, false, true, false]) ); } @@ -614,7 +624,11 @@ mod test { .unwrap(); assert_eq!( - sliced.validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![false, true, true, false, true]) ); } @@ -639,7 +653,9 @@ mod test { let array = sparse_array(nullable_fill()); assert_eq!( array - .validity_mask() + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .to_bit_buffer() .iter() @@ -653,7 +669,14 @@ mod test { #[test] fn sparse_validity_mask_non_null_fill() { let array = sparse_array(non_nullable_fill()); - assert!(array.validity_mask().unwrap().all_true()); + assert!( + array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -684,7 +707,11 @@ mod test { let sparse = Sparse::encode(&original.clone().into_array(), None) .vortex_expect("Sparse::encode should succeed for test data"); assert_eq!( - sparse.validity_mask().unwrap(), + sparse + .validity() + .unwrap() + .to_mask(sparse.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![ true, true, false, true, false, true, false, true, true, false, true, false, ]) @@ -698,7 +725,11 @@ mod test { let values = PrimitiveArray::from_option_iter([Some(0i16), Some(1), None, None, Some(4)]) .into_array(); let array = Sparse::try_new(indices, values, 10, Scalar::null_native::()).unwrap(); - let actual = array.validity_mask().unwrap(); + let actual = array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let expected = Mask::from_iter([ true, false, true, false, false, false, false, false, true, false, ]); diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 789a0ae91ad..9f3d19e1bd2 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -366,12 +366,18 @@ fn choose_max_dict_size(uncompressed_size: usize) -> usize { } fn collect_valid_primitive(parray: &PrimitiveArray) -> VortexResult { - let mask = parray.validity_mask()?; + let mask = parray.as_ref().validity()?.to_mask( + parray.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(parray.filter(mask)?.to_primitive()) } fn collect_valid_vbv(vbv: &VarBinViewArray) -> VortexResult<(ByteBuffer, Vec)> { - let mask = vbv.validity_mask()?; + let mask = vbv.as_ref().validity()?.to_mask( + vbv.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let buffer_and_value_byte_indices = match mask.bit_buffer() { AllOr::None => (Buffer::empty(), Vec::new()), _ => { diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 76a8909645e..662cb434d87 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -144,12 +144,21 @@ fn test_validity_vtable() { Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); let compressed = Zstd::from_primitive(&array, 3, 0).unwrap(); + let arr = compressed.as_array(); assert_eq!( - compressed.as_array().validity_mask().unwrap(), + arr.validity() + .unwrap() + .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(mask_bools) ); + let sliced = compressed.slice(1..4).unwrap(); assert_eq!( - compressed.slice(1..4).unwrap().validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, true, false]) ); } diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 92e45c5b0c0..f6917a0e9f2 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -9,6 +9,7 @@ use libfuzzer_sys::fuzz_target; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::builtins::ArrayBuiltins; @@ -49,7 +50,9 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { .clone() .apply(&filter_expr.clone().unwrap_or_else(|| lit(true))) .vortex_expect("filter expression evaluation should succeed in fuzz test"); - let mask = bool_mask.to_bool().to_mask_fill_null_false(); + let mask = bool_mask + .to_bool() + .to_mask_fill_null_false(&mut SESSION.create_execution_ctx()); let filtered = array_data .filter(mask) .vortex_expect("filter operation should succeed in fuzz test"); diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index f9ddba1fd31..503287c2dcf 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::Nullable; @@ -39,7 +41,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array() }) @@ -65,7 +72,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array(), )), @@ -80,7 +92,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array(), )) diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index 849a7fdd694..b4c714de335 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -45,8 +47,10 @@ pub fn compare_canonical_array( .iter() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) @@ -70,8 +74,10 @@ pub fn compare_canonical_array( .copied() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) @@ -98,8 +104,10 @@ pub fn compare_canonical_array( .copied() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index fbcc2856e61..9e5b90c7c05 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -24,7 +26,10 @@ use crate::array::take_canonical_array_non_nullable_indices; pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult { let validity = if array.dtype().is_nullable() { - let validity_buff = array.validity_mask()?.to_bit_buffer(); + let validity_buff = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from_iter( filter .iter() diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index d22b583a39b..593be73c598 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -5,7 +5,9 @@ use std::cmp::Ordering; use std::fmt::Debug; use vortex_array::ArrayRef; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; @@ -64,7 +66,14 @@ pub fn search_sorted_canonical_array( match array.dtype() { DType::Bool(_) => { let bool_array = array.to_bool(); - let validity = bool_array.validity_mask()?.to_bit_buffer(); + let validity = bool_array + .as_ref() + .validity()? + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); let opt_values = bool_array .to_bit_buffer() .iter() @@ -76,7 +85,14 @@ pub fn search_sorted_canonical_array( } DType::Primitive(p, _) => { let primitive_array = array.to_primitive(); - let validity = primitive_array.validity_mask()?.to_bit_buffer(); + let validity = primitive_array + .as_ref() + .validity()? + .to_mask( + primitive_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); match_each_native_ptype!(p, |P| { let opt_values = primitive_array .as_slice::

() @@ -91,7 +107,14 @@ pub fn search_sorted_canonical_array( } DType::Decimal(d, _) => { let decimal_array = array.to_decimal(); - let validity = decimal_array.validity_mask()?.to_bit_buffer(); + let validity = decimal_array + .as_ref() + .validity()? + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); let opt_values = buf diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index eeb1b8adf52..e1e4c388e55 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -28,7 +30,10 @@ pub fn slice_canonical_array( stop: usize, ) -> VortexResult { let validity = if array.dtype().is_nullable() { - let bool_buff = array.validity_mask()?.to_bit_buffer(); + let bool_buff = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from(bool_buff.slice(start..stop)) } else { Validity::NonNullable diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index 6c07c76c20d..096e5bbdc06 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -5,7 +5,9 @@ use std::cmp::Ordering; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -28,7 +30,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { let mut opt_values = bool_array .to_bit_buffer() .iter() - .zip(bool_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + bool_array + .as_ref() + .validity()? + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(b, v)| v.then_some(b)) .collect::>(); opt_values.sort(); @@ -41,7 +53,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { .as_slice::

() .iter() .copied() - .zip(primitive_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + primitive_array + .as_ref() + .validity()? + .to_mask( + primitive_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(p, v)| v.then_some(p)) .collect::>(); sort_primitive_slice(&mut opt_values); @@ -56,7 +78,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { .as_slice() .iter() .copied() - .zip(decimal_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + decimal_array + .as_ref() + .validity()? + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(p, v)| v.then_some(p)) .collect::>(); opt_values.sort(); diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index a113829077b..01c7eae4a39 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -43,7 +45,10 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort let nullable: Nullability = indices.contains(&None).into(); let validity = if array.dtype().is_nullable() || nullable == Nullability::Nullable { - let validity_idx = array.validity_mask()?.to_bit_buffer(); + let validity_idx = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from_iter( indices diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 25736c6df84..a849acdb20e 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -78,7 +78,7 @@ pub fn vortex_array::aggregate_fn::fns::count::Count::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::count::Count::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_array::aggregate_fn::fns::first @@ -126,7 +126,7 @@ pub fn vortex_array::aggregate_fn::fns::first::First::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::first::First::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub struct vortex_array::aggregate_fn::fns::first::FirstPartial @@ -338,7 +338,7 @@ pub fn vortex_array::aggregate_fn::fns::last::Last::serialize(&self, _options: & pub fn vortex_array::aggregate_fn::fns::last::Last::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub struct vortex_array::aggregate_fn::fns::last::LastPartial @@ -442,7 +442,7 @@ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Options = vortex_ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Partial = u64 -pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -506,7 +506,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial -pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::sum::Sum::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -816,7 +816,7 @@ pub fn vortex_array::aggregate_fn::fns::count::Count::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::count::Count::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::first::First @@ -852,7 +852,7 @@ pub fn vortex_array::aggregate_fn::fns::first::First::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::first::First::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::is_constant::IsConstant @@ -960,7 +960,7 @@ pub fn vortex_array::aggregate_fn::fns::last::Last::serialize(&self, _options: & pub fn vortex_array::aggregate_fn::fns::last::Last::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::min_max::MinMax @@ -1004,7 +1004,7 @@ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Options = vortex_ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Partial = u64 -pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -1040,7 +1040,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial -pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::sum::Sum::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -1266,8 +1266,6 @@ pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, pub trait vortex_array::arrays::bool::BoolArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::bool::BoolArrayExt::bool_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self) -> vortex_error::VortexResult> pub fn vortex_array::arrays::bool::BoolArrayExt::nullability(&self) -> vortex_array::dtype::Nullability @@ -1276,14 +1274,12 @@ pub fn vortex_array::arrays::bool::BoolArrayExt::to_bit_buffer(&self) -> vortex_ pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self) -> vortex_mask::Mask -pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn vortex_array::arrays::bool::BoolArrayExt::validity(&self) -> vortex_array::validity::Validity impl> vortex_array::arrays::bool::BoolArrayExt for T -pub fn T::bool_validity_mask(&self) -> vortex_mask::Mask - pub fn T::maybe_to_mask(&self) -> vortex_error::VortexResult> pub fn T::nullability(&self) -> vortex_array::dtype::Nullability @@ -1292,7 +1288,7 @@ pub fn T::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer pub fn T::to_mask(&self) -> vortex_mask::Mask -pub fn T::to_mask_fill_null_false(&self) -> vortex_mask::Mask +pub fn T::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn T::validity(&self) -> vortex_array::validity::Validity @@ -2722,8 +2718,6 @@ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::list_size(&self) -> u32 impl> vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt for T @@ -2736,8 +2730,6 @@ pub fn T::fixed_size_list_elements_at(&self, index: usize) -> vortex_error::Vort pub fn T::fixed_size_list_validity(&self) -> vortex_array::validity::Validity -pub fn T::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask - pub fn T::list_size(&self) -> u32 pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array @@ -2876,8 +2868,6 @@ pub fn vortex_array::arrays::list::ListArrayExt::list_elements_at(&self, index: pub fn vortex_array::arrays::list::ListArrayExt::list_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::list::ListArrayExt::list_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::list::ListArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::list::ListArrayExt::offset_at(&self, index: usize) -> vortex_error::VortexResult @@ -2898,8 +2888,6 @@ pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult vortex_array::validity::Validity -pub fn T::list_validity_mask(&self) -> vortex_mask::Mask - pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::offset_at(&self, index: usize) -> vortex_error::VortexResult @@ -3058,8 +3046,6 @@ pub fn vortex_array::arrays::listview::ListViewArrayExt::list_elements_at(&self, pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::listview::ListViewArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::listview::ListViewArrayExt::offset_at(&self, index: usize) -> usize @@ -3080,8 +3066,6 @@ pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult vortex_array::validity::Validity -pub fn T::listview_validity_mask(&self) -> vortex_mask::Mask - pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::offset_at(&self, index: usize) -> usize @@ -3206,8 +3190,6 @@ pub fn vortex_array::arrays::masked::MaskedArrayExt::child(&self) -> &vortex_arr pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::masked::MaskedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> impl> vortex_array::arrays::masked::MaskedArrayExt for T @@ -3216,8 +3198,6 @@ pub fn T::child(&self) -> &vortex_array::ArrayRef pub fn T::masked_validity(&self) -> vortex_array::validity::Validity -pub fn T::masked_validity_mask(&self) -> vortex_mask::Mask - pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3794,8 +3774,6 @@ pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity(&self) -> vo pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::primitive::PrimitiveArrayExt for T pub fn T::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle @@ -3812,8 +3790,6 @@ pub fn T::validity(&self) -> vortex_array::validity::Validity pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::primitive::chunk_range(chunk_idx: usize, offset: usize, array_len: usize) -> core::ops::range::Range pub fn vortex_array::arrays::primitive::patch_chunk(decoded_values: &mut [T], patches_indices: &[I], patches_values: &[T], patches_offset: usize, chunk_offsets_slice: &[C], chunk_idx: usize, offset_within_chunk: usize) where T: vortex_array::dtype::NativePType, I: vortex_array::dtype::UnsignedPType, C: vortex_array::dtype::UnsignedPType @@ -4658,8 +4634,6 @@ pub fn vortex_array::arrays::varbin::VarBinArrayExt::validity_child(&self) -> co pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::varbin::VarBinArrayExt for T pub fn T::bytes_at(&self, index: usize) -> vortex_buffer::ByteBuffer @@ -4680,8 +4654,6 @@ pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> pub fn T::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn T::varbin_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::varbin::varbin_scalar(value: vortex_buffer::ByteBuffer, dtype: &vortex_array::dtype::DType) -> vortex_array::scalar::Scalar pub type vortex_array::arrays::varbin::VarBinArray = vortex_array::Array @@ -5042,16 +5014,12 @@ pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::dtype_parts(&self) pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::varbinview::VarBinViewArrayExt for T pub fn T::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) pub fn T::varbinview_validity(&self) -> vortex_array::validity::Validity -pub fn T::varbinview_validity_mask(&self) -> vortex_mask::Mask - pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -6780,11 +6748,11 @@ pub mod vortex_array::arrow pub mod vortex_array::arrow::bool -pub fn vortex_array::arrow::bool::canonical_bool_to_arrow(array: &vortex_array::arrays::BoolArray) -> vortex_error::VortexResult +pub fn vortex_array::arrow::bool::canonical_bool_to_arrow(array: &vortex_array::arrays::BoolArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_array::arrow::byte_view -pub fn vortex_array::arrow::byte_view::canonical_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray) -> vortex_error::VortexResult +pub fn vortex_array::arrow::byte_view::canonical_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrow::byte_view::execute_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6794,7 +6762,7 @@ pub fn vortex_array::arrow::null::canonical_null_to_arrow(array: &vortex_array:: pub mod vortex_array::arrow::primitive -pub fn vortex_array::arrow::primitive::canonical_primitive_to_arrow(array: vortex_array::arrays::PrimitiveArray) -> vortex_error::VortexResult where ::Native: vortex_array::dtype::NativePType +pub fn vortex_array::arrow::primitive::canonical_primitive_to_arrow(array: vortex_array::arrays::PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult where ::Native: vortex_array::dtype::NativePType pub struct vortex_array::arrow::ArrowArrayStreamAdapter @@ -19082,7 +19050,7 @@ pub fn vortex_array::validity::Validity::take(&self, indices: &vortex_array::Arr pub fn vortex_array::validity::Validity::to_array(&self, len: usize) -> vortex_array::ArrayRef -pub fn vortex_array::validity::Validity::to_mask(&self, length: usize) -> vortex_mask::Mask +pub fn vortex_array::validity::Validity::to_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::validity::Validity::uncompressed_size(&self) -> usize @@ -21788,7 +21756,7 @@ pub fn vortex_array::Array::as_constant(&self) -> core::option::Option::filter(&self, mask: vortex_mask::Mask) -> vortex_error::VortexResult -pub fn vortex_array::Array::invalid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::invalid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::is_invalid(&self, index: usize) -> vortex_error::VortexResult @@ -21806,12 +21774,10 @@ pub fn vortex_array::Array::take(&self, indices: vortex_array::ArrayRef) -> v pub fn vortex_array::Array::to_canonical(&self) -> vortex_error::VortexResult -pub fn vortex_array::Array::valid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::valid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::validity(&self) -> vortex_error::VortexResult -pub fn vortex_array::Array::validity_mask(&self) -> vortex_error::VortexResult - impl vortex_array::Array pub fn vortex_array::Array::as_array(&self) -> &vortex_array::ArrayRef @@ -22006,7 +21972,7 @@ pub fn vortex_array::ArrayRef::filter(&self, mask: vortex_mask::Mask) -> vortex_ pub fn vortex_array::ArrayRef::into_canonical(self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::invalid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::invalid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::is(&self) -> bool @@ -22060,12 +22026,10 @@ pub fn vortex_array::ArrayRef::to_canonical(&self) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::try_downcast(self) -> core::result::Result, vortex_array::ArrayRef> -pub fn vortex_array::ArrayRef::valid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::valid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::validity(&self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::validity_mask(&self) -> vortex_error::VortexResult - pub fn vortex_array::ArrayRef::with_slot(self, slot_idx: usize, replacement: vortex_array::ArrayRef) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::with_slots(self, slots: alloc::vec::Vec>) -> vortex_error::VortexResult diff --git a/vortex-array/src/aggregate_fn/fns/count/mod.rs b/vortex-array/src/aggregate_fn/fns/count/mod.rs index da12b85b0ee..e25c42e0845 100644 --- a/vortex-array/src/aggregate_fn/fns/count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/count/mod.rs @@ -76,9 +76,9 @@ impl AggregateFnVTable for Count { &self, state: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - *state += batch.valid_count()? as u64; + *state += batch.valid_count(ctx)? as u64; Ok(true) } diff --git a/vortex-array/src/aggregate_fn/fns/first/mod.rs b/vortex-array/src/aggregate_fn/fns/first/mod.rs index 67affbd4fdc..93e1fea6043 100644 --- a/vortex-array/src/aggregate_fn/fns/first/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/first/mod.rs @@ -94,12 +94,12 @@ impl AggregateFnVTable for First { &self, partial: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { if partial.value.is_some() { return Ok(true); } - if let Some(idx) = batch.validity_mask()?.first() { + if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.first() { let scalar = batch.scalar_at(idx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index 3edebb91333..b684ea98b17 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -59,8 +59,8 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor } // Check validity masks match (null positions must be identical). - let a_mask = a.validity_mask()?; - let b_mask = b.validity_mask()?; + let a_mask = a.validity()?.to_mask(a.len(), ctx)?; + let b_mask = b.validity()?.to_mask(b.len(), ctx)?; if a_mask != b_mask { return Ok(false); } diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs index c1efdc91563..408fa9b7806 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs @@ -5,11 +5,20 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; -pub(super) fn check_bool_sorted(array: &BoolArray, strict: bool) -> VortexResult { - match array.validity_mask()? { +pub(super) fn check_bool_sorted( + array: &BoolArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let values = array.to_bit_buffer(); diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs index e56858fd015..a5e2d597485 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs @@ -6,21 +6,34 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::NativeDecimalType; use crate::match_each_decimal_value_type; -pub(super) fn check_decimal_sorted(array: &DecimalArray, strict: bool) -> VortexResult { +pub(super) fn check_decimal_sorted( + array: &DecimalArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { match_each_decimal_value_type!(array.values_type(), |S| { - compute_is_sorted::(array, strict) + compute_is_sorted::(array, strict, ctx) }) } -fn compute_is_sorted(array: &DecimalArray, strict: bool) -> VortexResult +fn compute_is_sorted( + array: &DecimalArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult where dyn Iterator: IsSortedIteratorExt, { - match array.validity_mask()? { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let buf = array.buffer::(); diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index e68e74da63d..f0a016796c9 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -108,7 +108,7 @@ fn is_sorted_impl(array: &ArrayRef, strict: bool, ctx: &mut ExecutionCtx) -> Vor // Enforce strictness before we even try to check if the array is sorted. if strict { - let invalid_count = array.invalid_count()?; + let invalid_count = array.invalid_count(ctx)?; match invalid_count { // We can keep going 0 => {} @@ -462,10 +462,10 @@ impl AggregateFnVTable for IsSorted { // Check within-batch sortedness. let batch_is_sorted = match c { - Canonical::Primitive(p) => check_primitive_sorted(p, partial.strict)?, - Canonical::Bool(b) => check_bool_sorted(b, partial.strict)?, + Canonical::Primitive(p) => check_primitive_sorted(p, partial.strict, ctx)?, + Canonical::Bool(b) => check_bool_sorted(b, partial.strict, ctx)?, Canonical::VarBinView(v) => check_varbinview_sorted(v, partial.strict)?, - Canonical::Decimal(d) => check_decimal_sorted(d, partial.strict)?, + Canonical::Decimal(d) => check_decimal_sorted(d, partial.strict, ctx)?, Canonical::Extension(e) => check_extension_sorted(e, partial.strict, ctx)?, Canonical::Null(_) => !partial.strict, // Struct, List, FixedSizeList should have been filtered out by return_dtype diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs index 21c80e7bd45..e5433b6b176 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs @@ -6,17 +6,32 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; use crate::match_each_native_ptype; -pub(super) fn check_primitive_sorted(array: &PrimitiveArray, strict: bool) -> VortexResult { - match_each_native_ptype!(array.ptype(), |P| { compute_is_sorted::

(array, strict) }) +pub(super) fn check_primitive_sorted( + array: &PrimitiveArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match_each_native_ptype!(array.ptype(), |P| { + compute_is_sorted::

(array, strict, ctx) + }) } -fn compute_is_sorted(array: &PrimitiveArray, strict: bool) -> VortexResult { - match array.validity_mask()? { +fn compute_is_sorted( + array: &PrimitiveArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let slice = array.as_slice::(); diff --git a/vortex-array/src/aggregate_fn/fns/last/mod.rs b/vortex-array/src/aggregate_fn/fns/last/mod.rs index a1cbf032b88..9edfce66437 100644 --- a/vortex-array/src/aggregate_fn/fns/last/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/last/mod.rs @@ -95,9 +95,9 @@ impl AggregateFnVTable for Last { &self, partial: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(idx) = batch.validity_mask()?.last() { + if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.last() { let scalar = batch.scalar_at(idx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs index 557598489e6..a4ac8d4f4c0 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs @@ -8,17 +8,25 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::dtype::Nullability::NonNullable; use crate::scalar::Scalar; -pub(super) fn accumulate_bool(partial: &mut MinMaxPartial, array: &BoolArray) -> VortexResult<()> { +pub(super) fn accumulate_bool( + partial: &mut MinMaxPartial, + array: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { if array.is_empty() { return Ok(()); } - let mask = array.validity_mask()?; + let mask = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let true_non_null = match &mask { Mask::AllTrue(_) => array.to_bit_buffer(), Mask::AllFalse(_) => return Ok(()), diff --git a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs index 93400f37275..8fa6b6b8dcd 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs @@ -7,6 +7,7 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; use crate::dtype::NativeDecimalType; @@ -18,30 +19,40 @@ use crate::scalar::Scalar; pub(super) fn accumulate_decimal( partial: &mut MinMaxPartial, array: &DecimalArray, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { match_each_decimal_value_type!(array.values_type(), |T| { - let local = compute_min_max_with_validity::(array)?; + let local = compute_min_max_with_validity::(array, ctx)?; partial.merge(local); Ok(()) }) } -fn compute_min_max_with_validity(array: &DecimalArray) -> VortexResult> +fn compute_min_max_with_validity( + array: &DecimalArray, + ctx: &mut ExecutionCtx, +) -> VortexResult> where D: Into + NativeDecimalType, { - Ok(match array.validity_mask()? { - Mask::AllTrue(_) => compute_min_max(array.buffer::().iter(), array.decimal_dtype()), - Mask::AllFalse(_) => None, - Mask::Values(v) => compute_min_max( - array - .buffer::() - .iter() - .zip(v.bit_buffer().iter()) - .filter_map(|(v, m)| m.then_some(v)), - array.decimal_dtype(), - ), - }) + Ok( + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { + Mask::AllTrue(_) => compute_min_max(array.buffer::().iter(), array.decimal_dtype()), + Mask::AllFalse(_) => None, + Mask::Values(v) => compute_min_max( + array + .buffer::() + .iter() + .zip(v.bit_buffer().iter()) + .filter_map(|(v, m)| m.then_some(v)), + array.decimal_dtype(), + ), + }, + ) } fn compute_min_max<'a, T>( diff --git a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs index 17e886b1efb..0b845e3276a 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs @@ -63,7 +63,7 @@ pub fn min_max(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult match c { - Canonical::Primitive(p) => accumulate_primitive(partial, p), - Canonical::Bool(b) => accumulate_bool(partial, b), + Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx), + Canonical::Bool(b) => accumulate_bool(partial, b, ctx), Canonical::VarBinView(v) => accumulate_varbinview(partial, v), - Canonical::Decimal(d) => accumulate_decimal(partial, d), + Canonical::Decimal(d) => accumulate_decimal(partial, d, ctx), Canonical::Extension(e) => accumulate_extension(partial, e, ctx), Canonical::Null(_) => Ok(()), Canonical::Struct(_) diff --git a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs index 1332fa7dd88..7bc46c90a1f 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs @@ -7,6 +7,7 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; use crate::dtype::Nullability::NonNullable; @@ -17,30 +18,40 @@ use crate::scalar::Scalar; pub(super) fn accumulate_primitive( partial: &mut MinMaxPartial, p: &PrimitiveArray, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { match_each_native_ptype!(p.ptype(), |T| { - let local = compute_min_max_with_validity::(p)?; + let local = compute_min_max_with_validity::(p, ctx)?; partial.merge(local); Ok(()) }) } -fn compute_min_max_with_validity(array: &PrimitiveArray) -> VortexResult> +fn compute_min_max_with_validity( + array: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult> where T: NativePType, PValue: From, { - Ok(match array.validity_mask()? { - Mask::AllTrue(_) => compute_min_max(array.as_slice::().iter()), - Mask::AllFalse(_) => None, - Mask::Values(v) => compute_min_max( - array - .as_slice::() - .iter() - .zip(v.bit_buffer().iter()) - .filter_map(|(v, m)| m.then_some(v)), - ), - }) + Ok( + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { + Mask::AllTrue(_) => compute_min_max(array.as_slice::().iter()), + Mask::AllFalse(_) => None, + Mask::Values(v) => compute_min_max( + array + .as_slice::() + .iter() + .zip(v.bit_buffer().iter()) + .filter_map(|(v, m)| m.then_some(v)), + ), + }, + ) } fn compute_min_max<'a, T>(iter: impl Iterator) -> Option diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs index 602963cc385..0e229ed7982 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs @@ -46,7 +46,7 @@ pub fn nan_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult VortexResult<()> { match batch { Columnar::Constant(c) => { @@ -150,7 +150,7 @@ impl AggregateFnVTable for NanCount { Ok(()) } Columnar::Canonical(c) => match c { - Canonical::Primitive(p) => accumulate_primitive(partial, p), + Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx), _ => vortex_bail!( "Unsupported canonical type for nan_count: {}", batch.dtype() diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs index 34781bafbd7..77ad78df1c4 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs @@ -4,13 +4,21 @@ use vortex_error::VortexResult; use vortex_mask::Mask; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; use crate::match_each_float_ptype; -pub(super) fn accumulate_primitive(count: &mut u64, p: &PrimitiveArray) -> VortexResult<()> { +pub(super) fn accumulate_primitive( + count: &mut u64, + p: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { match_each_float_ptype!(p.ptype(), |F| { - *count += compute_nan_count_with_validity(p.as_slice::(), p.validity_mask()?) as u64; + *count += compute_nan_count_with_validity( + p.as_slice::(), + p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?, + ) as u64; }); Ok(()) } diff --git a/vortex-array/src/aggregate_fn/fns/sum/bool.rs b/vortex-array/src/aggregate_fn/fns/sum/bool.rs index f80023e0b57..f5a02299278 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/bool.rs @@ -9,15 +9,20 @@ use vortex_mask::AllOr; use super::SumState; use super::checked_add_u64; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; -pub(super) fn accumulate_bool(inner: &mut SumState, b: &BoolArray) -> VortexResult { +pub(super) fn accumulate_bool( + inner: &mut SumState, + b: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { let SumState::Unsigned(acc) = inner else { vortex_panic!("expected unsigned sum state for bool input"); }; - let mask = b.validity_mask()?; + let mask = b.as_ref().validity()?.to_mask(b.as_ref().len(), ctx)?; let true_count = match mask.bit_buffer() { AllOr::None => return Ok(false), AllOr::All => b.to_bit_buffer().true_count() as u64, diff --git a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs index 91fd48e73ea..f5c801d8528 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use super::SumState; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; use crate::dtype::DecimalType; @@ -22,8 +23,12 @@ use crate::scalar::DecimalValue; /// Accumulate a decimal array into the sum state. /// Returns Ok(true) if saturated (overflow), Ok(false) if not. -pub(super) fn accumulate_decimal(inner: &mut SumState, d: &DecimalArray) -> VortexResult { - let mask = d.validity_mask()?; +pub(super) fn accumulate_decimal( + inner: &mut SumState, + d: &DecimalArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = d.as_ref().validity()?.to_mask(d.as_ref().len(), ctx)?; let validity = match &mask { Mask::AllTrue(_) => None, Mask::Values(mask_values) => Some(mask_values.bit_buffer()), diff --git a/vortex-array/src/aggregate_fn/fns/sum/mod.rs b/vortex-array/src/aggregate_fn/fns/sum/mod.rs index 7b45aa65fa9..cad8cedf626 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/mod.rs @@ -217,7 +217,7 @@ impl AggregateFnVTable for Sum { &self, partial: &mut Self::Partial, batch: &Columnar, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { // Constants compute scalar * len and combine via combine_partials. if let Columnar::Constant(c) = batch { @@ -234,9 +234,9 @@ impl AggregateFnVTable for Sum { let result = match batch { Columnar::Canonical(c) => match c { - Canonical::Primitive(p) => accumulate_primitive(&mut inner, p), - Canonical::Bool(b) => accumulate_bool(&mut inner, b), - Canonical::Decimal(d) => accumulate_decimal(&mut inner, d), + Canonical::Primitive(p) => accumulate_primitive(&mut inner, p, ctx), + Canonical::Bool(b) => accumulate_bool(&mut inner, b, ctx), + Canonical::Decimal(d) => accumulate_decimal(&mut inner, d, ctx), _ => vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype()), }, Columnar::Constant(_) => unreachable!(), diff --git a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs index ed8ecd17a6a..e236c624ad4 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs @@ -11,11 +11,16 @@ use vortex_mask::AllOr; use super::SumState; use super::checked_add_i64; use super::checked_add_u64; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::match_each_native_ptype; -pub(super) fn accumulate_primitive(inner: &mut SumState, p: &PrimitiveArray) -> VortexResult { - let mask = p.validity_mask()?; +pub(super) fn accumulate_primitive( + inner: &mut SumState, + p: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?; match mask.bit_buffer() { AllOr::None => Ok(false), AllOr::All => accumulate_primitive_all(inner, p), diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index f951f9423ae..b8ebee660bc 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -25,7 +25,6 @@ use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; use crate::VTable; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::sum::sum; @@ -41,7 +40,6 @@ use crate::arrays::Primitive; use crate::arrays::SliceArray; use crate::arrays::VarBin; use crate::arrays::VarBinView; -use crate::arrays::bool::BoolArrayExt; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; @@ -256,7 +254,7 @@ impl ArrayRef { } /// Returns the number of valid elements in the array. - pub fn valid_count(&self) -> VortexResult { + pub fn valid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { let len = self.len(); if let Some(Precision::Exact(invalid_count)) = self.statistics().get_as::(Stat::NullCount) @@ -268,8 +266,7 @@ impl ArrayRef { Validity::NonNullable | Validity::AllValid => len, Validity::AllInvalid => 0, Validity::Array(a) => { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let array_sum = sum(&a, &mut ctx)?; + let array_sum = sum(&a, ctx)?; array_sum .as_primitive() .as_::() @@ -285,8 +282,8 @@ impl ArrayRef { } /// Returns the number of invalid elements in the array. - pub fn invalid_count(&self) -> VortexResult { - Ok(self.len() - self.valid_count()?) + pub fn invalid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(self.len() - self.valid_count(ctx)?) } /// Returns the [`Validity`] of the array. @@ -294,15 +291,6 @@ impl ArrayRef { self.0.validity(self) } - /// Returns the canonical validity mask for the array. - pub fn validity_mask(&self) -> VortexResult { - match self.validity()? { - Validity::NonNullable | Validity::AllValid => Ok(Mask::new_true(self.len())), - Validity::AllInvalid => Ok(Mask::new_false(self.len())), - Validity::Array(a) => Ok(a.to_bool().to_mask()), - } - } - /// Returns the canonical representation of the array. pub fn into_canonical(self) -> VortexResult { self.execute(&mut LEGACY_SESSION.create_execution_ctx()) diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 55656409128..3cba3cb527a 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -392,12 +392,12 @@ impl Array { self.inner.as_constant() } - pub fn valid_count(&self) -> VortexResult { - self.inner.valid_count() + pub fn valid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + self.inner.valid_count(ctx) } - pub fn invalid_count(&self) -> VortexResult { - self.inner.invalid_count() + pub fn invalid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + self.inner.invalid_count(ctx) } pub fn append_to_builder( @@ -407,10 +407,6 @@ impl Array { ) -> VortexResult<()> { self.inner.append_to_builder(builder, ctx) } - - pub fn validity_mask(&self) -> VortexResult { - self.inner.validity_mask() - } } impl Deref for Array { diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 8a7eed44037..5ed85d7eab4 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; @@ -92,10 +93,6 @@ pub trait BoolArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn bool_validity_mask(&self) -> Mask { - self.validity().to_mask(self.as_ref().len()) - } - fn to_bit_buffer(&self) -> BitBuffer { let buffer = self.bits.as_host().clone(); BitBuffer::new_with_offset(buffer, self.as_ref().len(), self.offset) @@ -116,8 +113,11 @@ pub trait BoolArrayExt: TypedArrayRef { .vortex_expect("cannot convert nullable boolean array to mask") } - fn to_mask_fill_null_false(&self) -> Mask { - let validity_mask = self.bool_validity_mask(); + fn to_mask_fill_null_false(&self, ctx: &mut ExecutionCtx) -> Mask { + let validity_mask = self + .validity() + .to_mask(self.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"); let buffer = match validity_mask { Mask::AllTrue(_) => self.to_bit_buffer(), Mask::AllFalse(_) => return Mask::new_false(self.as_ref().len()), diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index 19b1508fd84..b3b3a269795 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -28,7 +28,7 @@ impl TakeExecute for Bool { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let indices_nulls_zeroed = match indices.validity_mask()? { + let indices_nulls_zeroed = match indices.validity()?.to_mask(indices.len(), ctx)? { Mask::AllTrue(_) => indices.clone(), Mask::AllFalse(_) => { return Ok(Some( diff --git a/vortex-array/src/arrays/bool/test_harness.rs b/vortex-array/src/arrays/bool/test_harness.rs index 52cffafdbd9..056d122b3dc 100644 --- a/vortex-array/src/arrays/bool/test_harness.rs +++ b/vortex-array/src/arrays/bool/test_harness.rs @@ -4,13 +4,20 @@ use vortex_error::VortexExpect; use vortex_error::vortex_panic; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; impl BoolArray { pub fn opt_bool_vec(&self) -> Vec> { - self.validity_mask() - .vortex_expect("validity_mask") + self.validity() + .vortex_expect("failed to get validity") + .to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter() .zip(self.to_bit_buffer().iter()) @@ -19,8 +26,13 @@ impl BoolArray { } pub fn bool_vec(&self) -> Vec { - self.validity_mask() - .vortex_expect("validity_mask") + self.validity() + .vortex_expect("failed to get validity") + .to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter() .zip(self.to_bit_buffer().iter()) diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 1cd48c57319..38b0f4313ea 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -31,7 +31,10 @@ fn take_chunked( .cast(DType::Primitive(PType::U64, indices.dtype().nullability()))? .execute::(ctx)?; - let indices_mask = indices.validity_mask()?; + let indices_mask = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let indices_values = indices.as_slice::(); let n = indices_values.len(); @@ -96,8 +99,13 @@ fn take_chunked( // 4. Single take to restore original order and expand duplicates. // Carry the original index validity so null indices produce null outputs. - let take_validity = - Validity::from_mask(indices.validity_mask()?, indices.dtype().nullability()); + let take_validity = Validity::from_mask( + indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?, + indices.dtype().nullability(), + ); flat.take(PrimitiveArray::new(final_take.freeze(), take_validity).into_array()) } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 96aa71fd9ac..e01a37f7caa 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -6,6 +6,8 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; @@ -18,7 +20,12 @@ use crate::validity::Validity; impl TakeReduce for Constant { fn take(array: ArrayView<'_, Constant>, indices: &ArrayRef) -> VortexResult> { - let result = match indices.validity_mask()?.bit_buffer() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let result = match indices + .validity()? + .to_mask(indices.len(), &mut ctx)? + .bit_buffer() + { AllOr::All => { let scalar = Scalar::try_new( array @@ -64,7 +71,9 @@ mod tests { use vortex_mask::AllOr; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; @@ -98,7 +107,12 @@ mod tests { ) ); assert_eq!( - taken.validity_mask().unwrap().indices(), + taken + .validity() + .unwrap() + .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .indices(), AllOr::Some(valid_indices) ); } @@ -117,7 +131,15 @@ mod tests { taken.to_primitive(), PrimitiveArray::new(buffer![42i32, 42, 42], Validity::AllValid) ); - assert_eq!(taken.validity_mask().unwrap().indices(), AllOr::All); + assert_eq!( + taken + .validity() + .unwrap() + .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .indices(), + AllOr::All + ); } #[rstest] diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 137f48fd8eb..9adc39212cc 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -323,6 +323,8 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; @@ -475,7 +477,12 @@ mod tests { let struct_array = array.as_array().to_struct(); assert_eq!(struct_array.len(), 3); - assert_eq!(struct_array.valid_count().unwrap(), 0); + assert_eq!( + struct_array + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 0 + ); let field = struct_array .unmasked_field_by_name("non_null_field") diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 80377102b7d..f602b827c03 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -146,6 +146,8 @@ mod tests { use super::upcast_decimal_values; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builtins::ArrayBuiltins; use crate::canonical::ToCanonical; @@ -383,7 +385,15 @@ mod tests { assert_eq!(casted.len(), 3); // Check validity is preserved - let mask = casted.validity_mask().unwrap(); + let mask = casted + .as_ref() + .validity() + .unwrap() + .to_mask( + casted.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(mask.value(0)); assert!(!mask.value(1)); assert!(mask.value(2)); diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index e697a277cca..de01076b231 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -89,6 +89,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; @@ -123,7 +125,14 @@ mod tests { p.buffer::().as_slice(), vec![4200, 800, 4200, 1000, 4200] ); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index b5f007a05d3..d3ef863990c 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -12,7 +12,9 @@ use vortex_error::vortex_ensure; use vortex_mask::AllOr; use crate::ArrayRef; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -139,7 +141,10 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { } fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { - let codes_validity = self.codes().validity_mask()?; + let codes = self.codes(); + let codes_validity = codes + .validity()? + .to_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; let codes_primitive = self.codes().to_primitive(); let values_len = self.values().len(); @@ -290,7 +295,15 @@ mod test { PrimitiveArray::new(buffer![3, 6, 9], Validity::AllValid).into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -308,7 +321,15 @@ mod test { .into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -330,7 +351,15 @@ mod test { .into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -348,7 +377,15 @@ mod test { PrimitiveArray::new(buffer![3, 6, 9], Validity::NonNullable).into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index bcbf28c0d96..fbe840f9bee 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -54,7 +54,7 @@ fn filter_validity(validity: Validity, mask: &Arc) -> Validity { /// Check for some fast-path execution conditions before calling [`execute_filter`]. pub(super) fn execute_filter_fast_paths( array: ArrayView<'_, Filter>, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let true_count = array.mask.true_count(); @@ -70,7 +70,13 @@ pub(super) fn execute_filter_fast_paths( // Also check if the array itself is completely null, in which case we only care about the total // number of nulls, not the values. - if array.array().validity_mask()?.true_count() == 0 { + let child_arr = array.array(); + if child_arr + .validity()? + .to_mask(child_arr.len(), ctx)? + .true_count() + == 0 + { return Ok(Some( ConstantArray::new(Scalar::null(array.dtype().clone()), true_count).into_array(), )); diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index dcbee9f8f6f..22dd34b46bb 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -231,10 +231,6 @@ pub trait FixedSizeListArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], nullability) } - fn fixed_size_list_validity_mask(&self) -> vortex_mask::Mask { - self.fixed_size_list_validity().to_mask(self.as_ref().len()) - } - fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { debug_assert!( index < self.as_ref().len(), diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index 259eb382013..95cb101e9da 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -6,17 +6,19 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::ToCanonical; use crate::array::ArrayView; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::dict::TakeExecute; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; -use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::IntegerPType; use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; @@ -82,7 +84,7 @@ fn take_with_indices( // The result's nullability is the union of the input nullabilities. if array.dtype().is_nullable() || indices_array.dtype().is_nullable() { let indices_array = indices_array.as_view(); - take_nullable_fsl::(array, indices_array) + take_nullable_fsl::(array, indices_array, ctx) } else { let indices_array = indices_array.as_view(); take_non_nullable_fsl::(array, indices_array) @@ -143,13 +145,25 @@ fn take_non_nullable_fsl( fn take_nullable_fsl( array: ArrayView<'_, FixedSizeList>, indices_array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, ) -> VortexResult { let list_size = array.list_size() as usize; let indices: &[I] = indices_array.as_slice::(); let new_len = indices.len(); - let array_validity = array.fixed_size_list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let array_validity = array + .fixed_size_list_validity() + .to_mask(array.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"); + let indices_len = indices_array.as_ref().len(); + let indices_validity = match indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + { + Validity::NonNullable | Validity::AllValid => Mask::new_true(indices_len), + Validity::AllInvalid => Mask::new_false(indices_len), + Validity::Array(a) => a.to_bool().to_mask(), + }; // We must use placeholder zeros for null lists to maintain the array length without // propagating nullability to the element array's take operation. diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 75db7d0f08c..c941060a42d 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -288,10 +288,6 @@ pub trait ListArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn list_validity_mask(&self) -> vortex_mask::Mask { - self.list_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> VortexResult { vortex_ensure!( index <= self.as_ref().len(), diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 841fd4ac45b..22cfe224580 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -59,8 +59,11 @@ fn _take( indices_array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let data_validity = array.list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + .to_mask(indices_array.as_ref().len(), ctx)?; if !indices_validity.all_true() || !data_validity.all_true() { return _take_nullable::(array, indices_array, ctx); @@ -124,8 +127,11 @@ fn _take_nullable(ctx)?; let offsets: &[O] = offsets_array.as_slice(); let indices: &[I] = indices_array.as_slice(); - let data_validity = array.list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + .to_mask(indices_array.as_ref().len(), ctx)?; let mut new_offsets = PrimitiveBuilder::::with_capacity( Nullability::NonNullable, diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 72c376b2f44..8f9aee6406e 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -334,10 +334,6 @@ pub trait ListViewArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn listview_validity_mask(&self) -> vortex_mask::Mask { - self.listview_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> usize { assert!( index < self.as_ref().len(), diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 4f6d53e5b35..e902fa56a47 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -50,10 +50,6 @@ pub trait MaskedArrayExt: TypedArrayRef { self.as_ref().dtype().nullability(), ) } - - fn masked_validity_mask(&self) -> vortex_mask::Mask { - self.masked_validity().to_mask(self.as_ref().len()) - } } impl> MaskedArrayExt for T {} diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 760383bcce0..551ead31448 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -61,7 +61,8 @@ fn test_masked_child_with_validity() { let prim = array.as_array().to_primitive(); // Positions where validity is false should be null in masked_child. - assert_eq!(prim.valid_count().unwrap(), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); assert!(prim.is_valid(0).unwrap()); assert!(!prim.is_valid(1).unwrap()); assert!(prim.is_valid(2).unwrap()); @@ -76,7 +77,12 @@ fn test_masked_child_all_valid() { let array = MaskedArray::try_new(child, Validity::AllValid).unwrap(); assert_eq!(array.len(), 3); - assert_eq!(array.valid_count().unwrap(), 3); + assert_eq!( + array + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3 + ); assert_arrays_eq!( PrimitiveArray::from_option_iter([10i32, 20, 30].map(Some)), array diff --git a/vortex-array/src/arrays/masked/vtable/canonical.rs b/vortex-array/src/arrays/masked/vtable/canonical.rs index c612f522ef8..397cabf768a 100644 --- a/vortex-array/src/arrays/masked/vtable/canonical.rs +++ b/vortex-array/src/arrays/masked/vtable/canonical.rs @@ -7,6 +7,8 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; @@ -49,7 +51,8 @@ mod tests { let prim = canonical.into_primitive(); // Check that null positions match validity. - assert_eq!(prim.valid_count().unwrap(), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); assert!(prim.is_valid(0).unwrap()); assert!(!prim.is_valid(1).unwrap()); assert!(prim.is_valid(2).unwrap()); @@ -68,7 +71,13 @@ mod tests { let canonical = array.to_canonical()?; assert_eq!(canonical.dtype().nullability(), Nullability::Nullable); - assert_eq!(canonical.into_array().valid_count().unwrap(), 3); + assert_eq!( + canonical + .into_array() + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3 + ); Ok(()) } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 5f1f09e0644..2bfd91c61a8 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -155,7 +155,7 @@ impl VTable for Masked { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let validity_mask = array.masked_validity_mask(); + let validity_mask = array.masked_validity().to_mask(array.len(), ctx)?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index ad5558a8807..e6c69beee54 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -15,7 +15,9 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::NullArray; use crate::compute::conformance::consistency::test_array_consistency; use crate::compute::conformance::filter::test_filter_conformance; @@ -29,8 +31,13 @@ mod test { let sliced = nulls.slice(0..4).unwrap().to_null(); assert_eq!(sliced.len(), 4); + let sliced_arr = sliced.as_array(); assert!(matches!( - sliced.as_array().validity_mask().unwrap(), + sliced_arr + .validity() + .unwrap() + .to_mask(sliced_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::AllFalse(4) )); } @@ -44,8 +51,13 @@ mod test { .to_null(); assert_eq!(taken.len(), 5); + let taken_arr = taken.as_array(); assert!(matches!( - taken.as_array().validity_mask().unwrap(), + taken_arr + .validity() + .unwrap() + .to_mask(taken_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::AllFalse(5) )); } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index e64cc891d4e..cb1f132b70f 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -123,10 +123,6 @@ pub trait PrimitiveArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.as_ref().len()) - } - fn buffer_handle(&self) -> &BufferHandle { &self.buffer } diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index dae36a5a523..8638ea9af97 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -10,6 +10,8 @@ use vortex_mask::AllOr; use vortex_mask::Mask; use vortex_utils::aliases::hash_map::HashMap; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; @@ -29,7 +31,13 @@ impl PrimitiveArray { } match_each_native_ptype!(self.ptype(), |P| { - let (top, count) = typed_top_value(self.as_slice::

(), self.validity_mask()?); + let (top, count) = typed_top_value( + self.as_slice::

(), + self.as_ref().validity()?.to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?, + ); Ok(Some((top.into(), count))) }) } diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index 13cba60bfa6..e5258bccf65 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -78,7 +78,7 @@ impl CastKernel for Primitive { })); } - // Otherwise, cast the values element-wise. + // Otherwise, we need to cast the values one-by-one. Ok(Some(match_each_native_ptype!(new_ptype, |T| { match_each_native_ptype!(array.ptype(), |F| { PrimitiveArray::new(cast::(array.as_slice()), new_validity).into_array() @@ -116,6 +116,8 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; @@ -228,7 +230,11 @@ mod test { PrimitiveArray::from_option_iter([None, Some(0u32), Some(10)]) ); assert_eq!( - p.validity_mask().unwrap(), + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from(BitBuffer::from(vec![false, true, true])) ); } diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index d0de9a81a4f..3ee48ce4c05 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -51,6 +51,8 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::compute::fill_null::BoolArray; use crate::assert_arrays_eq; @@ -68,7 +70,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([42u8, 8, 42, 10, 42])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -81,7 +90,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([255u8, 255, 255, 255, 255])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -96,7 +112,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -104,6 +127,13 @@ mod test { let arr = buffer![8u8, 10, 12, 14, 16].into_array(); let p = arr.fill_null(Scalar::from(255u8)).unwrap().to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } } diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index fb35ee503bc..a09bd7495af 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -10,7 +10,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; @@ -317,10 +316,6 @@ pub trait VarBinArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn varbin_validity_mask(&self) -> Mask { - self.varbin_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> usize { assert!( index <= self.as_ref().len(), diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index b4ad470760e..776f77a8796 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -145,7 +145,9 @@ mod test { use vortex_buffer::ByteBuffer; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; @@ -176,7 +178,16 @@ mod test { .to_bool(); assert_eq!( - &result.validity_mask().unwrap().to_bit_buffer(), + &result + .as_ref() + .validity() + .unwrap() + .to_mask( + result.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .to_bit_buffer(), &BitBuffer::from_iter([true, false, true]) ); assert_eq!( @@ -202,7 +213,16 @@ mod test { .to_bool(); assert_eq!( - result.validity_mask().unwrap().to_bit_buffer(), + result + .as_ref() + .validity() + .unwrap() + .to_mask( + result.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .to_bit_buffer(), BitBuffer::from_iter([false, false, true]) ); assert_eq!( diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index 8f4b6793a72..911bc9c5e00 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -68,7 +68,10 @@ fn filter_select_var_bin_by_slice( offsets.as_slice::(), values.bytes().as_slice(), mask_slices, - values.varbin_validity_mask(), + values + .varbin_validity() + .to_mask(values.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"), selection_count, ) }) diff --git a/vortex-array/src/arrays/varbin/compute/take.rs b/vortex-array/src/arrays/varbin/compute/take.rs index de8b1ffff77..0a6d5624229 100644 --- a/vortex-array/src/arrays/varbin/compute/take.rs +++ b/vortex-array/src/arrays/varbin/compute/take.rs @@ -37,8 +37,11 @@ impl TakeExecute for VarBin { .dtype() .clone() .union_nullability(indices.dtype().nullability()); - let array_validity = array.varbin_validity_mask(); - let indices_validity = indices.validity_mask()?; + let array_validity = array.varbin_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let array = match_each_integer_ptype!(indices.ptype(), |I| { // On take, offsets get widened to either 32- or 64-bit based on the original type, diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 37d055107cf..7db90077628 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -15,7 +15,6 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; @@ -544,10 +543,6 @@ pub trait VarBinViewArrayExt: TypedArrayRef { fn varbinview_validity(&self) -> Validity { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.dtype_parts().1) } - - fn varbinview_validity_mask(&self) -> Mask { - self.varbinview_validity().to_mask(self.as_ref().len()) - } } impl> VarBinViewArrayExt for T {} diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index 37bae3d9822..9667acbd5dd 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -11,6 +11,8 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::Ref; use crate::builders::ArrayBuilder; @@ -63,7 +65,10 @@ impl VarBinViewArray { where F: FnMut(&Ref), { - match self.validity_mask()? { + match self.as_ref().validity()?.to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? { Mask::AllTrue(_) => { for &view in self.views().iter() { if !view.is_inlined() { diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index 644465ecac3..e7b93bc79d6 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -32,7 +32,10 @@ impl TakeExecute for VarBinView { let validity = array.validity()?.take(indices)?; let indices = indices.clone().execute::(ctx)?; - let indices_mask = indices.validity_mask()?; + let indices_mask = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let views_buffer = match_each_integer_ptype!(indices.ptype(), |I| { take_views(array.views(), indices.as_slice::(), &indices_mask) }); diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index dfa2ae8e71c..57f8aab8067 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -54,8 +54,8 @@ impl ZipKernel for VarBinView { let mut views_builder = BufferMut::::with_capacity(len); let mut validity_builder = LazyBitBufferBuilder::new(len); - let true_validity = if_true.varbinview_validity_mask(); - let false_validity = if_false.varbinview_validity_mask(); + let true_validity = if_true.varbinview_validity().to_mask(len, ctx)?; + let false_validity = if_false.varbinview_validity().to_mask(len, ctx)?; let mask = mask.try_to_mask_fill_null_false(ctx)?; let if_false_view = if_false; diff --git a/vortex-array/src/arrow/executor/bool.rs b/vortex-array/src/arrow/executor/bool.rs index 28862a150d0..44fb9d4f683 100644 --- a/vortex-array/src/arrow/executor/bool.rs +++ b/vortex-array/src/arrow/executor/bool.rs @@ -14,10 +14,18 @@ use crate::arrays::bool::BoolArrayExt; use crate::arrow::null_buffer::to_null_buffer; /// Convert a canonical BoolArray directly to Arrow. -pub fn canonical_bool_to_arrow(array: &BoolArray) -> VortexResult { +pub fn canonical_bool_to_arrow( + array: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { Ok(Arc::new(ArrowBooleanArray::new( array.to_bit_buffer().into(), - to_null_buffer(array.validity_mask()?), + to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ), ))) } @@ -26,5 +34,5 @@ pub(super) fn to_arrow_bool( ctx: &mut ExecutionCtx, ) -> VortexResult { let bool_array = array.execute::(ctx)?; - canonical_bool_to_arrow(&bool_array) + canonical_bool_to_arrow(&bool_array, ctx) } diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index 9b19ddcc722..653817a982e 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -22,6 +22,7 @@ use crate::dtype::arrow::FromArrowType; /// Convert a canonical VarBinViewArray directly to Arrow. pub fn canonical_varbinview_to_arrow( array: &VarBinViewArray, + ctx: &mut ExecutionCtx, ) -> VortexResult { let views = ScalarBuffer::::from(array.views_handle().as_host().clone().into_arrow_buffer()); @@ -30,7 +31,12 @@ pub fn canonical_varbinview_to_arrow( .iter() .map(|buffer| buffer.as_host().clone().into_arrow_buffer()) .collect(); - let nulls = to_null_buffer(array.validity_mask()?); + let nulls = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); // SAFETY: our own VarBinView array is considered safe. Ok(Arc::new(unsafe { @@ -68,5 +74,5 @@ pub(super) fn to_arrow_byte_view( let array = array.cast(DType::from_arrow((&T::DATA_TYPE, Nullability::Nullable)))?; let varbinview = array.execute::(ctx)?; - canonical_varbinview_to_arrow::(&varbinview) + canonical_varbinview_to_arrow::(&varbinview, ctx) } diff --git a/vortex-array/src/arrow/executor/decimal.rs b/vortex-array/src/arrow/executor/decimal.rs index 234b253f903..bb4e12ef1fb 100644 --- a/vortex-array/src/arrow/executor/decimal.rs +++ b/vortex-array/src/arrow/executor/decimal.rs @@ -32,16 +32,21 @@ pub(super) fn to_arrow_decimal( let decimal_array = array.execute::(ctx)?; match data_type { - DataType::Decimal32(..) => to_arrow_decimal32(decimal_array), - DataType::Decimal64(..) => to_arrow_decimal64(decimal_array), - DataType::Decimal128(..) => to_arrow_decimal128(decimal_array), - DataType::Decimal256(..) => to_arrow_decimal256(decimal_array), + DataType::Decimal32(..) => to_arrow_decimal32(decimal_array, ctx), + DataType::Decimal64(..) => to_arrow_decimal64(decimal_array, ctx), + DataType::Decimal128(..) => to_arrow_decimal128(decimal_array, ctx), + DataType::Decimal256(..) => to_arrow_decimal256(decimal_array, ctx), _ => unreachable!("to_arrow_decimal called with non-decimal type"), } } -fn to_arrow_decimal32(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal32(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -84,8 +89,13 @@ fn to_arrow_decimal32(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal64(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal64(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -123,8 +133,13 @@ fn to_arrow_decimal64(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal128(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal128(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -157,8 +172,13 @@ fn to_arrow_decimal128(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal256(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal256(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) diff --git a/vortex-array/src/arrow/executor/primitive.rs b/vortex-array/src/arrow/executor/primitive.rs index 7910ed5fa31..2e7d8a9716c 100644 --- a/vortex-array/src/arrow/executor/primitive.rs +++ b/vortex-array/src/arrow/executor/primitive.rs @@ -20,11 +20,15 @@ use crate::dtype::Nullability; /// Convert a canonical PrimitiveArray directly to Arrow. pub fn canonical_primitive_to_arrow( array: PrimitiveArray, + ctx: &mut ExecutionCtx, ) -> VortexResult where T::Native: NativePType, { - let validity = array.validity_mask()?; + let validity = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let null_buffer = to_null_buffer(validity); let buffer = array.into_buffer::().into_arrow_scalar_buffer(); Ok(Arc::new(ArrowPrimitiveArray::::new(buffer, null_buffer))) @@ -40,5 +44,5 @@ where // We use nullable here so we can essentially ignore nullability during the cast. let array = array.cast(DType::Primitive(T::Native::PTYPE, Nullability::Nullable))?; let primitive = array.execute::(ctx)?; - canonical_primitive_to_arrow::(primitive) + canonical_primitive_to_arrow::(primitive, ctx) } diff --git a/vortex-array/src/arrow/executor/temporal.rs b/vortex-array/src/arrow/executor/temporal.rs index 29bcc15763e..b3941b69916 100644 --- a/vortex-array/src/arrow/executor/temporal.rs +++ b/vortex-array/src/arrow/executor/temporal.rs @@ -157,7 +157,10 @@ where primitive.ptype() ); - let validity = primitive.validity_mask()?; + let validity = primitive + .as_ref() + .validity()? + .to_mask(primitive.as_ref().len(), ctx)?; let buffer = primitive.to_buffer::(); let values = buffer.into_arrow_scalar_buffer(); diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index 4c5eb5d9a61..5cf2f0fa9fe 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -12,6 +12,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; @@ -116,8 +118,17 @@ impl ArrayBuilder for BoolBuilder { let bool_array = array.to_bool(); self.inner.append_buffer(&bool_array.to_bit_buffer()); - self.nulls - .append_validity_mask(bool_array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + bool_array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index d3811314a04..9f33722dcb7 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -13,7 +13,9 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -202,8 +204,17 @@ impl ArrayBuilder for DecimalBuilder { .extend(decimal_array.buffer::().iter().copied()); }); - self.nulls - .append_validity_mask(decimal_array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + decimal_array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index f26281d5363..87b623f5f21 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -13,6 +13,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; @@ -243,8 +245,10 @@ impl ArrayBuilder for FixedSizeListBuilder { self.elements_builder.extend_from_array(fsl.elements()); self.nulls.append_validity_mask( array - .validity_mask() - .vortex_expect("validity_mask in extend_from_array_unchecked"), + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), ); } diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 5eb6d12223b..4c49185d380 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -14,6 +14,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::ListArray; use crate::arrays::listview::ListViewArrayExt; use crate::builders::ArrayBuilder; @@ -223,8 +225,10 @@ impl ArrayBuilder for ListBuilder { // Append validity information. self.nulls.append_validity_mask( array - .validity_mask() - .vortex_expect("validity_mask in extend_from_array_unchecked"), + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), ); // Note that `ListViewArray` has `n` offsets and sizes, not `n+1` offsets like `ListArray`. diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index a694bbbf035..c2f5c1471ee 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -20,7 +20,9 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -303,8 +305,13 @@ impl ArrayBuilder for ListViewBuilder { .vortex_expect("ListViewArray::rebuild(MakeExact) failed in extend_from_array"); debug_assert!(listview.is_zero_copy_to_list()); - self.nulls - .append_validity_mask(listview.listview_validity_mask()); + self.nulls.append_validity_mask( + array + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), + ); // Bulk append the new elements (which should have no gaps or overlaps). let old_elements_len = self.elements_builder.len(); diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index 8321c03500c..6952db2cfd2 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -12,6 +12,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -185,8 +187,17 @@ impl ArrayBuilder for PrimitiveBuilder { ); self.values.extend_from_slice(array.as_slice::()); - self.nulls - .append_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index 87f7ecb5e88..f37b88d7c80 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -13,6 +13,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::StructArray; use crate::arrays::struct_::StructArrayExt; use crate::builders::ArrayBuilder; @@ -175,8 +177,13 @@ impl ArrayBuilder for StructBuilder { builder.extend_from_array(a); } - self.nulls - .append_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + array + .validity() + .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, capacity: usize) { @@ -203,6 +210,8 @@ impl ArrayBuilder for StructBuilder { #[cfg(test)] mod tests { use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::assert_arrays_eq; @@ -246,7 +255,12 @@ mod tests { let struct_ = builder.finish(); assert_eq!(struct_.len(), 3); assert_eq!(struct_.dtype(), &dtype); - assert_eq!(struct_.valid_count().unwrap(), 1); + assert_eq!( + struct_ + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1 + ); } #[test] diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index e3adcb8f630..6d802696c74 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -20,6 +20,8 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::build_views::BinaryView; use crate::arrays::varbinview::compact::BufferUtilization; @@ -295,7 +297,17 @@ impl ArrayBuilder for VarBinViewBuilder { let array = array.to_varbinview(); self.flush_in_progress(); - self.push_only_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.push_only_validity_mask( + array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); let view_adjustment = self.completed @@ -312,7 +324,16 @@ impl ArrayBuilder for VarBinViewBuilder { .map(|view| adjustment.adjust_view(view)), ), ViewAdjustment::Rewriting(adjustment) => { - match array.validity_mask().vortex_expect("validity_mask") { + match array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") + { Mask::AllTrue(_) => { for (idx, &view) in array.views().iter().enumerate() { let new_view = self.push_view(view, &adjustment, &array, idx); diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index cab73acc521..b46a14d634b 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -129,7 +129,7 @@ fn test_cast_from_null(array: &ArrayRef) { fn test_cast_to_non_nullable(array: &ArrayRef) { if array - .invalid_count() + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("invalid_count should succeed in conformance test") == 0 { @@ -286,11 +286,15 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip }); assert_eq!( array - .validity_mask() - .vortex_expect("validity_mask should succeed in conformance test"), + .validity() + .vortex_expect("validity_mask should succeed in conformance test") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), casted - .validity_mask() + .validity() .vortex_expect("validity_mask should succeed in conformance test") + .to_mask(casted.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") ); for i in 0..array.len().min(10) { let original = array diff --git a/vortex-array/src/compute/conformance/consistency.rs b/vortex-array/src/compute/conformance/consistency.rs index dec1f8d1d61..9579454d24e 100644 --- a/vortex-array/src/compute/conformance/consistency.rs +++ b/vortex-array/src/compute/conformance/consistency.rs @@ -1037,10 +1037,10 @@ fn test_slice_aggregate_consistency(array: &ArrayRef) { // Test null count through invalid_count let sliced_invalid_count = sliced - .invalid_count() + .invalid_count(&mut ctx) .vortex_expect("invalid_count should succeed in conformance test"); let canonical_invalid_count = canonical_sliced - .invalid_count() + .invalid_count(&mut ctx) .vortex_expect("invalid_count should succeed in conformance test"); assert_eq!( sliced_invalid_count, canonical_invalid_count, diff --git a/vortex-array/src/compute/conformance/mask.rs b/vortex-array/src/compute/conformance/mask.rs index debdff9a806..5c448db126f 100644 --- a/vortex-array/src/compute/conformance/mask.rs +++ b/vortex-array/src/compute/conformance/mask.rs @@ -6,6 +6,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; @@ -279,7 +281,8 @@ fn test_nullable_mask_input(array: &ArrayRef) { let validity = crate::validity::Validity::from_iter(validity_values.clone()); let nullable_mask = BoolArray::new(bool_array.to_bit_buffer(), validity); - let mask_array = nullable_mask.to_mask_fill_null_false(); + let mask_array = + nullable_mask.to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); let masked = array .clone() .mask((!&mask_array).into_array()) diff --git a/vortex-array/src/mask.rs b/vortex-array/src/mask.rs index c23a6f79652..43fee47c486 100644 --- a/vortex-array/src/mask.rs +++ b/vortex-array/src/mask.rs @@ -34,7 +34,10 @@ impl Executable for Mask { } Columnar::Canonical(a) => { let bool = a.into_array().execute::(ctx)?; - let mask = bool.validity_mask()?; + let mask = bool + .as_ref() + .validity()? + .to_mask(bool.as_ref().len(), ctx)?; let bits = bool.into_bit_buffer(); // To handle nullable boolean arrays, we treat nulls as false in the mask. // TODO(ngates): is this correct? Feels like we should just force the caller to diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index d25d6446ce6..1d37d82fb75 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -763,7 +763,10 @@ impl Patches { take_indices_with_search_fn( patch_indices_slice, take_slice, - take_indices.validity_mask()?, + take_indices + .as_ref() + .validity()? + .to_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { self.search_index_chunked_batch( @@ -778,7 +781,10 @@ impl Patches { take_indices_with_search_fn( patch_indices_slice, take_slice, - take_indices.validity_mask()?, + take_indices + .as_ref() + .validity()? + .to_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { let Some(offset) = ::from(self.offset) else { @@ -1157,7 +1163,15 @@ mod test { ); assert_arrays_eq!(primitive_indices, PrimitiveArray::from_iter([0u64])); assert_eq!( - primitive_values.validity_mask().unwrap(), + primitive_values + .as_ref() + .validity() + .unwrap() + .to_mask( + primitive_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_iter(vec![true]) ); } @@ -1191,7 +1205,15 @@ mod test { assert_arrays_eq!(taken.indices(), PrimitiveArray::from_iter([0u64, 1])); assert_eq!( - primitive_values.validity_mask().unwrap(), + primitive_values + .as_ref() + .validity() + .unwrap() + .to_mask( + primitive_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_iter([true, false]) ); } diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index f88dbdaf356..539614874a9 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -223,7 +223,7 @@ impl ScalarFnVTable for CaseWhen { let condition = args.get(i * 2)?; let cond_bool = condition.execute::(ctx)?; - let cond_mask = cond_bool.to_mask_fill_null_false(); + let cond_mask = cond_bool.to_mask_fill_null_false(ctx); let effective_mask = &remaining & &cond_mask; if effective_mask.all_false() { diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 54949ef5d70..5277e4bf437 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -118,7 +118,7 @@ impl ScalarFnVTable for Zip { let mask = mask_array .execute::(ctx)? - .to_mask_fill_null_false(); + .to_mask_fill_null_false(ctx); let return_dtype = if_true .dtype() diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index bd2e3cf7dea..b9d7adc902b 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -179,7 +179,11 @@ impl StatsSetRef<'_> { }) .transpose()? } - Stat::NullCount => self.dyn_array_ref.invalid_count().ok().map(Into::into), + Stat::NullCount => self + .dyn_array_ref + .invalid_count(&mut ctx) + .ok() + .map(Into::into), Stat::IsConstant => { if self.dyn_array_ref.is_empty() { None diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index 6486a53735f..fe1fc88bb15 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -8,6 +8,8 @@ use goldenfile::differs::binary_diff; use itertools::Itertools; use vortex_error::VortexResult; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; @@ -26,7 +28,10 @@ pub fn check_metadata(name: &str, metadata: &[u8]) { /// Outputs the indices of the true values in a BoolArray pub fn to_int_indices(indices_bits: BoolArray) -> VortexResult> { let buffer = indices_bits.to_bit_buffer(); - let mask = indices_bits.validity_mask()?; + let mask = indices_bits.as_ref().validity()?.to_mask( + indices_bits.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(buffer .iter() .enumerate() diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index a5f6903e712..f86256128fd 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -151,22 +151,38 @@ impl Validity { pub fn take(&self, indices: &ArrayRef) -> VortexResult { match self { - Self::NonNullable => match indices.validity_mask()?.bit_buffer() { - AllOr::All => { - if indices.dtype().is_nullable() { - Ok(Self::AllValid) - } else { - Ok(Self::NonNullable) + Self::NonNullable => { + let len = indices.len(); + let indices_mask = match indices.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + match indices_mask.bit_buffer() { + AllOr::All => { + if indices.dtype().is_nullable() { + Ok(Self::AllValid) + } else { + Ok(Self::NonNullable) + } } + AllOr::None => Ok(Self::AllInvalid), + AllOr::Some(buf) => Ok(Validity::from(buf.clone())), } - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - }, - Self::AllValid => match indices.validity_mask()?.bit_buffer() { - AllOr::All => Ok(Self::AllValid), - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - }, + } + Self::AllValid => { + let len = indices.len(); + let indices_mask = match indices.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + match indices_mask.bit_buffer() { + AllOr::All => Ok(Self::AllValid), + AllOr::None => Ok(Self::AllInvalid), + AllOr::Some(buf) => Ok(Validity::from(buf.clone())), + } + } Self::AllInvalid => Ok(Self::AllInvalid), Self::Array(is_valid) => { let maybe_is_valid = is_valid.take(indices.clone())?; @@ -208,11 +224,11 @@ impl Validity { /// Converts this validity into a [`Mask`] of the given length. /// /// Valid elements are `true` and invalid elements are `false`. - pub fn to_mask(&self, length: usize) -> Mask { + pub fn to_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult { match self { - Self::NonNullable | Self::AllValid => Mask::new_true(length), - Self::AllInvalid => Mask::new_false(length), - Self::Array(a) => a.to_bool().to_mask(), + Self::NonNullable | Self::AllValid => Ok(Mask::new_true(length)), + Self::AllInvalid => Ok(Mask::new_false(length)), + Self::Array(arr) => arr.clone().execute::(ctx), } } @@ -376,10 +392,13 @@ impl Validity { /// Create Validity by copying the given array's validity. #[inline] pub fn copy_from_array(array: &ArrayRef) -> VortexResult { - Ok(Validity::from_mask( - array.validity_mask()?, - array.dtype().nullability(), - )) + let len = array.len(); + let mask = match array.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + Ok(Validity::from_mask(mask, array.dtype().nullability())) } /// Create Validity from boolean array with given nullability of the array. diff --git a/vortex-array/src/variants.rs b/vortex-array/src/variants.rs index f60eea316a9..257d6287b0c 100644 --- a/vortex-array/src/variants.rs +++ b/vortex-array/src/variants.rs @@ -100,7 +100,9 @@ impl ArrayRef { .clone() .fill_null(Scalar::bool(false, self.dtype().nullability()))?; - Ok(array.execute::(ctx)?.to_mask_fill_null_false()) + Ok(array + .execute::(ctx)? + .to_mask_fill_null_false(ctx)) } } diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 3d2557c63c7..6c204c7263a 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -107,7 +107,11 @@ impl Scheme for ALPScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let alp_encoded = alp_encode(data.array_as_primitive(), None)?; + let alp_encoded = alp_encode( + data.array_as_primitive(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; // Compress the ALP ints. let compressed_alp_ints = diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index 4f357f048aa..d393db58cb1 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -3,6 +3,8 @@ //! Bool compression statistics. +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_error::VortexResult; @@ -42,7 +44,10 @@ impl BoolStats { }); } - let validity = input.validity_mask()?; + let validity = input.as_ref().validity()?.to_mask( + input.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 37b06c52eca..8a1a85075ab 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -8,6 +8,8 @@ use std::hash::Hash; use itertools::Itertools; use num_traits::Float; use rustc_hash::FxBuildHasher; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::NativePType; @@ -203,7 +205,10 @@ where HashSet::with_hasher(FxBuildHasher) }; - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let mut runs = 1; let head_idx = validity diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index 2203cad30ef..af99efe3c0a 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -7,6 +7,8 @@ use std::hash::Hash; use num_traits::PrimInt; use rustc_hash::FxBuildHasher; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::IntegerPType; @@ -349,7 +351,10 @@ where }); } - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 2ae0434b243..917fd1b7cd4 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -18,7 +18,9 @@ use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use cudarc::driver::sys::CUevent_flags; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::scalar::Scalar; @@ -373,7 +375,12 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable); // Encode: ALP → FoR → BitPacked - let alp = alp_encode(float_prim.as_view(), Some(exponents)).vortex_expect("alp_encode"); + let alp = alp_encode( + float_prim.as_view(), + Some(exponents), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("alp_encode"); assert!(alp.patches().is_none()); let for_arr = FoRData::encode(alp.encoded().to_primitive()).vortex_expect("for encode"); let bp = diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 2bc12a536d0..67167b57ba4 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -519,6 +519,8 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::session::VortexSession; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use super::*; use crate::CanonicalCudaExt; @@ -874,7 +876,11 @@ mod tests { .collect(); let float_prim = PrimitiveArray::new(Buffer::from(floats.clone()), NonNullable); - let alp = alp_encode(float_prim.as_view(), Some(exponents))?; + let alp = alp_encode( + float_prim.as_view(), + Some(exponents), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!(alp.patches().is_none()); let for_arr = FoR::encode(alp.encoded().to_primitive())?; let bp = BitPacked::encode(for_arr.encoded(), 6)?; diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 2f6e2a74901..fae81d76ae8 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -121,6 +121,8 @@ where #[cfg(test)] mod tests { use vortex::array::IntoArray; + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::assert_arrays_eq; use vortex::array::patches::Patches; @@ -205,7 +207,11 @@ mod tests { Some(5.0), ]; let prim = PrimitiveArray::from_option_iter(values); - let alp_array = alp_encode(prim.as_view(), None)?; + let alp_array = alp_encode( + prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let cpu_result = alp_array.to_canonical()?.into_array(); @@ -232,7 +238,11 @@ mod tests { Buffer::from(vec![1.0f32, 2.0, 3.0, 4.0, 5.0]), Validity::AllValid, ); - let alp_array = alp_encode(values.as_view(), None)?; + let alp_array = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let cpu_result = alp_array.to_canonical()?.into_array(); diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index 029a5ae0638..c63687037a4 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -375,7 +375,9 @@ pub fn data_chunk_to_vortex( mod tests { use std::ffi::CString; + use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex::array::arrays::listview::ListViewArrayExt; @@ -506,7 +508,15 @@ mod tests { assert_eq!(values_slice, values); assert_eq!( - vortex_values.validity_mask().unwrap(), + vortex_values + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } @@ -611,7 +621,15 @@ mod tests { assert_eq!(vortex_slice, values); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } @@ -778,7 +796,15 @@ mod tests { PrimitiveArray::from_option_iter([Some(1i32), Some(2), Some(3), Some(4)]) ); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(2, vec![0]) ); } @@ -891,7 +917,15 @@ mod tests { assert_eq!(sizes.as_slice::()[1], 0); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 5b52f6d6c53..66c96e3a2d0 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -135,6 +135,7 @@ pub fn precision_to_duckdb_storage_size(decimal_dtype: &DecimalDType) -> VortexR #[cfg(test)] mod tests { + use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DecimalArray; use vortex::dtype::DecimalDType; @@ -148,7 +149,10 @@ mod tests { pub(crate) fn new_zero_copy_exporter( array: &DecimalArray, ) -> VortexResult> { - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dest_values_type = precision_to_duckdb_storage_size(&array.decimal_dtype())?; assert_eq!(array.values_type(), dest_values_type); diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index 6377906940c..d576a85c148 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -42,16 +42,19 @@ pub(crate) fn new_exporter_with_flatten( ) -> VortexResult> { // Grab the cache dictionary values. let values = array.values(); + let codes = array.codes(); + let codes_len = codes.len(); + if let Some(constant) = values.as_opt::() { return constant::new_exporter_with_mask( - ConstantArray::new(constant.scalar().clone(), array.codes().len()), - array.codes().validity_mask()?, + ConstantArray::new(constant.scalar().clone(), codes_len), + codes.validity()?.to_mask(codes_len, ctx)?, cache, ctx, ); } - let codes_mask = array.codes().validity_mask()?; + let codes_mask = codes.validity()?.to_mask(codes_len, ctx)?; match codes_mask { Mask::AllTrue(_) => {} diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 5f684622ee6..0a196598171 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -10,7 +10,9 @@ use std::sync::Arc; use paste::paste; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::NullArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::struct_::StructArrayExt; @@ -251,7 +253,7 @@ pub unsafe extern "C-unwind" fn vx_array_invalid_count( try_or_default(error_out, || { vortex_ensure!(!array.is_null()); let array = vx_array::as_ref(array); - array.invalid_count() + array.invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) }) } diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index c45f232b920..88c7286b3b4 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -28,7 +28,9 @@ use jni::sys::jshort; use jni::sys::jstring; use vortex::array::ArrayRef; use vortex::array::ArrayView; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::VarBin; use vortex::array::arrays::VarBinView; use vortex::array::arrays::extension::ExtensionArrayExt; @@ -280,7 +282,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNullCount( ) -> jint { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let count = array_ref.inner.invalid_count()?; + let count = array_ref + .inner + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx())?; Ok(jint::try_from(count).unwrap_or(-1)) }) } diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 1ffb669d08e..c301a696784 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -260,7 +260,9 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::IntoArray as _; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinArray; @@ -516,7 +518,12 @@ mod tests { .unwrap() .await .unwrap(); - let expected = array.validity_mask().unwrap().into_array(); + let expected = array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); assert_arrays_eq!( actual .to_canonical() diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 0d804ee6fb8..e8c78c2e2a7 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -199,8 +199,10 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; @@ -393,7 +395,12 @@ mod tests { .unwrap(); assert_eq!( - result.validity_mask().unwrap().bit_buffer(), + result + .validity() + .unwrap() + .to_mask(result.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .bit_buffer(), AllOr::Some(&validity_boolean_buffer) ); assert_eq!( diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index f8a0cee3581..0c602cd78fa 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -16,7 +16,9 @@ use itertools::Itertools; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Field; @@ -239,7 +241,10 @@ impl LayoutStrategy for TableStrategy { if is_nullable { columns.push(( sequence_pointer.advance(), - chunk.validity_mask()?.into_array(), + chunk + .validity()? + .to_mask(chunk.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .into_array(), )); } diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index f0cc68e2f34..6f6a0666d8c 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -24,7 +24,9 @@ use pyo3::types::PyRangeMethods; use pyo3_bytes::PyBytes; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::Chunked; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::chunked::ChunkedArrayExt; @@ -525,7 +527,9 @@ impl PyArray { /// ``` fn filter(slf: Bound, mask: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); - let mask = (&*mask as &ArrayRef).to_bool().to_mask_fill_null_false(); + let mask = (&*mask as &ArrayRef) + .to_bool() + .to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); let inner = slf.filter(mask)?.to_canonical()?.into_array(); Ok(PyArrayRef::from(inner)) } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs index 6cf60cf4270..4c4175b773d 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -130,17 +132,72 @@ impl FlatLayoutFixture for AlpFixture { "f64_boundary_specials", ]), vec![ - alp_encode(f64_prices.as_view(), None)?.into_array(), - alp_encode(f32_near_int.as_view(), None)?.into_array(), - alp_encode(f64_negative_near_int.as_view(), None)?.into_array(), - alp_encode(f64_currency.as_view(), None)?.into_array(), - alp_encode(f64_nullable.as_view(), None)?.into_array(), - alp_encode(f64_patched.as_view(), None)?.into_array(), - alp_encode(f64_patch_heavy.as_view(), None)?.into_array(), - alp_encode(f64_special_values.as_view(), None)?.into_array(), - alp_encode(f32_special_values.as_view(), None)?.into_array(), - alp_encode(f64_extremes.as_view(), None)?.into_array(), - alp_encode(f64_boundary_specials.as_view(), None)?.into_array(), + alp_encode( + f64_prices.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f32_near_int.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_negative_near_int.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_currency.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_nullable.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_patched.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_patch_heavy.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_special_values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f32_special_values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_extremes.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_boundary_specials.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), ], N, Validity::NonNullable, diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 54fe1b61926..4a0edf99389 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -14,7 +14,9 @@ use rand::RngExt; use rand::SeedableRng; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; @@ -102,7 +104,12 @@ mod setup { /// Create ALP <- FoR <- BitPacked encoding tree for f64 pub fn alp_for_bp_f64() -> ArrayRef { let (_, _, float_array) = setup_primitive_arrays(); - let alp_compressed = alp_encode(float_array.as_view(), None).unwrap(); + let alp_compressed = alp_encode( + float_array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); // Manually construct ALP <- FoR <- BitPacked tree let for_array = FoR::encode(alp_compressed.encoded().to_primitive()).unwrap(); diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 4cedf23c07d..3c8777bc3d3 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -15,6 +15,7 @@ use rand::SeedableRng; use rand::prelude::IndexedRandom; use rand::rngs::StdRng; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; @@ -257,13 +258,25 @@ fn bench_alp_compress_f64(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &float_array) - .bench_refs(|a| alp_encode(a.as_view(), None).unwrap()); + .bench_refs(|a| { + alp_encode( + a.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + }); } #[divan::bench(name = "alp_decompress_f64")] fn bench_alp_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); - let compressed = alp_encode(float_array.as_view(), None).unwrap(); + let compressed = alp_encode( + float_array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &compressed) From ee2af71a10104d93c581c2f60b98ef1eb7bd4590 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 15 Apr 2026 23:24:35 +0100 Subject: [PATCH 057/250] chore: sort and dedup formats in sql engine runner (#7459) Dedup formats in SQL engine runner to prevent benchmarking the same format multiple times, as passed on the CLI. Signed-off-by: Alexander Droste --- vortex-bench/src/runner.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index 6c9064b6a29..dc7d729232e 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -13,6 +13,7 @@ use std::time::Instant; use indicatif::ProgressBar; use vortex::error::vortex_panic; +use vortex::utils::aliases::hash_set::HashSet; use crate::Benchmark; use crate::BenchmarkDataset; @@ -67,6 +68,7 @@ pub struct SqlBenchmarkRunner { benchmark_dataset: BenchmarkDataset, storage: String, expected_row_counts: Option>, + /// Deduplicated, preserving insertion order. formats: Vec, memory_tracker: Option, hide_progress_bar: bool, @@ -79,10 +81,12 @@ impl SqlBenchmarkRunner { pub fn new( benchmark: &B, engine: Engine, - formats: Vec, + formats: impl IntoIterator, track_memory: bool, hide_progress_bar: bool, ) -> anyhow::Result { + let mut seen = HashSet::new(); + let formats: Vec = formats.into_iter().filter(|f| seen.insert(*f)).collect(); let storage = url_scheme_to_storage(benchmark.data_url())?; let memory_tracker = track_memory.then(BenchmarkMemoryTracker::new); From e8374a416ccedb08eaf8d85957feb5f51b29a0e8 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 16 Apr 2026 10:17:32 +0100 Subject: [PATCH 058/250] chore: bump duckdb to 1.5.2 (#7461) Signed-off-by: Alexander Droste --- .github/workflows/bench-pr.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/sql-benchmarks.yml | 2 +- docs/user-guide/duckdb.md | 8 ++++---- vortex-duckdb/README.md | 4 ++-- vortex-duckdb/build.rs | 2 +- vortex-duckdb/src/convert/dtype.rs | 1 + 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 34610171419..1a5de341360 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -49,7 +49,7 @@ jobs: - name: Install DuckDB run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.4.2/duckdb_cli-linux-amd64.zip | funzip > duckdb + wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.2/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb echo "$PWD" >> $GITHUB_PATH diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 645faf9fb37..0b3874d0c68 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -63,7 +63,7 @@ jobs: - name: Install DuckDB run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.4.2/duckdb_cli-linux-amd64.zip | funzip > duckdb + wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.2/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb echo "$PWD" >> $GITHUB_PATH diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 2492e3d35e4..59c27a7ca70 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -131,7 +131,7 @@ jobs: - name: Install DuckDB run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.4.2/duckdb_cli-linux-amd64.zip | funzip > duckdb + wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.2/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb echo "$PWD" >> $GITHUB_PATH diff --git a/docs/user-guide/duckdb.md b/docs/user-guide/duckdb.md index 19631816165..3fc0b03be9b 100644 --- a/docs/user-guide/duckdb.md +++ b/docs/user-guide/duckdb.md @@ -46,10 +46,10 @@ COPY (SELECT * FROM my_table) TO 'output.vortex' (FORMAT vortex); Controls which filesystem implementation is used for reading and writing Vortex files. -| Value | Description | -|-------|-------------| -| `'vortex'` (default) | Uses Vortex's built-in object store filesystem. Supports `file://` and `s3://` schemes. | -| `'duckdb'` | Uses DuckDB's built-in filesystem, including any filesystem extensions such as `httpfs`. | +| Value | Description | +| -------------------- | ---------------------------------------------------------------------------------------- | +| `'vortex'` (default) | Uses Vortex's built-in object store filesystem. Supports `file://` and `s3://` schemes. | +| `'duckdb'` | Uses DuckDB's built-in filesystem, including any filesystem extensions such as `httpfs`. | ```sql SET vortex_filesystem = 'duckdb'; diff --git a/vortex-duckdb/README.md b/vortex-duckdb/README.md index 416bbac7fcb..9230e2420be 100644 --- a/vortex-duckdb/README.md +++ b/vortex-duckdb/README.md @@ -64,8 +64,8 @@ By default, our tests use a precompiled build which means you don't get an 2. If there is an api difference between duckdb-vortex's duckdb submodule and vortex's vortex-duckdb/duckdb submodule, checkout duckdb-vortex to previous - commit. For example, if duckdb-vortex's HEAD uses 1.5 API but vortex's HEAD - uses 1.4.2, checkout duckdb-vortex at 8a41ee6ebd9. + commit. For example, if duckdb-vortex's HEAD uses 1.6 API but vortex's HEAD + uses 1.5.2, checkout duckdb-vortex at 8a41ee6ebd9. 3. Update duckdb-vortex's submodules. Replace vortex/ submodule by a softlink to your local vortex repository. diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs index 3ef95f3224a..e23edf5ad72 100644 --- a/vortex-duckdb/build.rs +++ b/vortex-duckdb/build.rs @@ -383,7 +383,7 @@ fn main() { // e.g. reordering fields in C++ structs. let version = env::var("DUCKDB_VERSION") // You can also change this version to a commit hash - .unwrap_or_else(|_| "1.5.0".to_owned()); + .unwrap_or_else(|_| "1.5.2".to_owned()); let version = DuckDBVersion::from(&version); match &version { DuckDBVersion::Release(v) => println!("cargo:info=Using DuckDB release version: {v}"), diff --git a/vortex-duckdb/src/convert/dtype.rs b/vortex-duckdb/src/convert/dtype.rs index 401b31a90c6..2b7c656be64 100644 --- a/vortex-duckdb/src/convert/dtype.rs +++ b/vortex-duckdb/src/convert/dtype.rs @@ -171,6 +171,7 @@ impl FromLogicalType for DType { DUCKDB_TYPE::DUCKDB_TYPE_BIGNUM => todo!(), DUCKDB_TYPE::DUCKDB_TYPE_STRING_LITERAL => todo!(), DUCKDB_TYPE::DUCKDB_TYPE_INTEGER_LITERAL => todo!(), + DUCKDB_TYPE::DUCKDB_TYPE_GEOMETRY => todo!(), }) } } From 17e28017c7ebfd14439176e241db843cc175dde2 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 16 Apr 2026 11:20:07 +0100 Subject: [PATCH 059/250] chore: unify ci timeouts to 4 tiers (#7460) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit all GitHub Actions workflow timeouts against actual run durations and consolidate to 4 consistent tiers: - 10 min: quick utility jobs (labels, lints, deploys, reports) - 30 min: standard CI build/test/lint, docs build - 120 min: heavy compute (benchmarks, packaging, publishing, fuzz-coverage) - 240 min: marathon fuzz runs and corpus minimization Notably, fuzz-coverage was hitting its old 60-min timeout and getting cancelled on most runs — bumping to 120 fixes that. ### Timeout tiers | Tier | Timeout | Purpose | Safety margin | |------|---------|---------|---------------| | ⚡ | **10 min** | Quick utility jobs (labels, lints, deploys, reports) | ~100×+ over typical | | 🔨 | **30 min** | Standard CI build/test/lint, docs, packaging, publishing | ~5× over slowest job in tier | | 🏋️ | **120 min** | Heavy compute (benchmarks, python packaging, fuzz-coverage) | ~2× over slowest job in tier | | 🔬 | **240 min** | Marathon fuzz runs and corpus minimization | Matches libfuzzer/wasmfuzz time configs | ### Per-job breakdown #### ⚡ Tier 1 — 10 min | Workflow | Job | Previous timeout | Max observed duration | |----------|-----|-----------------|----------------------| | `approvals.yml` | `check-approvals` | ❌ none (6h default) | < 1 min | | `bench-dispatch.yml` | `remove-bench-label` | 2 min | < 1 min | | `bench-dispatch.yml` | `remove-sql-label` | 2 min | < 1 min | | `bench.yml` | `commit-metadata` | 10 min | < 1 min | | `ci.yml` | `lint-toml` | 40 min | ~9 sec | | `ci.yml` | `validate-workflow-yaml` | 40 min | ~8 sec | | `ci.yml` | `python-lint` | 40 min | ~39 sec | | `ci.yml` | `cpp-lint` | 10 min | ~8 sec | | `ci.yml` | `ffi-c-test` | 10 min | ~49 sec | | `docs.yml` | `deploy` | 10 min | ~31 sec | | `labels.yml` | `check_changelog_label` | ❌ none | < 1 min | | `package.yml` | `prepare-all` | ❌ none | < 1 min | | `publish-benchmarks-website.yml` | `publish` | ❌ none | < 1 min | | `release-drafter.yml` | `update_release_draft` | ❌ none | < 1 min | | `report-fuzz-crash.yml` | `report` | 10 min | < 1 min | | `reuse.yml` | `reuse-check` | ❌ none | < 1 min | | `stale.yml` | `close-issues` | 10 min | < 1 min | | `typos.yml` | `spelling` | ❌ none | < 1 min | | `web.yml` | `changes` | ❌ none | < 1 min | | `web.yml` | `deploy` | 10 min | < 1 min | #### 🔨 Tier 2 — 30 min | Workflow | Job | Previous timeout | Max observed duration | |----------|-----|-----------------|----------------------| | `ci.yml` | `python-test` | 40 min | ~2 min 10 sec | | `ci.yml` | `python-wheel-build` | 40 min | ~2 min 44 sec | | `ci.yml` | `rust-docs` | 40 min | ~2 min 20 sec | | `ci.yml` | `build-rust` (all matrix entries) | 40 min | ~2 min 18 sec | | `ci.yml` | `check-min-deps` | 40 min | ~1 min 23 sec | | `ci.yml` | `rust-lint` | 40 min | ~2 min 22 sec | | `ci.yml` | `rust-lint-no-default` | 40 min | ~1 min 35 sec | | `ci.yml` | `public-api` | 40 min | ~1 min 17 sec | | `ci.yml` | `rust-coverage` | 40 min | ~4 min 47 sec | | `ci.yml` | `rust-test-sanitizer` | 40 min | ~3 min 19 sec | | `ci.yml` | `rust-ffi-test-sanitizer` | 40 min | ~1 min 16 sec | | `ci.yml` | `cuda-build-lint` | 40 min | ~3 min 9 sec | | `ci.yml` | `cuda-test` | 30 min | ~3 min 32 sec | | `ci.yml` | `cuda-test-sanitizer` | 30 min | ~3 min 14 sec | | `ci.yml` | `cuda-test-cudf` | 30 min | ~2 min | | `ci.yml` | `rust-test-other` (windows/arm64) | 40 min | ~5 min 58 sec | | `ci.yml` | `build-java` | 40 min | ~2 min 24 sec | | `ci.yml` | `bench-codspeed` (all shards) | 40 min | ~2 min 37 sec | | `ci.yml` | `license-check-and-audit-check` | 40 min | ~57 sec | | `ci.yml` | `cxx-test` | 40 min | ~1 min 28 sec | | `ci.yml` | `sqllogic-test` | ❌ none (6h default) | ~1 min 38 sec | | `ci.yml` | `wasm-integration` | 40 min | ~2 min 16 sec | | `ci.yml` | `miri` | 40 min | ~1 min 44 sec | | `ci.yml` | `generated-files` | 40 min | ~45 sec | | `ci.yml` | `check-java-publish-build` | 40 min | ~2 min 47 sec | | `ci.yml` | `rust-publish-dry-run` | 40 min | ~36 sec | | `close-fixed-fuzzer-issues.yml` | `close-fixed` | 60 min | ~2 min | | `compat-gen-upload.yml` | `dry-run` | ❌ none | ~3 min | | `compat-gen-upload.yml` | `upload` | ❌ none | ~1 min 18 sec | | `compat-validation.yml` | `compat-test` | 120 min | ~3 min 20 sec | | `docs.yml` | `build` | 120 min | ~15 min | | `package.yml` | `prepare-java-macos` | ❌ none | ~6 min | | `package.yml` | `prepare-java-linux` | 120 min | ~6 min 22 sec | | `publish.yml` | `publish-rust` | 120 min | ~1 min 37 sec | | `publish.yml` | `publish-python` | 120 min | ~34 sec | | `publish.yml` | `publish-java` | 120 min | ~10 min 12 sec | | `web.yml` | `check` | 30 min | ~3 min | | `web.yml` | `build` | 30 min | ~3 min | #### 🏋️ Tier 3 — 120 min | Workflow | Job | Previous timeout | Max observed duration | |----------|-----|-----------------|----------------------| | `bench-pr.yml` | `bench` | 120 min | ~42 min | | `bench.yml` | `bench` | 120 min | ~42 min | | `claude.yml` | `claude` | ❌ none | variable (AI agent) | | `fuzz-coverage.yml` | `coverage` | 60 min ⚠️ | **~60 min (was hitting timeout!)** | | `fuzzer-fix-automation.yml` | `attempt-fix` | 90 min | ~30 min | | `package.yml` | `prepare-python` | 120 min | ~40 min | | `release-binaries.yml` | `build` | 120 min | ~40 min | | `sql-benchmarks.yml` | `bench` | 120 min | ~60 min (nightly SF=100) | #### 🔬 Tier 4 — 240 min | Workflow | Job | Previous timeout | Max observed duration | |----------|-----|-----------------|----------------------| | `run-fuzzer.yml` | `fuzz` | 230 min | ~50 min (libfuzzer `max_total_time=2700s`) | | `wasm-fuzz.yml` | `wasm-fuzz` | 270 min | up to 3h30m (wasmfuzz `--timeout=3h30m`) | | `minimize_fuzz_corpus_workflow.yml` | `minimize` | ❌ none (6h default) | variable | Also lowered wasmfuzz `--timeout` from `4h` to `3h30m` so the fuzz step itself fits within 240 min with ~30 min headroom for build/setup/teardown. ### Notable fixes - **`fuzz-coverage` was silently broken**: the old 60-min timeout was being hit on 4 out of 5 recent runs (all cancelled at ~61 min). Bumping to 120 min fixes this. - **14 jobs had no timeout at all**, defaulting to the GitHub Actions 6-hour maximum. All now have explicit timeouts. - **`sqllogic-test`** in CI had no timeout — now set to 30 min. Signed-off-by: Alexander Droste --- .github/workflows/approvals.yml | 1 + .github/workflows/bench-dispatch.yml | 4 +- .github/workflows/ci.yml | 51 ++++++++++--------- .github/workflows/claude.yml | 1 + .../workflows/close-fixed-fuzzer-issues.yml | 2 +- .github/workflows/compat-gen-upload.yml | 2 + .github/workflows/compat-validation.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/fuzz-coverage.yml | 2 +- .github/workflows/fuzzer-fix-automation.yml | 2 +- .github/workflows/labels.yml | 1 + .../minimize_fuzz_corpus_workflow.yml | 1 + .github/workflows/package.yml | 4 +- .../workflows/publish-benchmarks-website.yml | 1 + .github/workflows/publish.yml | 6 +-- .github/workflows/release-drafter.yml | 1 + .github/workflows/reuse.yml | 1 + .github/workflows/run-fuzzer.yml | 2 +- .github/workflows/typos.yml | 1 + .github/workflows/wasm-fuzz.yml | 4 +- .github/workflows/web.yml | 1 + 21 files changed, 53 insertions(+), 39 deletions(-) diff --git a/.github/workflows/approvals.yml b/.github/workflows/approvals.yml index 8939e8dec66..57439b97b32 100644 --- a/.github/workflows/approvals.yml +++ b/.github/workflows/approvals.yml @@ -7,6 +7,7 @@ on: jobs: check-approvals: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Check required approvals uses: actions/github-script@450193c5abd4cdb17ba9f3ffcfe8f635c4bb6c2a diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/bench-dispatch.yml index 264dfaf94e8..e18bcdca211 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/bench-dispatch.yml @@ -18,7 +18,7 @@ permissions: jobs: remove-bench-label: runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 10 if: github.event.label.name == 'action/benchmark' steps: - uses: actions-ecosystem/action-remove-labels@v1 @@ -34,7 +34,7 @@ jobs: remove-sql-label: runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 10 if: github.event.label.name == 'action/benchmark-sql' steps: - uses: actions-ecosystem/action-remove-labels@v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5126ed3e392..17c557c8480 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,14 +28,14 @@ env: jobs: lint-toml: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - uses: spiraldb/actions/.github/actions/lint-toml@0.18.5 validate-workflow-yaml: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: Validate YAML file @@ -52,7 +52,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=python-lint', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 10 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -79,7 +79,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=python-test', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 30 env: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" @@ -117,7 +117,7 @@ jobs: python-wheel-build: name: "Python (wheel build)" runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 - name: Rust Dependency Cache @@ -157,7 +157,7 @@ jobs: rust-docs: name: "Rust (docs)" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/tag=rust-docs', github.run_id) @@ -177,7 +177,7 @@ jobs: build-rust: name: "Rust build (${{matrix.config.name}})" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner={1}/image=ubuntu24-full-x64-pre-v2/tag={2}', github.run_id, matrix.config.runner, matrix.config.name) @@ -225,7 +225,7 @@ jobs: check-min-deps: name: "Check build with minimal dependencies" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-min-deps', github.run_id) @@ -241,7 +241,7 @@ jobs: rust-lint: name: "Rust (lint)" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-lint', github.run_id) @@ -282,7 +282,7 @@ jobs: rust-lint-no-default: name: "Rust (lint, no default)" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-lint-no-default', github.run_id) @@ -301,7 +301,7 @@ jobs: public-api: name: "Public API lock files" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=public-api', github.run_id) @@ -335,7 +335,7 @@ jobs: rust-coverage: name: "Rust tests (coverage) (${{ matrix.suite }})" - timeout-minutes: 40 + timeout-minutes: 30 permissions: id-token: write strategy: @@ -398,7 +398,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 30 env: ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" LSAN_OPTIONS: "report_objects=1" @@ -459,7 +459,7 @@ jobs: - sanitizer: tsan sanitizer_flags: "-Zsanitizer=thread" name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})" - timeout-minutes: 40 + timeout-minutes: 30 env: ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" LSAN_OPTIONS: "report_objects=1" @@ -511,7 +511,7 @@ jobs: cuda-build-lint: if: github.repository == 'vortex-data/vortex' name: "CUDA build & lint" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build steps: - uses: runs-on/action@v2 @@ -642,7 +642,7 @@ jobs: rust-test-other: name: "Rust tests (${{ matrix.os }})" - timeout-minutes: 40 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -700,7 +700,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=java', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 30 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -724,7 +724,7 @@ jobs: - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" } name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) @@ -757,7 +757,7 @@ jobs: license-check-and-audit-check: name: License Check and Audit Check runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 30 strategy: matrix: checks: @@ -773,7 +773,7 @@ jobs: cxx-test: name: "C++ build" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=cxx-build', github.run_id) @@ -806,6 +806,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id) || 'ubuntu-latest' }} + timeout-minutes: 30 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -824,7 +825,7 @@ jobs: wasm-integration: name: "WASM integration smoke test" runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup-rust @@ -847,7 +848,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 30 env: MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE RUSTFLAGS: "-A warnings" @@ -870,7 +871,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=generated-files', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 40 + timeout-minutes: 30 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -932,7 +933,7 @@ jobs: runs-on: ${{ matrix.target.runs-on }} container: image: "ubuntu:20.04" - timeout-minutes: 40 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -963,7 +964,7 @@ jobs: rust-publish-dry-run: name: "Rust publish dry-run" - timeout-minutes: 40 + timeout-minutes: 30 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 448cc5e2c50..79553228a71 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -23,6 +23,7 @@ jobs: (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) runs-on: ubuntu-latest + timeout-minutes: 120 permissions: contents: read pull-requests: read diff --git a/.github/workflows/close-fixed-fuzzer-issues.yml b/.github/workflows/close-fixed-fuzzer-issues.yml index 44d51663bb5..e660e8a9860 100644 --- a/.github/workflows/close-fixed-fuzzer-issues.yml +++ b/.github/workflows/close-fixed-fuzzer-issues.yml @@ -29,7 +29,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=arm64-medium/disk=large/tag=fuzzer-cleanup-{1}', github.run_id, matrix.target) || 'ubuntu-latest' }} - timeout-minutes: 60 + timeout-minutes: 30 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' diff --git a/.github/workflows/compat-gen-upload.yml b/.github/workflows/compat-gen-upload.yml index da17ec9a28c..6ec81264ba9 100644 --- a/.github/workflows/compat-gen-upload.yml +++ b/.github/workflows/compat-gen-upload.yml @@ -27,6 +27,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-gen-dry-run', github.run_id) || 'ubuntu-latest' }} + timeout-minutes: 30 permissions: id-token: write contents: read @@ -77,6 +78,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-gen-upload', github.run_id) || 'ubuntu-latest' }} + timeout-minutes: 30 environment: compat-upload permissions: id-token: write diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index f479420cd48..da6be24f97f 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -27,7 +27,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=compat-validation', github.run_id) || 'ubuntu-latest' }} - timeout-minutes: 120 + timeout-minutes: 30 steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 307526d92f3..b1870c2e7f6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,7 +17,7 @@ permissions: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 30 steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup-rust diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index 3aab125d671..0b3c2c14e54 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false matrix: fuzz_target: [array_ops, file_io, compress_roundtrip] - timeout-minutes: 60 + timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=arm64-medium/disk=large', github.run_id) diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index 40691fec51a..9eca51e865e 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -32,7 +32,7 @@ jobs: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 permissions: contents: write diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index a6d4e9364e3..e6eb3f56e40 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -13,6 +13,7 @@ jobs: check_changelog_label: name: Validate Changelog Label runs-on: ubuntu-latest + timeout-minutes: 10 permissions: pull-requests: read # Grant permission to read PR information steps: diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index 47475c8e8f9..9193b3d1d68 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -43,6 +43,7 @@ jobs: ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=arm64-medium/disk=large/tag={1}-minimize', github.run_id, inputs.fuzz_target) || 'ubuntu-latest' }} + timeout-minutes: 240 steps: - name: Bail if not on develop if: github.ref != 'refs/heads/develop' diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index fbaca0f8407..e137726b26c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -84,6 +84,7 @@ jobs: prepare-java-macos: runs-on: "macos-latest" + timeout-minutes: 30 steps: - uses: actions/checkout@v6 with: @@ -108,7 +109,7 @@ jobs: runs-on: ${{ matrix.target.runs-on }} container: image: "ubuntu:20.04" - timeout-minutes: 120 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -140,5 +141,6 @@ jobs: prepare-all: needs: [prepare-python, prepare-java-macos, prepare-java-linux] runs-on: ubuntu-latest + timeout-minutes: 10 steps: - run: echo "All packages built successfully" diff --git a/.github/workflows/publish-benchmarks-website.yml b/.github/workflows/publish-benchmarks-website.yml index 824ff80e6a8..e7eeefb8ecc 100644 --- a/.github/workflows/publish-benchmarks-website.yml +++ b/.github/workflows/publish-benchmarks-website.yml @@ -9,6 +9,7 @@ on: jobs: publish: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read packages: write diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2d3e2388922..2bb9a4135d0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,7 +20,7 @@ jobs: publish-rust: runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 30 needs: [package] steps: - uses: actions/checkout@v6 @@ -49,7 +49,7 @@ jobs: publish-python: needs: [package] runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 30 permissions: id-token: write # IMPORTANT: mandatory for trusted publishing deployments: write @@ -76,7 +76,7 @@ jobs: publish-java: needs: [package] runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 30 env: # Picked up by gradle-git-version GIT_VERSION: ${{ github.event.release.tag_name }} diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index df9bc92e455..f87f34483b2 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -25,6 +25,7 @@ jobs: # write permission is required to create a github release contents: write runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: release-drafter/release-drafter@v7.1.1 with: diff --git a/.github/workflows/reuse.yml b/.github/workflows/reuse.yml index e2891f48165..6003dc47354 100644 --- a/.github/workflows/reuse.yml +++ b/.github/workflows/reuse.yml @@ -15,6 +15,7 @@ on: jobs: reuse-check: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - name: REUSE Compliance Check diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index de1f1e0bf3b..959e6b345d7 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -65,7 +65,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" - timeout-minutes: 230 # almost 4 hours + timeout-minutes: 240 # 4 hours runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner={1}/disk=large/tag={2}-fuzz', github.run_id, inputs.runner, inputs.fuzz_name || inputs.fuzz_target) diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 6f51e873c61..41f96cd6853 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -16,6 +16,7 @@ jobs: spelling: name: Spell Check with Typos runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout Actions Repository uses: actions/checkout@v6 diff --git a/.github/workflows/wasm-fuzz.yml b/.github/workflows/wasm-fuzz.yml index 4ff51ee78f3..9b6e205a2cc 100644 --- a/.github/workflows/wasm-fuzz.yml +++ b/.github/workflows/wasm-fuzz.yml @@ -23,7 +23,7 @@ jobs: wasm-fuzz: name: "Build & Fuzz WASM" runs-on: ubuntu-latest - timeout-minutes: 270 + timeout-minutes: 240 steps: - uses: actions/checkout@v6 @@ -63,7 +63,7 @@ jobs: # Capture exit code - wasmfuzz exits with error on crash set +e wasmfuzz fuzz \ - --timeout=4h \ + --timeout=3h30m \ --cores 2 \ --dir corpus-wasm/ \ target/wasm32-wasip1/release/array_ops_wasm.wasm diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 6fd0a72ea83..f0d455f9a90 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -22,6 +22,7 @@ jobs: changes: name: Detect Changes runs-on: ubuntu-latest + timeout-minutes: 10 permissions: pull-requests: read outputs: From 82c938bb0bbfed7266c500ab2eef5d9e5f7f137a Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 16 Apr 2026 12:12:27 +0100 Subject: [PATCH 060/250] Correct rust version channel (#7463) I had noticed when setting up machines for benchmarking that 1.90 isn't actually a valid rust version and changing it to 1.90.0 was required to make it work --------- Signed-off-by: Robert Kruszewski --- .github/workflows/web.yml | 4 +++- Cargo.toml | 2 +- rust-toolchain.toml | 2 +- vortex-web/crate/Cargo.toml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index f0d455f9a90..a7b8008a9cf 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -58,7 +58,9 @@ jobs: cache: "npm" cache-dependency-path: vortex-web/package-lock.json - run: npm ci - - run: npm run wasm + - env: + RUSTFLAGS: --cfg getrandom_backend\"unsupported" + run: npm run wasm - run: npm run format:check - run: npm run lint - run: npm run typecheck diff --git a/Cargo.toml b/Cargo.toml index f4cfcecbf38..fc77bcbb9ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ keywords = ["vortex"] license = "Apache-2.0" readme = "README.md" repository = "https://github.com/spiraldb/vortex" -rust-version = "1.90" +rust-version = "1.90.0" version = "0.1.0" [workspace.dependencies] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c8eba8175ae..e2ce7163390 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.90" +channel = "1.90.0" components = ["rust-src", "rustfmt", "clippy", "rust-analyzer"] profile = "minimal" \ No newline at end of file diff --git a/vortex-web/crate/Cargo.toml b/vortex-web/crate/Cargo.toml index 25b16583380..434932c1d25 100644 --- a/vortex-web/crate/Cargo.toml +++ b/vortex-web/crate/Cargo.toml @@ -2,7 +2,7 @@ name = "vortex-web-wasm" version = "0.1.0" edition = "2024" -rust-version = "1.90" +rust-version = "1.90.0" license = "Apache-2.0" description = "WASM bindings for the Vortex web explorer" publish = false From 216499fcc1484c73b4a80c3d11c1caed51906e8a Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 16 Apr 2026 12:26:36 +0100 Subject: [PATCH 061/250] fix: wipe duckdb archive as part of `cargo clean` (#7465) The `OUT_DIR` gets wiped as part of `cargo clean`. Signed-off-by: Alexander Droste --- vortex-duckdb/build.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs index e23edf5ad72..340651027a1 100644 --- a/vortex-duckdb/build.rs +++ b/vortex-duckdb/build.rs @@ -392,8 +392,8 @@ fn main() { let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let duckdb_dir = crate_dir.join("duckdb"); - let target_dir = crate_dir.parent().unwrap().join("target"); - let library_dir = target_dir.join(format!("duckdb-lib-{version}")); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let library_dir = out_dir.join(format!("duckdb-lib-{version}")); let library_dir_str = library_dir.display(); println!("cargo:rustc-link-search=native={library_dir_str}"); @@ -411,11 +411,10 @@ fn main() { // println!("cargo:rustc-link-arg=-Wl,-rpath,{duckdb_lib}"); // } // - // Alternatively, set LD_LIBRARY_PATH (Linux) or DYLD_LIBRARY_PATH (macOS) at runtime: - // LD_LIBRARY_PATH=/path/to/target/duckdb-lib-vX.Y.Z cargo run --bin ... + // Alternatively, set LD_LIBRARY_PATH (Linux) or DYLD_LIBRARY_PATH (macOS) at runtime. println!("cargo:lib_dir={library_dir_str}"); - let source_dir = target_dir.join(format!("duckdb-source-{version}")); + let source_dir = out_dir.join(format!("duckdb-source-{version}")); let source_archive_url = match &version { DuckDBVersion::Release(v) => format!("{DUCKDB_SOURCE_RELEASE_URL}/v{v}.zip"), DuckDBVersion::Commit(c) => format!("{DUCKDB_SOURCE_COMMIT_URL}/{c}.zip"), From 5e5475aba8b0adab75b8be15ce87a7d8e2d29c9e Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 16 Apr 2026 12:31:25 +0100 Subject: [PATCH 062/250] Unify benchmark file download logic (#7462) Stop repeating same download logic all over codebase There's still slight variation of this logic in statpopgen vcf download Signed-off-by: Robert Kruszewski --- vortex-bench/src/clickbench/benchmark.rs | 3 +- vortex-bench/src/clickbench/data.rs | 103 ++---------------- vortex-bench/src/datasets/data_downloads.rs | 109 ++++++++++++++++---- vortex-bench/src/utils/file.rs | 10 +- vortex-bench/src/vector_dataset/download.rs | 87 +--------------- 5 files changed, 104 insertions(+), 208 deletions(-) diff --git a/vortex-bench/src/clickbench/benchmark.rs b/vortex-bench/src/clickbench/benchmark.rs index 5e14cbcf40e..2e197bef8e1 100644 --- a/vortex-bench/src/clickbench/benchmark.rs +++ b/vortex-bench/src/clickbench/benchmark.rs @@ -6,7 +6,6 @@ use std::fs; use std::path::Path; use anyhow::Result; -use reqwest::Client; use url::Url; use vortex::error::VortexExpect; @@ -89,7 +88,7 @@ impl Benchmark for ClickBenchBenchmark { } let basepath = clickbench_flavor(self.flavor).to_data_path(); - self.flavor.download(Client::default(), basepath).await?; + self.flavor.download(basepath).await?; Ok(()) } diff --git a/vortex-bench/src/clickbench/data.rs b/vortex-bench/src/clickbench/data.rs index c0ac621a6b4..af11e468d45 100644 --- a/vortex-bench/src/clickbench/data.rs +++ b/vortex-bench/src/clickbench/data.rs @@ -6,31 +6,22 @@ use std::fmt::Display; use std::path::Path; use std::str::FromStr; use std::sync::LazyLock; -use std::time::Duration; use arrow_schema::DataType; use arrow_schema::Field; use arrow_schema::Schema; use arrow_schema::TimeUnit; -use bytes::Bytes; use clap::ValueEnum; -use futures::StreamExt; -use indicatif::ProgressBar; -use indicatif::ProgressStyle; -use reqwest::IntoUrl; use serde::Deserialize; use serde::Serialize; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use tokio::task::JoinSet; use tracing::info; -use tracing::warn; use vortex::error::VortexExpect; use crate::Format; // Re-export for use by clickbench_benchmark pub use crate::conversions::convert_parquet_directory_to_vortex; -use crate::idempotent_async; +use crate::datasets::data_downloads::download_data; pub static HITS_SCHEMA: LazyLock = LazyLock::new(|| { use DataType::*; @@ -190,22 +181,14 @@ impl Display for Flavor { impl Flavor { // TODO(joe): move these elsewhere. - pub async fn download( - &self, - client: reqwest::Client, - basepath: impl AsRef, - ) -> anyhow::Result<()> { + pub async fn download(&self, basepath: impl AsRef) -> anyhow::Result<()> { let basepath = basepath.as_ref(); match self { Flavor::Single => { let output_path = basepath.join(Format::Parquet.name()).join("hits.parquet"); - idempotent_async(output_path.as_path(), |output_path| async move { - info!("Downloading single clickbench file"); - let url = "https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_single/hits.parquet"; - download_large_file(&client, url, &output_path).await?; - anyhow::Ok(()) - }) - .await?; + info!("Downloading single clickbench file"); + let url = "https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_single/hits.parquet"; + download_data(output_path, url).await?; } Flavor::Partitioned => { // The clickbench-provided file is missing some higher-level type info, so we reprocess it @@ -213,17 +196,10 @@ impl Flavor { let mut tasks = (0_u32..100).map(|idx| { let output_path = basepath.join(Format::Parquet.name()).join(format!("hits_{idx}.parquet")); - let client = client.clone(); - idempotent_async(output_path, move|output_path| async move { - info!("Downloading file {idx}"); - let url = format!("https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_{idx}.parquet"); - let body = retry_get(&client, url).await?; - let mut file = File::create(output_path).await?; - file.write_all(&body).await?; - - anyhow::Ok(()) - }) + info!("Downloading file {idx}"); + let url = format!("https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_{idx}.parquet"); + download_data(output_path, url) }).collect::>(); while let Some(task) = tasks.join_next().await { @@ -234,66 +210,3 @@ impl Flavor { Ok(()) } } - -/// Downloads a large file with streaming and progress indication. -async fn download_large_file( - client: &reqwest::Client, - url: &str, - output_path: &Path, -) -> anyhow::Result<()> { - let response = client.get(url).send().await?.error_for_status()?; - - let total_size = response.content_length().unwrap_or(0); - - let progress = ProgressBar::new(total_size); - progress.set_style( - ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} ({bytes_per_sec})", - ) - .expect("valid template"), - ); - - let mut file = File::create(output_path).await?; - let mut stream = response.bytes_stream(); - - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - file.write_all(&chunk).await?; - progress.inc(chunk.len() as u64); - } - - progress.finish(); - Ok(()) -} - -async fn retry_get(client: &reqwest::Client, url: impl IntoUrl) -> anyhow::Result { - let url = url.as_str(); - let make_req = || async { client.get(url).send().await }; - - let mut body = None; - - for attempt in 0..3 { - match make_req().await.and_then(|r| r.error_for_status()) { - Ok(r) => match r.bytes().await { - Ok(b) => { - body = Some(b); - break; - } - Err(e) => { - warn!("Request errored with {e}, retying for the {attempt} time"); - } - }, - Err(e) => { - warn!("Request errored with {e}, retying for the {attempt} time"); - } - } - - // Very basic backoff mechanism - tokio::time::sleep(Duration::from_secs(attempt + 1)).await; - } - - match body { - Some(v) => Ok(v), - None => anyhow::bail!("Exahusted retry attempts for {url}"), - } -} diff --git a/vortex-bench/src/datasets/data_downloads.rs b/vortex-bench/src/datasets/data_downloads.rs index e846edbeaec..a603436b458 100644 --- a/vortex-bench/src/datasets/data_downloads.rs +++ b/vortex-bench/src/datasets/data_downloads.rs @@ -8,48 +8,117 @@ use std::path::PathBuf; use std::time::Duration; use anyhow::Context; +use anyhow::Error; use anyhow::Result; use bzip2::read::BzDecoder; +use futures::StreamExt; +use indicatif::ProgressBar; +use indicatif::ProgressStyle; +use parking_lot::RwLock; use reqwest::Client; +use reqwest::Response; use tokio::fs::File as TokioFile; use tokio::io::AsyncWriteExt; use tracing::info; +use tracing::warn; use crate::utils::file::idempotent; use crate::utils::file::idempotent_async; -pub async fn download_data(fname: PathBuf, data_url: &str) -> Result { - idempotent_async(&fname, async |path| { - info!( - "Downloading {} from {}", - fname.to_str().context("Failed to convert path to string")?, - data_url - ); - let mut file = TokioFile::create(path) +async fn retry_get>, R: Fn() -> F>( + make_req: R, + tmp_path: PathBuf, +) -> Result<()> { + const MAX_ATTEMPTS: u32 = 3; + let mut last_err: Option = None; + let progress = RwLock::new(None::); + + let retry = async || -> Result<()> { + let mut file = TokioFile::create(tmp_path) .await .context("Failed to create file")?; - let mut response = Client::builder() - .read_timeout(Duration::from_secs(60)) - .timeout(Duration::from_secs(60 * 15)) - .build() - .context("Failed to build HTTP client")? - .get(data_url) - .send() + let response = make_req() .await .context("Failed to send HTTP request")? .error_for_status() .context("HTTP request returned error status")?; - while let Some(chunk) = response - .chunk() - .await - .context("Failed to read response chunk")? - { + *progress.write() = response.content_length().map(|total| { + let progress = ProgressBar::new(total); + progress.set_style( + ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} ({bytes_per_sec})", + ) + .expect("valid template"), + ); + progress + }); + + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; AsyncWriteExt::write_all(&mut file, &chunk) .await .context("Failed to write to file")?; + if let Some(p) = progress.write().as_mut() { + p.inc(chunk.len() as u64) + } } + + AsyncWriteExt::flush(&mut file).await?; Ok(()) + }; + + for attempt in 0..MAX_ATTEMPTS { + let outcome = retry.clone()().await; + + match outcome { + Ok(_) => { + if let Some(p) = progress.write().take() { + p.finish_and_clear() + } + return Ok(()); + } + Err(e) => { + if let Some(p) = progress.write().take() { + p.abandon() + } + last_err = Some(e) + } + } + let backoff = Duration::from_secs(1u64 << attempt); + warn!( + "download attempt {} failed; retrying in {:?}", + attempt + 1, + backoff + ); + tokio::time::sleep(backoff).await; + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("retry_get exhausted with no recorded error"))) +} + +pub async fn download_data(fname: PathBuf, data_url: impl AsRef) -> Result { + let client = Client::builder() + .read_timeout(Duration::from_secs(60)) + .timeout(Duration::from_secs(60 * 15)) + .build() + .context("Failed to build HTTP client")?; + + idempotent_async(&fname, async |path| { + let url = data_url.as_ref(); + info!( + "Downloading {} from {}", + fname.to_str().context("Failed to convert path to string")?, + url + ); + retry_get( + async || { + let res = client.get(url).send().await?; + Ok(res) + }, + path, + ) + .await }) .await } diff --git a/vortex-bench/src/utils/file.rs b/vortex-bench/src/utils/file.rs index f13dbdf93c8..b6d202bb29a 100644 --- a/vortex-bench/src/utils/file.rs +++ b/vortex-bench/src/utils/file.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fs::create_dir_all; +use std::fs; use std::future::Future; use std::path::Path; use std::path::PathBuf; @@ -25,10 +25,10 @@ pub fn idempotent( if !data_path.exists() { // Ensure parent directory exists if let Some(parent) = data_path.parent() { - create_dir_all(parent).context("Failed to create parent directories")?; + fs::create_dir_all(parent).context("Failed to create parent directories")?; } f(temp_path.as_path())?; - std::fs::rename(temp_path, &data_path).context("Failed to rename temp file")?; + fs::rename(temp_path, &data_path).context("Failed to rename temp file")?; } Ok(data_path) } @@ -44,10 +44,10 @@ where if !data_path.exists() { // Ensure parent directory exists if let Some(parent) = data_path.parent() { - create_dir_all(parent).context("Failed to create parent directories")?; + fs::create_dir_all(parent).context("Failed to create parent directories")?; } f(temp_path.clone()).await?; - std::fs::rename(temp_path, &data_path).context("Failed to rename temp file")?; + fs::rename(temp_path, &data_path).context("Failed to rename temp file")?; } Ok(data_path) } diff --git a/vortex-bench/src/vector_dataset/download.rs b/vortex-bench/src/vector_dataset/download.rs index da4b43262b6..228c51db771 100644 --- a/vortex-bench/src/vector_dataset/download.rs +++ b/vortex-bench/src/vector_dataset/download.rs @@ -14,21 +14,11 @@ //! `test.parquet` and (when present) `neighbors.parquet` live alongside the train files. use std::path::PathBuf; -use std::time::Duration; use anyhow::Context; use anyhow::Result; -use bytes::Bytes; -use futures::StreamExt; -use indicatif::ProgressBar; -use indicatif::ProgressStyle; -use reqwest::Client; -use reqwest::IntoUrl; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use tokio::task::JoinSet; use tracing::info; -use tracing::warn; use crate::datasets::data_downloads::download_data; use crate::utils::file::idempotent_async; @@ -113,23 +103,12 @@ pub async fn download(ds: VectorDataset, layout: TrainLayout) -> Result> = JoinSet::new(); for (url, target) in urls.into_iter().zip(train_targets.iter().cloned()) { - let client = client.clone(); tasks.spawn(async move { idempotent_async(target, |tmp| async move { info!("downloading {}", url); - if spec.layout().is_partitioned() { - download_with_retry(&client, &url, &tmp).await?; - } else { - download_with_progress(&client, &url, &tmp).await?; - } - Ok(()) + download_data(tmp, &url).await }) .await?; Ok(()) @@ -159,67 +138,3 @@ pub async fn download(ds: VectorDataset, layout: TrainLayout) -> Result Result<()> { - let response = client - .get(url) - .send() - .await - .with_context(|| format!("GET {url}"))? - .error_for_status()?; - let total = response.content_length().unwrap_or(0); - - let progress = ProgressBar::new(total); - progress.set_style( - ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} ({bytes_per_sec})", - ) - .expect("valid template"), - ); - - let mut file = File::create(output).await?; - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - file.write_all(&chunk).await?; - progress.inc(chunk.len() as u64); - } - progress.finish_and_clear(); - file.flush().await?; - Ok(()) -} - -/// Buffer-the-whole-body download with simple exponential backoff. Used for partitioned -/// shards because we already have download concurrency at the shard granularity. -async fn download_with_retry(client: &Client, url: &str, output: &PathBuf) -> Result<()> { - let body = retry_get(client, url).await?; - let mut file = File::create(output).await?; - file.write_all(&body).await?; - file.flush().await?; - Ok(()) -} - -async fn retry_get(client: &Client, url: impl IntoUrl + Clone) -> Result { - const MAX_ATTEMPTS: u32 = 4; - let mut last_err: Option = None; - for attempt in 0..MAX_ATTEMPTS { - let outcome: Result = async { - let resp = client.get(url.clone()).send().await?.error_for_status()?; - Ok(resp.bytes().await?) - } - .await; - match outcome { - Ok(b) => return Ok(b), - Err(e) => last_err = Some(e), - } - let backoff = Duration::from_secs(1u64 << attempt); - warn!( - "download attempt {} failed; retrying in {:?}", - attempt + 1, - backoff - ); - tokio::time::sleep(backoff).await; - } - Err(last_err.unwrap_or_else(|| anyhow::anyhow!("retry_get exhausted with no recorded error"))) -} From 9f9f8f1271a13509274fa054ab5f3e55692c5cf6 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 16 Apr 2026 12:42:38 +0100 Subject: [PATCH 063/250] Fix web deployment github action (#7467) I accidentally broke it in #7463 --------- Signed-off-by: Robert Kruszewski --- .github/workflows/web.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index a7b8008a9cf..534e2e3a100 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -59,8 +59,8 @@ jobs: cache-dependency-path: vortex-web/package-lock.json - run: npm ci - env: - RUSTFLAGS: --cfg getrandom_backend\"unsupported" - run: npm run wasm + RUSTFLAGS: --cfg getrandom_backend="unsupported" + run: npm run wasm - run: npm run format:check - run: npm run lint - run: npm run typecheck @@ -87,7 +87,9 @@ jobs: cache: "npm" cache-dependency-path: vortex-web/package-lock.json - run: npm ci - - run: npm run build + - env: + RUSTFLAGS: --cfg getrandom_backend="unsupported" + run: npm run build - name: Upload build artifact uses: actions/upload-artifact@v4 with: From f924a343ebb71f983882007e301263d798db97ca Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Apr 2026 14:12:24 +0100 Subject: [PATCH 064/250] Unify all runtime env var access to functions wrapping LazyLock (#7468) ## Summary Env variable access isn't free, this is a fairly common pattern we already used in a couple of places, this PR just makes it consistent across the codebase. ## API Changes Replaced the experimental patches from a public `LazyLock` to a function. I think this should've been a feature an not a runtime thing, but that's a different change. I've also removed the `EnvVarGuard` from `vortex-utils`, it was only used in one place and I've just replaced it with `temp-env` that we already use elsewhere. ## Testing For the few tests that toggled env variables, I've changed them to only run on nextest, relying on it process isolation. --------- Signed-off-by: Adam Gutglick --- Cargo.lock | 1 + Cargo.toml | 1 + encodings/alp/src/lib.rs | 4 +- encodings/fastlanes/src/lib.rs | 4 +- vortex-array/public-api.lock | 4 +- vortex-array/src/aggregate_fn/accumulator.rs | 4 +- .../src/aggregate_fn/accumulator_grouped.rs | 4 +- vortex-array/src/arrays/patched/mod.rs | 7 +- vortex-array/src/executor.rs | 42 +- vortex-btrblocks/src/schemes/float.rs | 4 +- vortex-btrblocks/src/schemes/integer.rs | 4 +- vortex-error/Cargo.toml | 2 +- vortex-error/src/lib.rs | 9 +- vortex-file/src/strategy.rs | 4 +- vortex-layout/Cargo.toml | 2 +- vortex-layout/src/display.rs | 400 +++++++++--------- vortex-layout/src/layouts/flat/mod.rs | 6 +- vortex-utils/Cargo.toml | 1 - vortex-utils/src/env.rs | 197 --------- vortex-utils/src/lib.rs | 2 - 20 files changed, 260 insertions(+), 442 deletions(-) delete mode 100644 vortex-utils/src/env.rs diff --git a/Cargo.lock b/Cargo.lock index 9a31e702e77..adcab4a2864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10741,6 +10741,7 @@ dependencies = [ "rstest", "rustc-hash", "sketches-ddsketch 0.4.0", + "temp-env", "termtree", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index fc77bcbb9ce..a6429bf04e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -236,6 +236,7 @@ taffy = "0.9.0" take_mut = "0.2.2" tar = "0.4" target-lexicon = "0.13" +temp-env = "0.3" tempfile = "3" termtree = { version = "1.0" } test-with = "0.15" diff --git a/encodings/alp/src/lib.rs b/encodings/alp/src/lib.rs index 05ce75763b2..7da676d6c6c 100644 --- a/encodings/alp/src/lib.rs +++ b/encodings/alp/src/lib.rs @@ -22,7 +22,7 @@ use vortex_array::ArrayVTable; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::nan_count::NanCount; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; -use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; @@ -33,7 +33,7 @@ mod alp_rd; pub fn initialize(session: &VortexSession) { // If we're using the experimental Patched encoding, register a shim // for ALP with interior patches to decode as Patched array. - if *USE_EXPERIMENTAL_PATCHES { + if use_experimental_patches() { session.arrays().register(ALPPatchedPlugin); } else { session.arrays().register(ALP); diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 217d05d73b8..bed17366676 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -29,7 +29,7 @@ use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::session::AggregateFnSessionExt; -use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; @@ -37,7 +37,7 @@ use vortex_session::VortexSession; pub fn initialize(session: &VortexSession) { // If we're using the experimental Patched encoding, register a shim // for BitPacked with interior patches decode as Patched array. - if *USE_EXPERIMENTAL_PATCHES { + if use_experimental_patches() { session.arrays().register(BitPackedPatchedPlugin); } else { session.arrays().register(BitPacked); diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index a849acdb20e..243c59cafa0 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3466,8 +3466,6 @@ pub fn vortex_array::arrays::patched::PatchedSlotsView<'a>::fmt(&self, f: &mut c impl<'a> core::marker::Copy for vortex_array::arrays::patched::PatchedSlotsView<'a> -pub static vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES: std::sync::lazy_lock::LazyLock - pub trait vortex_array::arrays::patched::PatchedArrayExt: vortex_array::arrays::patched::PatchedArraySlotsExt pub fn vortex_array::arrays::patched::PatchedArrayExt::lane_range(&self, chunk: usize, lane: usize) -> vortex_error::VortexResult> @@ -3512,6 +3510,8 @@ pub fn T::patch_values(&self) -> &vortex_array::ArrayRef pub fn T::slots_view(&self) -> vortex_array::arrays::patched::PatchedSlotsView<'_> +pub fn vortex_array::arrays::patched::use_experimental_patches() -> bool + pub type vortex_array::arrays::patched::PatchedArray = vortex_array::Array pub mod vortex_array::arrays::primitive diff --git a/vortex-array/src/aggregate_fn/accumulator.rs b/vortex-array/src/aggregate_fn/accumulator.rs index 71ed56b3110..9c864eea6ca 100644 --- a/vortex-array/src/aggregate_fn/accumulator.rs +++ b/vortex-array/src/aggregate_fn/accumulator.rs @@ -14,7 +14,7 @@ use crate::aggregate_fn::AggregateFnRef; use crate::aggregate_fn::AggregateFnVTable; use crate::aggregate_fn::session::AggregateFnSessionExt; use crate::dtype::DType; -use crate::executor::MAX_ITERATIONS; +use crate::executor::max_iterations; use crate::scalar::Scalar; /// Reference-counted type-erased accumulator. @@ -108,7 +108,7 @@ impl DynAccumulator for Accumulator { let kernels = &session.aggregate_fns().kernels; let mut batch = batch.clone(); - for _ in 0..*MAX_ITERATIONS { + for _ in 0..max_iterations() { if batch.is::() { break; } diff --git a/vortex-array/src/aggregate_fn/accumulator_grouped.rs b/vortex-array/src/aggregate_fn/accumulator_grouped.rs index 24f8f0157ec..665dcec0747 100644 --- a/vortex-array/src/aggregate_fn/accumulator_grouped.rs +++ b/vortex-array/src/aggregate_fn/accumulator_grouped.rs @@ -32,7 +32,7 @@ use crate::builders::builder_with_capacity; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::IntegerPType; -use crate::executor::MAX_ITERATIONS; +use crate::executor::max_iterations; use crate::match_each_integer_ptype; /// Reference-counted type-erased grouped accumulator. @@ -165,7 +165,7 @@ impl GroupedAccumulator { let session = ctx.session().clone(); let kernels = &session.aggregate_fns().grouped_kernels; - for _ in 0..*MAX_ITERATIONS { + for _ in 0..max_iterations() { if elements.is::() { break; } diff --git a/vortex-array/src/arrays/patched/mod.rs b/vortex-array/src/arrays/patched/mod.rs index b0453c68677..0ba25ab7929 100644 --- a/vortex-array/src/arrays/patched/mod.rs +++ b/vortex-array/src/arrays/patched/mod.rs @@ -108,5 +108,8 @@ const fn patch_lanes() -> usize { /// array, and eliminate the interior patches. /// /// The builtin compressor will also generate Patched arrays. -pub static USE_EXPERIMENTAL_PATCHES: LazyLock = - LazyLock::new(|| env::var("VORTEX_EXPERIMENTAL_PATCHED_ARRAY").is_ok()); +pub fn use_experimental_patches() -> bool { + static USE_EXPERIMENTAL_PATCHES: LazyLock = + LazyLock::new(|| env::var("VORTEX_EXPERIMENTAL_PATCHED_ARRAY").is_ok_and(|v| v == "1")); + *USE_EXPERIMENTAL_PATCHES +} diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 6233295958b..74a013bf5bb 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -38,18 +38,21 @@ use crate::memory::HostAllocatorRef; use crate::memory::MemorySessionExt; use crate::optimizer::ArrayOptimizer; -/// Maximum number of iterations to attempt when executing an array before giving up and returning -/// an error. -pub(crate) static MAX_ITERATIONS: LazyLock = - LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { - Ok(val) => val - .parse::() - .unwrap_or_else(|e| vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}")), - Err(VarError::NotPresent) => 128, - Err(VarError::NotUnicode(_)) => { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") - } - }); +/// Returns the maximum number of iterations to attempt when executing an array before giving up and returning +/// an error, can be by the `VORTEX_MAX_ITERATIONS` env variables, otherwise defaults to 128. +pub(crate) fn max_iterations() -> usize { + static MAX_ITERATIONS: LazyLock = + LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { + Ok(val) => val.parse::().unwrap_or_else(|e| { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") + }), + Err(VarError::NotPresent) => 128, + Err(VarError::NotUnicode(_)) => { + vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") + } + }); + *MAX_ITERATIONS +} /// Marker trait for types that an [`ArrayRef`] can be executed into. /// @@ -95,22 +98,11 @@ impl ArrayRef { /// For safety, we will error when the number of execution iterations reaches a configurable /// maximum (default 128, override with `VORTEX_MAX_ITERATIONS`). pub fn execute_until(self, ctx: &mut ExecutionCtx) -> VortexResult { - static MAX_ITERATIONS: LazyLock = - LazyLock::new(|| match std::env::var("VORTEX_MAX_ITERATIONS") { - Ok(val) => val.parse::().unwrap_or_else(|e| { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid usize: {e}") - }), - Err(VarError::NotPresent) => 128, - Err(VarError::NotUnicode(_)) => { - vortex_panic!("VORTEX_MAX_ITERATIONS is not a valid unicode string") - } - }); - let mut current = self.optimize()?; // Stack frames: (parent, slot_idx, done_predicate_for_slot) let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); - for _ in 0..*MAX_ITERATIONS { + for _ in 0..max_iterations() { // Check for termination: use the stack frame's done predicate, or the root matcher. let is_done = stack .last() @@ -179,7 +171,7 @@ impl ArrayRef { vortex_bail!( "Exceeded maximum execution iterations ({}) while executing array", - *MAX_ITERATIONS, + max_iterations(), ) } } diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 6c204c7263a..f6d2389f08b 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -17,7 +17,7 @@ use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; -use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; use vortex_compressor::estimate::CompressionEstimate; @@ -120,7 +120,7 @@ impl Scheme for ALPScheme { let alp_stats = alp_encoded.as_array().statistics().to_owned(); let exponents = alp_encoded.exponents(); - if *USE_EXPERIMENTAL_PATCHES { + if use_experimental_patches() { let patches = alp_encoded.patches(); // Create ALP array without interior patches. diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 47fc52225aa..cb765fbec5a 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -11,7 +11,7 @@ use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Patched; -use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::scalar::Scalar; use vortex_compressor::builtins::FloatDictScheme; @@ -348,7 +348,7 @@ impl Scheme for BitPackingScheme { let ptype = packed.dtype().as_ptype(); let mut parts = BitPacked::into_parts(packed); - let array = if *USE_EXPERIMENTAL_PATCHES { + let array = if use_experimental_patches() { let patches = parts.patches.take(); // Transpose patches into G-ALP style PatchedArray, wrapping an inner BitPackedArray. let array = BitPacked::try_new( diff --git a/vortex-error/Cargo.toml b/vortex-error/Cargo.toml index 2856a2724ae..fd85dd770dd 100644 --- a/vortex-error/Cargo.toml +++ b/vortex-error/Cargo.toml @@ -29,7 +29,7 @@ tokio = { workspace = true, features = ["rt"], optional = true } [dev-dependencies] serial_test = { version = "3.3.1" } -temp-env = "0.3" +temp-env = { workspace = true } [lints] workspace = true diff --git a/vortex-error/src/lib.rs b/vortex-error/src/lib.rs index 156ab574706..e48dafcf6f9 100644 --- a/vortex-error/src/lib.rs +++ b/vortex-error/src/lib.rs @@ -20,6 +20,7 @@ use std::io; use std::num::TryFromIntError; use std::ops::Deref; use std::sync::Arc; +use std::sync::LazyLock; /// A string that can be used as an error message. #[derive(Debug)] @@ -38,7 +39,7 @@ where reason = "intentionally panic in debug mode when VORTEX_PANIC_ON_ERR is set" )] fn from(msg: T) -> Self { - if env::var("VORTEX_PANIC_ON_ERR").as_deref().unwrap_or("") == "1" { + if panic_on_err() { panic!("{}\nBacktrace:\n{}", msg.into(), Backtrace::capture()); } else { Self(msg.into()) @@ -46,6 +47,12 @@ where } } +fn panic_on_err() -> bool { + static PANIC_ON_ERR: LazyLock = + LazyLock::new(|| env::var("VORTEX_PANIC_ON_ERR").is_ok_and(|v| v == "1")); + *PANIC_ON_ERR +} + impl AsRef for ErrString { fn as_ref(&self) -> &str { &self.0 diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 0a3e31f32e3..41d9e3c2ad2 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -26,7 +26,7 @@ use vortex_array::arrays::Primitive; use vortex_array::arrays::Struct; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinView; -use vortex_array::arrays::patched::USE_EXPERIMENTAL_PATCHES; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::dtype::FieldPath; use vortex_btrblocks::BtrBlocksCompressorBuilder; use vortex_btrblocks::SchemeExt; @@ -90,7 +90,7 @@ pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { allowed.insert(Masked.id()); allowed.insert(Dict.id()); - if *USE_EXPERIMENTAL_PATCHES { + if use_experimental_patches() { allowed.insert(Patched.id()); } diff --git a/vortex-layout/Cargo.toml b/vortex-layout/Cargo.toml index 8bbccf09b91..30a0953444a 100644 --- a/vortex-layout/Cargo.toml +++ b/vortex-layout/Cargo.toml @@ -55,10 +55,10 @@ vortex-utils = { workspace = true, features = ["dashmap"] } [dev-dependencies] futures = { workspace = true, features = ["executor"] } rstest = { workspace = true } +temp-env = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } vortex-array = { path = "../vortex-array", features = ["_test-harness"] } vortex-io = { path = "../vortex-io", features = ["tokio"] } -vortex-utils = { workspace = true, features = ["_test-harness"] } [features] _test-harness = [] diff --git a/vortex-layout/src/display.rs b/vortex-layout/src/display.rs index 39605dfb5d1..5acf685e946 100644 --- a/vortex-layout/src/display.rs +++ b/vortex-layout/src/display.rs @@ -231,7 +231,6 @@ mod tests { use vortex_buffer::buffer; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSessionExt; - use vortex_utils::env::EnvVarGuard; use crate::IntoLayout; use crate::OwnedLayoutChildren; @@ -248,231 +247,244 @@ mod tests { /// Test display_tree with inline array_tree metadata (no segment source needed). #[test] fn test_display_tree_inline_array_tree() { - let _guard = EnvVarGuard::set("FLAT_LAYOUT_INLINE_ARRAY_NODE", "1"); - block_on(|handle| async move { - let session = SESSION.clone().with_handle(handle); - let ctx = ArrayContext::empty(); - let segments = Arc::new(TestSegments::default()); - - // Create nullable i64 array (2 buffers: data + validity) - let (ptr1, eof1) = SequenceId::root().split(); - let mut validity_builder = BitBufferMut::with_capacity(5); - for b in [true, false, true, true, false] { - validity_builder.append(b); - } - let validity = Validity::Array( - BoolArray::new(validity_builder.freeze(), Validity::NonNullable).into_array(), - ); - let array1 = PrimitiveArray::new(buffer![1i64, 2, 3, 4, 5], validity); - let layout1 = FlatLayoutStrategy::default() - .write_stream( - ctx.clone(), - Arc::::clone(&segments), - array1.into_array().to_array_stream().sequenced(ptr1), - eof1, - &session, - ) - .await - .unwrap(); - - // Create utf8 array (2 buffers: views + data) - let (ptr2, eof2) = SequenceId::root().split(); - let mut builder = VarBinViewBuilder::with_capacity(DType::Utf8(NonNullable), 5); - for s in [ - "hello world this is long", - "another long string", - "short", - "medium str", - "x", - ] { - builder.append_value(s); - } - let layout2 = FlatLayoutStrategy::default() - .write_stream( - ctx.clone(), - Arc::::clone(&segments), - builder - .finish() - .into_array() - .to_array_stream() - .sequenced(ptr2), - eof2, - &session, - ) - .await - .unwrap(); - - // Create struct layout - let struct_layout = StructLayout::new( - 5, - DType::Struct( - StructFields::new( - vec![FieldName::from("numbers"), FieldName::from("strings")].into(), + // LazyLock caches the env var on first read, so only nextest (separate processes) can isolate it. + if std::env::var("NEXTEST_RUN_ID").is_ok() { + temp_env::with_var("FLAT_LAYOUT_INLINE_ARRAY_NODE", Some("1"), || { + block_on(|handle| async move { + let session = SESSION.clone().with_handle(handle); + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + + // Create nullable i64 array (2 buffers: data + validity) + let (ptr1, eof1) = SequenceId::root().split(); + let mut validity_builder = BitBufferMut::with_capacity(5); + for b in [true, false, true, true, false] { + validity_builder.append(b); + } + let validity = Validity::Array( + BoolArray::new(validity_builder.freeze(), Validity::NonNullable) + .into_array(), + ); + let array1 = PrimitiveArray::new(buffer![1i64, 2, 3, 4, 5], validity); + let layout1 = FlatLayoutStrategy::default() + .write_stream( + ctx.clone(), + Arc::::clone(&segments), + array1.into_array().to_array_stream().sequenced(ptr1), + eof1, + &session, + ) + .await + .unwrap(); + + // Create utf8 array (2 buffers: views + data) + let (ptr2, eof2) = SequenceId::root().split(); + let mut builder = VarBinViewBuilder::with_capacity(DType::Utf8(NonNullable), 5); + for s in [ + "hello world this is long", + "another long string", + "short", + "medium str", + "x", + ] { + builder.append_value(s); + } + let layout2 = FlatLayoutStrategy::default() + .write_stream( + ctx.clone(), + Arc::::clone(&segments), + builder + .finish() + .into_array() + .to_array_stream() + .sequenced(ptr2), + eof2, + &session, + ) + .await + .unwrap(); + + // Create struct layout + let struct_layout = StructLayout::new( + 5, + DType::Struct( + StructFields::new( + vec![FieldName::from("numbers"), FieldName::from("strings")].into(), + vec![ + DType::Primitive(PType::I64, Nullability::Nullable), + DType::Utf8(NonNullable), + ], + ), + NonNullable, + ), vec![ - DType::Primitive(PType::I64, Nullability::Nullable), - DType::Utf8(NonNullable), + ChunkedLayout::new( + 5, + DType::Primitive(PType::I64, Nullability::Nullable), + OwnedLayoutChildren::layout_children(vec![layout1]), + ) + .into_layout(), + layout2, ], - ), - NonNullable, - ), - vec![ - ChunkedLayout::new( - 5, - DType::Primitive(PType::I64, Nullability::Nullable), - OwnedLayoutChildren::layout_children(vec![layout1]), ) - .into_layout(), - layout2, - ], - ) - .into_layout(); + .into_layout(); - let output = format!("{}", struct_layout.display_tree_verbose(true)); + let output = format!("{}", struct_layout.display_tree_verbose(true)); - let expected = "\ + let expected = "\ vortex.struct, dtype: {numbers=i64?, strings=utf8}, children: 2, rows: 5 ├── numbers: vortex.chunked, dtype: i64?, children: 1, rows: 5 │ └── [0]: vortex.flat, dtype: i64?, metadata: 171 bytes, rows: 5, segment 0, buffers=[40B, 1B], total=41B └── strings: vortex.flat, dtype: utf8, metadata: 110 bytes, rows: 5, segment 1, buffers=[43B, 80B], total=123B "; - assert_eq!(output, expected); - }) + assert_eq!(output, expected); + }) + }) + } } /// Test display_tree_with_segments using async segment source to fetch buffer sizes. #[test] fn test_display_tree_with_segment_source() { - // Ensure inline array node is disabled for this test - let _guard = EnvVarGuard::remove("FLAT_LAYOUT_INLINE_ARRAY_NODE"); - block_on(|handle| async move { - let session = SESSION.clone().with_handle(handle); - let ctx = ArrayContext::empty(); - let segments = Arc::new(TestSegments::default()); - - // Create simple i32 array - let (ptr1, eof1) = SequenceId::root().split(); - let array1 = PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::NonNullable); - let layout1 = FlatLayoutStrategy::default() - .write_stream( - ctx.clone(), - Arc::::clone(&segments), - array1.into_array().to_array_stream().sequenced(ptr1), - eof1, - &session, - ) - .await - .unwrap(); - - // Create another i32 array - let (ptr2, eof2) = SequenceId::root().split(); - let array2 = PrimitiveArray::new(buffer![6i32, 7, 8, 9, 10], Validity::NonNullable); - let layout2 = FlatLayoutStrategy::default() - .write_stream( - ctx.clone(), - Arc::::clone(&segments), - array2.into_array().to_array_stream().sequenced(ptr2), - eof2, - &session, - ) - .await - .unwrap(); - - // Create chunked layout - let chunked_layout = ChunkedLayout::new( - 10, - DType::Primitive(PType::I32, NonNullable), - OwnedLayoutChildren::layout_children(vec![layout1, layout2]), - ) - .into_layout(); - - let output = chunked_layout - .display_tree_with_segments(segments) - .await - .unwrap(); - - let expected = "\ + if std::env::var("NEXTEST_RUN_ID").is_ok() { + temp_env::with_var("FLAT_LAYOUT_INLINE_ARRAY_NODE", None::<&str>, || { + block_on(|handle| async move { + let session = SESSION.clone().with_handle(handle); + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + + // Create simple i32 array + let (ptr1, eof1) = SequenceId::root().split(); + let array1 = + PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::NonNullable); + let layout1 = FlatLayoutStrategy::default() + .write_stream( + ctx.clone(), + Arc::::clone(&segments), + array1.into_array().to_array_stream().sequenced(ptr1), + eof1, + &session, + ) + .await + .unwrap(); + + // Create another i32 array + let (ptr2, eof2) = SequenceId::root().split(); + let array2 = + PrimitiveArray::new(buffer![6i32, 7, 8, 9, 10], Validity::NonNullable); + let layout2 = FlatLayoutStrategy::default() + .write_stream( + ctx.clone(), + Arc::::clone(&segments), + array2.into_array().to_array_stream().sequenced(ptr2), + eof2, + &session, + ) + .await + .unwrap(); + + // Create chunked layout + let chunked_layout = ChunkedLayout::new( + 10, + DType::Primitive(PType::I32, NonNullable), + OwnedLayoutChildren::layout_children(vec![layout1, layout2]), + ) + .into_layout(); + + let output = chunked_layout + .display_tree_with_segments(segments) + .await + .unwrap(); + + let expected = "\ vortex.chunked, dtype: i32, children: 2, rows: 10 ├── [0]: vortex.flat, dtype: i32, rows: 5, segment 0, buffers=[20B], total=20B └── [1]: vortex.flat, dtype: i32, rows: 5, segment 1, buffers=[20B], total=20B "; - assert_eq!(output.to_string(), expected); - }) + assert_eq!(output.to_string(), expected); + }) + }) + } } /// Test display_array_tree with inline array node metadata. #[test] fn test_display_array_tree_with_inline_node() { - let _guard = EnvVarGuard::set("FLAT_LAYOUT_INLINE_ARRAY_NODE", "1"); - - let ctx = ArrayContext::empty(); - let segments = Arc::new(TestSegments::default()); - let (ptr, eof) = SequenceId::root().split(); - - // Create a simple primitive array - let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::AllValid); - let layout = block_on(|handle| async { - let session = SESSION.clone().with_handle(handle); - FlatLayoutStrategy::default() - .write_stream( - ctx.clone(), - Arc::::clone(&segments), - array.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await - .unwrap() - }); - - let flat_layout = layout.as_::(); - - let array_tree = flat_layout - .array_tree() - .expect("array_tree should be populated when FLAT_LAYOUT_INLINE_ARRAY_NODE is set"); - - let parts = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) - .expect("should parse array_tree"); - assert_eq!(parts.buffer_lengths(), vec![20]); // 5 i32 values = 20 bytes - - assert_eq!( - layout.display_tree().to_string(), - "\ + if std::env::var("NEXTEST_RUN_ID").is_ok() { + temp_env::with_var("FLAT_LAYOUT_INLINE_ARRAY_NODE", Some("1"), || { + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + + // Create a simple primitive array + let array = PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::AllValid); + let layout = block_on(|handle| async { + let session = SESSION.clone().with_handle(handle); + FlatLayoutStrategy::default() + .write_stream( + ctx.clone(), + Arc::::clone(&segments), + array.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await + .unwrap() + }); + + let flat_layout = layout.as_::(); + + let array_tree = flat_layout.array_tree().expect( + "array_tree should be populated when FLAT_LAYOUT_INLINE_ARRAY_NODE is set", + ); + + let parts = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) + .expect("should parse array_tree"); + assert_eq!(parts.buffer_lengths(), vec![20]); // 5 i32 values = 20 bytes + + assert_eq!( + layout.display_tree().to_string(), + "\ vortex.flat, dtype: i32?, segment 0, buffers=[20B], total=20B " - ); + ); + }) + } } /// Test display_tree without inline array node (shows segment ID). #[test] fn test_display_tree_without_inline_node() { - let _guard = EnvVarGuard::set("FLAT_LAYOUT_INLINE_ARRAY_NODE", "1"); - - let ctx = ArrayContext::empty(); - let segments = Arc::new(TestSegments::default()); - let (ptr, eof) = SequenceId::root().split(); - - // Create a simple primitive array - let array = PrimitiveArray::new(buffer![10i64, 20, 30], Validity::NonNullable); - let layout = block_on(|handle| async { - let session = SESSION.clone().with_handle(handle); - FlatLayoutStrategy::default() - .write_stream( - ctx, - Arc::::clone(&segments), - array.into_array().to_array_stream().sequenced(ptr), - eof, - &session, - ) - .await - .unwrap() - }); - - // Test display_tree exact output (with inline array_tree enabled by env var from other test) - assert_eq!( - layout.display_tree().to_string(), - "\ + if std::env::var("NEXTEST_RUN_ID").is_ok() { + temp_env::with_var("FLAT_LAYOUT_INLINE_ARRAY_NODE", Some("1"), || { + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + + // Create a simple primitive array + let array = PrimitiveArray::new(buffer![10i64, 20, 30], Validity::NonNullable); + let layout = block_on(|handle| async { + let session = SESSION.clone().with_handle(handle); + FlatLayoutStrategy::default() + .write_stream( + ctx, + Arc::::clone(&segments), + array.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await + .unwrap() + }); + + // Test display_tree exact output (with inline array_tree enabled by env var from other test) + assert_eq!( + layout.display_tree().to_string(), + "\ vortex.flat, dtype: i64, segment 0, buffers=[24B], total=24B " - ); + ); + }) + } } } diff --git a/vortex-layout/src/layouts/flat/mod.rs b/vortex-layout/src/layouts/flat/mod.rs index 186ada897a7..b167e633170 100644 --- a/vortex-layout/src/layouts/flat/mod.rs +++ b/vortex-layout/src/layouts/flat/mod.rs @@ -6,6 +6,7 @@ pub mod writer; use std::env; use std::sync::Arc; +use std::sync::LazyLock; use vortex_array::DeserializeMetadata; use vortex_array::ProstMetadata; @@ -30,9 +31,10 @@ use crate::segments::SegmentSource; use crate::vtable; /// Check if inline array node is enabled. -/// This checks the env var each time to allow tests to toggle the behavior. pub(super) fn flat_layout_inline_array_node() -> bool { - env::var("FLAT_LAYOUT_INLINE_ARRAY_NODE").is_ok() + static FLAT_LAYOUT_INLINE_ARRAY_NODE: LazyLock = + LazyLock::new(|| env::var("FLAT_LAYOUT_INLINE_ARRAY_NODE").is_ok_and(|v| v == "1")); + *FLAT_LAYOUT_INLINE_ARRAY_NODE } vtable!(Flat); diff --git a/vortex-utils/Cargo.toml b/vortex-utils/Cargo.toml index 0a1dd1ad0c8..66b2576f98d 100644 --- a/vortex-utils/Cargo.toml +++ b/vortex-utils/Cargo.toml @@ -24,4 +24,3 @@ workspace = true [features] dyn-traits = [] -_test-harness = ["dashmap", "parking_lot"] diff --git a/vortex-utils/src/env.rs b/vortex-utils/src/env.rs deleted file mode 100644 index 19e78e1f12b..00000000000 --- a/vortex-utils/src/env.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! Environment variable utilities for testing. - -use dashmap::DashMap; -use parking_lot::Mutex; -use vortex_error::vortex_panic; - -/// Global registry of locks per environment variable key. -/// -/// Each mutex is lazily created and leaked to get a 'static lifetime. -/// This is acceptable for test utilities that live for the process duration. -static ENV_LOCKS: std::sync::LazyLock>> = - std::sync::LazyLock::new(DashMap::new); - -/// Get or create a static mutex for the given key. -fn get_or_create_lock(key: &'static str) -> &'static Mutex<()> { - *ENV_LOCKS - .entry(key) - .or_insert_with(|| Box::leak(Box::new(Mutex::new(())))) -} - -/// RAII guard to set/remove an environment variable for the duration of a scope. -/// -/// Removes the variable when dropped, ensuring test isolation. -/// -/// This guard holds a mutex lock for the specific environment variable key, -/// ensuring that only one guard can exist for a given key at a time. This -/// prevents tests from accidentally having overlapping guards for the same -/// env var. -/// -/// # Example -/// -/// ``` -/// use vortex_utils::env::EnvVarGuard; -/// -/// // Set an env var for the duration of this scope -/// let _guard = EnvVarGuard::set("MY_TEST_VAR", "1"); -/// assert_eq!(std::env::var("MY_TEST_VAR").ok(), Some("1".to_string())); -/// -/// // Or remove an env var -/// let _guard2 = EnvVarGuard::remove("OTHER_VAR"); -/// assert!(std::env::var("OTHER_VAR").is_err()); -/// ``` -/// -/// # Panics -/// -/// Panics if a guard already exists for the same environment variable key -/// (detected via mutex lock contention). -/// -/// # Safety -/// -/// Environment variable modification is inherently unsafe in multi-threaded contexts. -/// This guard is intended for use in tests that are run serially or where env var -/// races are acceptable. The per-key locking ensures that the same env var isn't -/// modified concurrently by multiple guards. -pub struct EnvVarGuard { - key: &'static str, - /// We store this to ensure the mutex stays locked for our lifetime. - /// The () is just a dummy value - we only care about the lock. - #[expect(dead_code)] - lock_guard: parking_lot::MutexGuard<'static, ()>, -} - -/// Timeout for waiting on an env var lock. -const LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - -impl EnvVarGuard { - /// Acquire the lock for this key, waiting up to 10 seconds. - /// - /// If another guard holds the lock, this will wait for it to be released. - /// If the lock isn't released within 10 seconds, this panics to avoid deadlocks. - fn acquire_lock(key: &'static str) -> parking_lot::MutexGuard<'static, ()> { - let mutex = get_or_create_lock(key); - match mutex.try_lock_for(LOCK_TIMEOUT) { - Some(guard) => guard, - None => vortex_panic!( - "EnvVarGuard: timed out after {LOCK_TIMEOUT:?} waiting for environment variable '{key}'. \ - This likely indicates a deadlock - ensure guards for the same key are properly scoped \ - taken in lexicographical order and dropped before acquiring a new one." - ), - } - } - - /// Set an environment variable for the duration of this guard's lifetime. - pub fn set(key: &'static str, value: &str) -> Self { - let lock_guard = Self::acquire_lock(key); - - // SAFETY: We hold an exclusive lock for this key. - unsafe { - std::env::set_var(key, value); - } - - Self { key, lock_guard } - } - - /// Remove an environment variable for the duration of this guard's lifetime. - pub fn remove(key: &'static str) -> Self { - let lock_guard = Self::acquire_lock(key); - - // SAFETY: We hold an exclusive lock for this key. - unsafe { - std::env::remove_var(key); - } - - Self { key, lock_guard } - } -} - -impl Drop for EnvVarGuard { - fn drop(&mut self) { - // SAFETY: We hold an exclusive lock for this key. - unsafe { - std::env::remove_var(self.key); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_set_and_remove() { - let key = "VORTEX_TEST_ENV_VAR_SET"; - - // Initially not set - assert!(std::env::var(key).is_err()); - - { - let _guard = EnvVarGuard::set(key, "test_value"); - assert_eq!(std::env::var(key).unwrap(), "test_value"); - } - - // After guard drops, var is removed - assert!(std::env::var(key).is_err()); - } - - #[test] - fn test_remove() { - let key = "VORTEX_TEST_ENV_VAR_REMOVE"; - - // Set it first - unsafe { - std::env::set_var(key, "initial"); - } - - { - let _guard = EnvVarGuard::remove(key); - assert!(std::env::var(key).is_err()); - } - - // After guard drops, var is still removed - assert!(std::env::var(key).is_err()); - } - - /// Test that a second guard waits for the first to be released. - #[test] - fn test_second_guard_waits_for_first() { - use std::sync::Arc; - use std::sync::atomic::AtomicBool; - use std::sync::atomic::Ordering; - use std::thread; - use std::time::Duration; - - let key = "VORTEX_TEST_ENV_VAR_WAIT"; - let second_acquired = Arc::new(AtomicBool::new(false)); - let second_acquired_clone = Arc::clone(&second_acquired); - - // First guard in main thread - let _guard1 = EnvVarGuard::set(key, "first"); - assert_eq!(std::env::var(key).unwrap(), "first"); - - // Spawn thread that will wait for the lock - let handle = thread::spawn(move || { - let _guard2 = EnvVarGuard::set(key, "second"); - second_acquired_clone.store(true, Ordering::SeqCst); - assert_eq!(std::env::var(key).unwrap(), "second"); - }); - - // Give the thread time to start waiting - thread::sleep(Duration::from_millis(50)); - - // Second guard should NOT have acquired yet (still waiting) - assert!(!second_acquired.load(Ordering::SeqCst)); - - // Drop the first guard - this should allow the second to proceed - drop(_guard1); - - // Wait for second thread to complete - handle.join().unwrap(); - - // Now the second guard should have acquired and set the value - assert!(second_acquired.load(Ordering::SeqCst)); - } -} diff --git a/vortex-utils/src/lib.rs b/vortex-utils/src/lib.rs index ff1610ddead..cc650072eab 100644 --- a/vortex-utils/src/lib.rs +++ b/vortex-utils/src/lib.rs @@ -9,6 +9,4 @@ pub mod aliases; pub mod debug_with; #[cfg(feature = "dyn-traits")] pub mod dyn_traits; -#[cfg(feature = "_test-harness")] -pub mod env; pub mod iter; From d992309a24ec17c436b1c225b835a26d3e6b1856 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 16 Apr 2026 09:14:26 -0400 Subject: [PATCH 065/250] Deprecate scalar_at for execute_scalar (#7457) --- encodings/alp/benches/alp_compress.rs | 6 +- encodings/alp/public-api.lock | 4 +- encodings/alp/src/alp/ops.rs | 4 +- encodings/alp/src/alp_rd/array.rs | 11 +- encodings/alp/src/alp_rd/ops.rs | 8 +- encodings/bytebool/src/array.rs | 13 +- encodings/datetime-parts/public-api.lock | 4 +- .../datetime-parts/src/compute/compare.rs | 30 ++-- .../datetime-parts/src/compute/is_constant.rs | 2 +- encodings/datetime-parts/src/ops.rs | 10 +- encodings/decimal-byte-parts/public-api.lock | 4 +- .../src/decimal_byte_parts/compute/compare.rs | 4 +- .../decimal_byte_parts/compute/is_constant.rs | 2 +- .../src/decimal_byte_parts/mod.rs | 21 ++- .../fastlanes/benches/canonicalize_bench.rs | 9 +- encodings/fastlanes/public-api.lock | 6 +- .../src/bitpacking/array/bitpack_compress.rs | 6 +- .../bitpacking/array/bitpack_decompress.rs | 20 ++- .../src/bitpacking/compute/is_constant.rs | 4 +- .../fastlanes/src/bitpacking/compute/take.rs | 5 +- .../src/bitpacking/vtable/operations.rs | 13 +- .../fastlanes/src/delta/vtable/operations.rs | 12 +- .../fastlanes/src/for/array/for_compress.rs | 19 +- .../fastlanes/src/for/compute/is_constant.rs | 2 +- .../fastlanes/src/for/compute/is_sorted.rs | 7 +- .../fastlanes/src/for/vtable/operations.rs | 4 +- encodings/fastlanes/src/rle/array/mod.rs | 26 +-- .../fastlanes/src/rle/array/rle_decompress.rs | 2 +- .../fastlanes/src/rle/vtable/operations.rs | 20 ++- .../fsst/benches/chunked_dict_fsst_builder.rs | 5 +- encodings/fsst/benches/fsst_compress.rs | 15 +- encodings/fsst/benches/fsst_url_compare.rs | 13 +- encodings/fsst/public-api.lock | 2 +- encodings/fsst/src/array.rs | 7 +- encodings/fsst/src/compress.rs | 6 +- encodings/fsst/src/ops.rs | 4 +- encodings/parquet-variant/src/kernel.rs | 107 +++++++++-- encodings/parquet-variant/src/operations.rs | 79 ++++++--- encodings/pco/public-api.lock | 2 +- encodings/pco/src/array.rs | 7 +- encodings/pco/src/compute/cast.rs | 8 +- encodings/runend/public-api.lock | 2 +- encodings/runend/src/array.rs | 32 ++-- encodings/runend/src/compute/cast.rs | 30 +++- encodings/runend/src/compute/is_constant.rs | 2 +- encodings/runend/src/compute/is_sorted.rs | 17 +- encodings/runend/src/kernel.rs | 6 +- encodings/runend/src/ops.rs | 8 +- encodings/sequence/src/array.rs | 4 +- encodings/sequence/src/compress.rs | 7 +- encodings/sequence/src/compute/is_sorted.rs | 8 +- encodings/sparse/src/canonical.rs | 60 +++++-- encodings/sparse/src/lib.rs | 71 ++++++-- encodings/zigzag/public-api.lock | 2 +- encodings/zigzag/src/array.rs | 36 ++-- encodings/zigzag/src/compute/mod.rs | 4 +- encodings/zstd/public-api.lock | 2 +- encodings/zstd/src/array.rs | 7 +- encodings/zstd/src/test.rs | 7 +- encodings/zstd/src/zstd_buffers.rs | 17 +- fuzz/fuzz_targets/file_io.rs | 7 +- fuzz/src/array/compare.rs | 3 +- fuzz/src/array/fill_null.rs | 38 ++-- fuzz/src/array/mask.rs | 106 ++++++++--- fuzz/src/array/mod.rs | 15 +- fuzz/src/array/scalar_at.rs | 4 +- fuzz/src/array/search_sorted.rs | 3 +- fuzz/src/array/sort.rs | 12 +- fuzz/src/array/take.rs | 3 +- fuzz/src/gpu/mod.rs | 12 +- vortex-array/benches/cast_primitive.rs | 12 +- vortex-array/benches/chunk_array_builder.rs | 13 +- vortex-array/benches/dict_compress.rs | 15 +- vortex-array/benches/listview_rebuild.rs | 57 +++++- vortex-array/benches/scalar_at_struct.rs | 12 +- vortex-array/benches/take_primitive.rs | 21 ++- vortex-array/public-api.lock | 158 +++++++++-------- .../src/aggregate_fn/fns/first/mod.rs | 2 +- .../src/aggregate_fn/fns/is_constant/mod.rs | 32 ++-- .../src/aggregate_fn/fns/is_sorted/mod.rs | 36 ++-- vortex-array/src/aggregate_fn/fns/last/mod.rs | 2 +- vortex-array/src/aggregate_fn/foreign.rs | 4 + vortex-array/src/aggregate_fn/proto.rs | 4 + vortex-array/src/aggregate_fn/vtable.rs | 9 +- vortex-array/src/array/erased.rs | 29 +-- vortex-array/src/array/mod.rs | 36 ++-- vortex-array/src/array/typed.rs | 42 +++-- vortex-array/src/arrays/assertions.rs | 23 ++- vortex-array/src/arrays/bool/array.rs | 57 ++++-- vortex-array/src/arrays/chunked/array.rs | 41 ++++- vortex-array/src/arrays/chunked/tests.rs | 18 +- .../src/arrays/chunked/vtable/canonical.rs | 18 +- .../src/arrays/chunked/vtable/operations.rs | 4 +- .../src/arrays/constant/vtable/canonical.rs | 100 +++++++++-- .../src/arrays/decimal/vtable/operations.rs | 6 +- vortex-array/src/arrays/dict/compute/cast.rs | 19 +- .../src/arrays/dict/compute/fill_null.rs | 8 +- .../src/arrays/dict/compute/is_constant.rs | 4 +- .../src/arrays/dict/compute/is_sorted.rs | 2 +- vortex-array/src/arrays/dict/compute/rules.rs | 6 +- vortex-array/src/arrays/dict/take.rs | 7 +- vortex-array/src/arrays/dict/vtable/mod.rs | 2 +- .../src/arrays/dict/vtable/operations.rs | 6 +- .../src/arrays/extension/vtable/operations.rs | 4 +- .../src/arrays/filter/execute/listview.rs | 6 +- vortex-array/src/arrays/filter/vtable.rs | 4 +- .../arrays/fixed_size_list/compute/take.rs | 2 +- .../src/arrays/fixed_size_list/tests/basic.rs | 164 ++++++++++++++--- .../fixed_size_list/tests/degenerate.rs | 34 +++- .../arrays/fixed_size_list/tests/nested.rs | 167 +++++++++++++++--- .../fixed_size_list/tests/nullability.rs | 148 +++++++++++++--- .../src/arrays/fixed_size_list/tests/take.rs | 18 +- .../fixed_size_list/vtable/operations.rs | 4 +- vortex-array/src/arrays/list/array.rs | 8 +- vortex-array/src/arrays/list/compute/take.rs | 98 ++++++++-- vortex-array/src/arrays/list/tests.rs | 94 +++++++--- .../src/arrays/list/vtable/operations.rs | 4 +- vortex-array/src/arrays/listview/array.rs | 13 +- vortex-array/src/arrays/listview/rebuild.rs | 43 ++++- .../src/arrays/listview/tests/basic.rs | 40 ++++- .../src/arrays/listview/tests/filter.rs | 6 +- .../src/arrays/listview/tests/nested.rs | 21 ++- .../src/arrays/listview/tests/nullability.rs | 123 +++++++++++-- .../src/arrays/listview/tests/operations.rs | 113 ++++++++++-- .../src/arrays/listview/tests/take.rs | 6 +- .../src/arrays/listview/vtable/operations.rs | 4 +- vortex-array/src/arrays/masked/array.rs | 8 +- .../src/arrays/masked/compute/take.rs | 7 +- vortex-array/src/arrays/masked/tests.rs | 33 +++- .../src/arrays/masked/vtable/canonical.rs | 41 ++++- vortex-array/src/arrays/masked/vtable/mod.rs | 8 +- .../src/arrays/masked/vtable/operations.rs | 4 +- vortex-array/src/arrays/null/compute/cast.rs | 9 +- vortex-array/src/arrays/null/compute/mod.rs | 4 +- vortex-array/src/arrays/null/mod.rs | 5 +- vortex-array/src/arrays/patched/array.rs | 18 +- .../src/arrays/patched/vtable/operations.rs | 43 ++++- .../src/arrays/primitive/array/mod.rs | 4 +- .../src/arrays/primitive/compute/take/mod.rs | 14 +- vortex-array/src/arrays/scalar_fn/rules.rs | 6 +- .../src/arrays/scalar_fn/vtable/operations.rs | 36 ++-- vortex-array/src/arrays/shared/vtable.rs | 4 +- vortex-array/src/arrays/slice/vtable.rs | 4 +- vortex-array/src/arrays/struct_/array.rs | 12 +- .../src/arrays/struct_/compute/mod.rs | 7 +- vortex-array/src/arrays/struct_/tests.rs | 12 +- .../src/arrays/struct_/vtable/operations.rs | 4 +- vortex-array/src/arrays/varbin/array.rs | 11 +- vortex-array/src/arrays/varbin/builder.rs | 29 ++- .../src/arrays/varbin/vtable/canonical.rs | 8 +- .../src/arrays/variant/vtable/operations.rs | 4 +- vortex-array/src/arrow/datum.rs | 6 +- vortex-array/src/arrow/executor/list.rs | 2 +- vortex-array/src/arrow/record_batch.rs | 4 +- vortex-array/src/builders/decimal.rs | 14 +- vortex-array/src/builders/fixed_size_list.rs | 10 +- vortex-array/src/builders/list.rs | 28 ++- vortex-array/src/builders/mod.rs | 20 ++- vortex-array/src/builders/primitive.rs | 56 +++++- vortex-array/src/builders/tests.rs | 96 ++++++++-- vortex-array/src/builders/varbinview.rs | 2 +- .../src/compute/conformance/binary_numeric.rs | 2 +- vortex-array/src/compute/conformance/cast.rs | 26 +-- .../src/compute/conformance/consistency.rs | 102 +++++------ .../src/compute/conformance/filter.rs | 26 +-- vortex-array/src/compute/conformance/mask.rs | 40 ++--- vortex-array/src/compute/conformance/take.rs | 55 +++--- vortex-array/src/display/mod.rs | 18 +- vortex-array/src/expr/stats/mod.rs | 4 +- vortex-array/src/patches.rs | 88 +++++++-- vortex-array/src/scalar_fn/fns/between/mod.rs | 10 -- .../src/scalar_fn/fns/binary/boolean.rs | 50 +++++- .../src/scalar_fn/fns/binary/compare.rs | 27 ++- vortex-array/src/scalar_fn/fns/binary/mod.rs | 10 +- vortex-array/src/scalar_fn/fns/case_when.rs | 31 ++-- .../src/scalar_fn/fns/fill_null/kernel.rs | 10 +- vortex-array/src/scalar_fn/fns/is_not_null.rs | 18 +- vortex-array/src/scalar_fn/fns/is_null.rs | 18 +- .../src/scalar_fn/fns/list_contains/mod.rs | 47 +++-- vortex-array/src/search_sorted.rs | 4 +- vortex-array/src/stats/array.rs | 72 ++++---- vortex-array/src/stats/stats_set.rs | 7 +- vortex-array/src/validity.rs | 34 ++-- vortex-array/src/variants.rs | 9 +- vortex-btrblocks/src/schemes/patches.rs | 7 +- vortex-btrblocks/src/schemes/temporal.rs | 8 +- .../src/builtins/constant/mod.rs | 11 +- vortex-compressor/src/compressor.rs | 2 +- vortex-compressor/src/stats/bool.rs | 11 +- vortex-compressor/src/stats/float.rs | 5 +- vortex-compressor/src/stats/integer.rs | 7 +- vortex-compressor/src/stats/string.rs | 4 +- vortex-cuda/src/canonical.rs | 14 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 12 +- vortex-cuda/src/hybrid_dispatch/mod.rs | 6 +- vortex-cuda/src/kernel/arrays/constant.rs | 6 +- vortex-cuda/src/kernel/arrays/dict.rs | 44 ++--- vortex-cuda/src/kernel/arrays/shared.rs | 2 +- vortex-cuda/src/kernel/encodings/alp.rs | 6 +- vortex-cuda/src/kernel/encodings/bitpacked.rs | 14 +- .../src/kernel/encodings/date_time_parts.rs | 6 +- .../kernel/encodings/decimal_byte_parts.rs | 3 +- vortex-cuda/src/kernel/encodings/for_.rs | 6 +- vortex-cuda/src/kernel/encodings/runend.rs | 12 +- vortex-cuda/src/kernel/encodings/sequence.rs | 2 +- vortex-cuda/src/kernel/encodings/zigzag.rs | 2 +- vortex-cuda/src/kernel/encodings/zstd.rs | 13 +- .../src/kernel/encodings/zstd_buffers.rs | 4 +- vortex-cuda/src/kernel/filter/decimal.rs | 4 +- vortex-cuda/src/kernel/filter/primitive.rs | 4 +- vortex-cuda/src/kernel/filter/varbinview.rs | 2 +- vortex-cuda/src/kernel/patches/mod.rs | 9 +- vortex-cuda/src/kernel/slice/mod.rs | 38 ++-- vortex-cuda/src/lib.rs | 12 ++ vortex-duckdb/src/exporter/run_end.rs | 2 +- vortex-duckdb/src/exporter/struct_.rs | 2 +- vortex-ffi/src/array.rs | 14 +- vortex-file/src/tests.rs | 6 +- vortex-file/src/v2/file_stats_reader.rs | 2 +- vortex-jni/src/array.rs | 32 +++- vortex-layout/src/layouts/compressed.rs | 10 +- vortex-layout/src/layouts/flat/writer.rs | 3 +- vortex-layout/src/layouts/struct_/reader.rs | 18 +- vortex-layout/src/layouts/zoned/builder.rs | 4 +- vortex-layout/src/layouts/zoned/writer.rs | 12 +- vortex-layout/src/layouts/zoned/zone_map.rs | 8 +- vortex-python/src/arrays/mod.rs | 5 +- vortex-python/src/scan.rs | 4 +- .../src/encodings/turboquant/tests/compute.rs | 6 +- .../encodings/turboquant/tests/nullable.rs | 4 +- .../src/scalar_fns/cosine_similarity.rs | 8 +- vortex-tensor/src/scalar_fns/inner_product.rs | 10 +- vortex-tensor/src/scalar_fns/l2_denorm.rs | 8 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 4 +- vortex-test/compat-gen/src/adapter.rs | 5 +- .../arrays/synthetic/encodings/constant.rs | 4 +- vortex-tui/src/browse/ui/layouts.rs | 4 +- .../common_encoding_tree_throughput.rs | 5 +- vortex/benches/single_encoding_throughput.rs | 65 ++++--- 239 files changed, 3440 insertions(+), 1349 deletions(-) diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index 8817daedbbc..d0a23b64b21 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -12,6 +12,8 @@ use vortex_alp::ALPRDFloat; use vortex_alp::RDEncoder; use vortex_alp::alp_encode; use vortex_alp::decompress_into_array; +use vortex_array::Canonical; +use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -153,6 +155,6 @@ fn decompress_rd(bencher: Bencher, args: (usize, f6 let encoded = encoder.encode(primitive.as_view()); bencher - .with_inputs(|| &encoded) - .bench_refs(|encoded| encoded.to_canonical()); + .with_inputs(|| (&encoded, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(encoded, ctx)| (**encoded).clone().into_array().execute::(ctx)); } diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index a2fa68e3cfa..b4dcad31098 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -52,7 +52,7 @@ pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vort impl vortex_array::array::vtable::operations::OperationsVTable for vortex_alp::ALP -pub fn vortex_alp::ALP::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALP>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALP::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALP>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_alp::ALP @@ -188,7 +188,7 @@ pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: & impl vortex_array::array::vtable::operations::OperationsVTable for vortex_alp::ALPRD -pub fn vortex_alp::ALPRD::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALPRD>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRD::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_alp::ALPRD>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_alp::ALPRD diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index dcd8cecf1d5..a8850744056 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -18,7 +18,7 @@ impl OperationsVTable for ALP { fn scalar_at( array: ArrayView<'_, ALP>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { if let Some(patches) = array.patches() && let Some(patch) = patches.get_patched(index)? @@ -26,7 +26,7 @@ impl OperationsVTable for ALP { return patch.cast(array.dtype()); } - let encoded_val = array.encoded().scalar_at(index)?; + let encoded_val = array.encoded().execute_scalar(index, ctx)?; Ok(match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| { let encoded_val: ::ALPInt = diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 0cdac5c4001..ae224c52b7d 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -19,8 +19,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::Precision; use vortex_array::TypedArrayRef; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -405,7 +407,10 @@ impl ALPRDData { ) -> VortexResult> { left_parts_patches .map(|patches| { - if !patches.values().all_valid()? { + if !patches + .values() + .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? + { vortex_bail!("patches must be all valid: {}", patches.values()); } // TODO(ngates): assert the DType, don't cast it. @@ -586,7 +591,9 @@ fn validate_parts( left_parts.dtype(), ); vortex_ensure!( - patches.values().all_valid()?, + patches + .values() + .all_valid(&mut LEGACY_SESSION.create_execution_ctx())?, "patches must be all valid: {}", patches.values() ); diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 90f1307f59f..be30a3f7f03 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -16,7 +16,7 @@ impl OperationsVTable for ALPRD { fn scalar_at( array: ArrayView<'_, ALPRD>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // The left value can either be a direct value, or an exception. // The exceptions array represents exception positions with non-null values. @@ -32,7 +32,7 @@ impl OperationsVTable for ALPRD { _ => { let left_code: u16 = array .left_parts() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("left_code must be non-null"); @@ -44,7 +44,7 @@ impl OperationsVTable for ALPRD { Ok(if array.dtype().as_ptype() == PType::F32 { let right: u32 = array .right_parts() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("non-null"); @@ -53,7 +53,7 @@ impl OperationsVTable for ALPRD { } else { let right: u64 = array .right_parts() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("non-null"); diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 42cc48c094e..1043aefa967 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -347,6 +347,8 @@ impl From>> for ByteBoolData { mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; @@ -366,15 +368,16 @@ mod tests { let arr = ByteBool::from_vec(v, Validity::AllValid); assert_eq!(v_len, arr.len()); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); for idx in 0..arr.len() { - assert!(arr.is_valid(idx).unwrap()); + assert!(arr.is_valid(idx, &mut ctx).unwrap()); } let v = vec![Some(true), None, Some(false)]; let arr = ByteBool::from_option_vec(v); - assert!(arr.is_valid(0).unwrap()); - assert!(!arr.is_valid(1).unwrap()); - assert!(arr.is_valid(2).unwrap()); + assert!(arr.is_valid(0, &mut ctx).unwrap()); + assert!(!arr.is_valid(1, &mut ctx).unwrap()); + assert!(arr.is_valid(2, &mut ctx).unwrap()); assert_eq!(arr.len(), 3); let v: Vec> = vec![None, None]; @@ -384,7 +387,7 @@ mod tests { assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { - assert!(!arr.is_valid(idx).unwrap()); + assert!(!arr.is_valid(idx, &mut ctx).unwrap()); } assert_eq!(arr.len(), 2); } diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index 555681ac824..e5935cf15a8 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -48,7 +48,7 @@ pub fn vortex_datetime_parts::DateTimeParts::validate(&self, _data: &Self::Array impl vortex_array::array::vtable::operations::OperationsVTable for vortex_datetime_parts::DateTimeParts -pub fn vortex_datetime_parts::DateTimeParts::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_datetime_parts::DateTimeParts>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimeParts::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_datetime_parts::DateTimeParts>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_datetime_parts::DateTimeParts @@ -68,7 +68,7 @@ pub fn vortex_datetime_parts::DateTimeParts::slice(array: vortex_array::array::v impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_datetime_parts::DateTimeParts -pub fn vortex_datetime_parts::DateTimeParts::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_datetime_parts::DateTimeParts::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_datetime_parts::DateTimeParts diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index bd50a7a5d0f..3d2c76e81e3 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -25,7 +25,7 @@ impl CompareKernel for DateTimeParts { lhs: ArrayView<'_, Self>, rhs: &ArrayRef, operator: CompareOperator, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let Some(rhs_const) = rhs.as_constant() else { return Ok(None); @@ -51,17 +51,17 @@ impl CompareKernel for DateTimeParts { let ts_parts = timestamp::split(timestamp, options.unit)?; match operator { - CompareOperator::Eq => compare_eq(lhs, &ts_parts, nullability), - CompareOperator::NotEq => compare_ne(lhs, &ts_parts, nullability), + CompareOperator::Eq => compare_eq(lhs, &ts_parts, nullability, ctx), + CompareOperator::NotEq => compare_ne(lhs, &ts_parts, nullability, ctx), // lt and lte have identical behavior, as we optimize // for the case that all days on the lhs are smaller. // If that special case is not hit, we return `Ok(None)` to // signal that the comparison wasn't handled within dtp. - CompareOperator::Lt => compare_lt(lhs, &ts_parts, nullability), - CompareOperator::Lte => compare_lt(lhs, &ts_parts, nullability), + CompareOperator::Lt => compare_lt(lhs, &ts_parts, nullability, ctx), + CompareOperator::Lte => compare_lt(lhs, &ts_parts, nullability, ctx), // (Like for lt, lte) - CompareOperator::Gt => compare_gt(lhs, &ts_parts, nullability), - CompareOperator::Gte => compare_gt(lhs, &ts_parts, nullability), + CompareOperator::Gt => compare_gt(lhs, &ts_parts, nullability, ctx), + CompareOperator::Gte => compare_gt(lhs, &ts_parts, nullability, ctx), } } } @@ -70,9 +70,10 @@ fn compare_eq( lhs: ArrayView, ts_parts: ×tamp::TimestampParts, nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut comparison = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Eq, nullability)?; - if comparison.statistics().compute_max::() == Some(false) { + if comparison.statistics().compute_max::(ctx) == Some(false) { // All values are different. return Ok(Some(comparison)); } @@ -85,7 +86,7 @@ fn compare_eq( )? .binary(comparison, Operator::And)?; - if comparison.statistics().compute_max::() == Some(false) { + if comparison.statistics().compute_max::(ctx) == Some(false) { // All values are different. return Ok(Some(comparison)); } @@ -105,6 +106,7 @@ fn compare_ne( lhs: ArrayView, ts_parts: ×tamp::TimestampParts, nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut comparison = compare_dtp( lhs.days(), @@ -112,7 +114,7 @@ fn compare_ne( CompareOperator::NotEq, nullability, )?; - if comparison.statistics().compute_min::() == Some(true) { + if comparison.statistics().compute_min::(ctx) == Some(true) { // All values are different. return Ok(Some(comparison)); } @@ -125,7 +127,7 @@ fn compare_ne( )? .binary(comparison, Operator::Or)?; - if comparison.statistics().compute_min::() == Some(true) { + if comparison.statistics().compute_min::(ctx) == Some(true) { // All values are different. return Ok(Some(comparison)); } @@ -145,9 +147,10 @@ fn compare_lt( lhs: ArrayView, ts_parts: ×tamp::TimestampParts, nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let days_lt = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Lt, nullability)?; - if days_lt.statistics().compute_min::() == Some(true) { + if days_lt.statistics().compute_min::(ctx) == Some(true) { // All values on the lhs are smaller. return Ok(Some(days_lt)); } @@ -159,9 +162,10 @@ fn compare_gt( lhs: ArrayView, ts_parts: ×tamp::TimestampParts, nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let days_gt = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Gt, nullability)?; - if days_gt.statistics().compute_min::() == Some(true) { + if days_gt.statistics().compute_min::(ctx) == Some(true) { // All values on the lhs are larger. return Ok(Some(days_gt)); } diff --git a/encodings/datetime-parts/src/compute/is_constant.rs b/encodings/datetime-parts/src/compute/is_constant.rs index 94e6806e785..d84e294f7aa 100644 --- a/encodings/datetime-parts/src/compute/is_constant.rs +++ b/encodings/datetime-parts/src/compute/is_constant.rs @@ -37,6 +37,6 @@ impl DynAggregateKernel for DateTimePartsIsConstantKernel { let result = is_constant(array.days(), ctx)? && is_constant(array.seconds(), ctx)? && is_constant(array.subseconds(), ctx)?; - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/encodings/datetime-parts/src/ops.rs b/encodings/datetime-parts/src/ops.rs index 2c224b078d5..75ddcd87fa8 100644 --- a/encodings/datetime-parts/src/ops.rs +++ b/encodings/datetime-parts/src/ops.rs @@ -20,7 +20,7 @@ impl OperationsVTable for DateTimeParts { fn scalar_at( array: ArrayView<'_, DateTimeParts>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let DType::Extension(ext) = array.dtype().clone() else { vortex_panic!( @@ -33,25 +33,25 @@ impl OperationsVTable for DateTimeParts { vortex_panic!(Compute: "must decode TemporalMetadata from extension metadata"); }; - if !array.as_ref().is_valid(index)? { + if !array.as_ref().is_valid(index, ctx)? { return Ok(Scalar::null(DType::Extension(ext))); } let days: i64 = array .days() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("days fits in i64"); let seconds: i64 = array .seconds() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("seconds fits in i64"); let subseconds: i64 = array .subseconds() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() .vortex_expect("subseconds fits in i64"); diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 57a83581aa2..4c4162b1f92 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -46,7 +46,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, _data: &Self impl vortex_array::array::vtable::operations::OperationsVTable for vortex_decimal_byte_parts::DecimalByteParts -pub fn vortex_decimal_byte_parts::DecimalByteParts::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_decimal_byte_parts::DecimalByteParts>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalByteParts::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_decimal_byte_parts::DecimalByteParts>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_decimal_byte_parts::DecimalByteParts @@ -66,7 +66,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::slice(array: vortex_array::a impl vortex_array::scalar_fn::fns::binary::compare::CompareKernel for vortex_decimal_byte_parts::DecimalByteParts -pub fn vortex_decimal_byte_parts::DecimalByteParts::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_decimal_byte_parts::DecimalByteParts::compare(lhs: vortex_array::array::view::ArrayView<'_, Self>, rhs: &vortex_array::array::erased::ArrayRef, operator: vortex_array::scalar_fn::fns::operators::CompareOperator, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_decimal_byte_parts::DecimalByteParts diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index 43f06b6e695..e6cc8d4d72d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -33,7 +33,7 @@ impl CompareKernel for DecimalByteParts { lhs: ArrayView<'_, Self>, rhs: &ArrayRef, operator: CompareOperator, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let Some(rhs_const) = rhs.as_constant() else { return Ok(None); @@ -65,7 +65,7 @@ impl CompareKernel for DecimalByteParts { // (depending on the `sign`) than all values in MSP. // If the LHS or the RHS contain nulls, then we must fallback to the canonicalized // implementation which does null-checking instead. - if lhs.array().all_valid()? && rhs.all_valid()? { + if lhs.array().all_valid(ctx)? && rhs.all_valid(ctx)? { Ok(Some( ConstantArray::new( unconvertible_value(sign, operator, nullability), diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs index 4d271eb80b3..f4da528a9b3 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/is_constant.rs @@ -35,6 +35,6 @@ impl DynAggregateKernel for DecimalBytePartsIsConstantKernel { }; let result = is_constant(array.msp(), ctx)?; - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 390fd15f0fe..80a7772d986 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -303,10 +303,10 @@ impl OperationsVTable for DecimalByteParts { fn scalar_at( array: ArrayView<'_, DecimalByteParts>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(joe): support parts len != 1 - let scalar = array.msp().scalar_at(index)?; + let scalar = array.msp().execute_scalar(index, ctx)?; // Note. values in msp, can only be signed integers upto size i64. let primitive_scalar = scalar.as_primitive(); @@ -329,6 +329,8 @@ impl ValidityChild for DecimalByteParts { #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; @@ -357,18 +359,27 @@ mod tests { .unwrap() .into_array(); - assert_eq!(Scalar::null(dtype.clone()), array.scalar_at(0).unwrap()); + assert_eq!( + Scalar::null(dtype.clone()), + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( Scalar::try_new( dtype.clone(), Some(ScalarValue::Decimal(DecimalValue::I64(200))) ) .unwrap(), - array.scalar_at(1).unwrap() + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); assert_eq!( Scalar::try_new(dtype, Some(ScalarValue::Decimal(DecimalValue::I64(400)))).unwrap(), - array.scalar_at(2).unwrap() + array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); } } diff --git a/encodings/fastlanes/benches/canonicalize_bench.rs b/encodings/fastlanes/benches/canonicalize_bench.rs index 666bf1420a9..bbcdf6a72c1 100644 --- a/encodings/fastlanes/benches/canonicalize_bench.rs +++ b/encodings/fastlanes/benches/canonicalize_bench.rs @@ -53,8 +53,13 @@ fn into_canonical_non_nullable( .collect::>(); bencher - .with_inputs(|| ChunkedArray::from_iter(chunks.clone()).into_array()) - .bench_refs(|chunked| chunked.to_canonical()); + .with_inputs(|| { + ( + ChunkedArray::from_iter(chunks.clone()).into_array(), + SESSION.create_execution_ctx(), + ) + }) + .bench_refs(|(chunked, ctx)| chunked.clone().execute::(ctx)); } #[cfg(not(codspeed))] diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 0a84e217c76..4f8e8f67cfe 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -320,7 +320,7 @@ pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: & impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::Delta -pub fn vortex_fastlanes::Delta::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::Delta>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::Delta::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::Delta>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fastlanes::Delta @@ -408,7 +408,7 @@ pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vo impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::FoR -pub fn vortex_fastlanes::FoR::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::FoR>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoR::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::FoR>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_fastlanes::FoR @@ -514,7 +514,7 @@ pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vo impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::RLE>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::RLE>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fastlanes::RLE diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 7c7e1bc5d26..727e69dad04 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -52,7 +52,11 @@ pub fn bitpack_encode( // Check array contains no negative values. if array.ptype().is_signed_int() { let has_negative_values = match_each_integer_ptype!(array.ptype(), |P| { - array.statistics().compute_min::

().unwrap_or_default() < 0 + array + .statistics() + .compute_min::

(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap_or_default() + < 0 }); if has_negative_values { vortex_bail!(InvalidArgument: "cannot bitpack_encode array containing negative integers") diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 465c467bcfb..337f82691a6 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -97,7 +97,7 @@ pub fn apply_patches_to_uninit_range_fn T>( let indices = patches.indices().clone().execute::(ctx)?; let values = patches.values().clone().execute::(ctx)?; - assert!(values.all_valid()?, "Patch values must be all valid"); + assert!(values.all_valid(ctx)?, "Patch values must be all valid"); let values = values.as_slice::(); match_each_unsigned_integer_ptype!(indices.ptype(), |P| { @@ -369,11 +369,12 @@ mod tests { // Verify the validity mask was correctly applied. assert_eq!(result.len(), 5); - assert!(!result.scalar_at(0).unwrap().is_null()); - assert!(result.scalar_at(1).unwrap().is_null()); - assert!(!result.scalar_at(2).unwrap().is_null()); - assert!(!result.scalar_at(3).unwrap().is_null()); - assert!(result.scalar_at(4).unwrap().is_null()); + let mut ctx = SESSION.create_execution_ctx(); + assert!(!result.execute_scalar(0, &mut ctx).unwrap().is_null()); + assert!(result.execute_scalar(1, &mut ctx).unwrap().is_null()); + assert!(!result.execute_scalar(2, &mut ctx).unwrap().is_null()); + assert!(!result.execute_scalar(3, &mut ctx).unwrap().is_null()); + assert!(result.execute_scalar(4, &mut ctx).unwrap().is_null()); Ok(()) } @@ -456,9 +457,10 @@ mod tests { // Verify length. assert_eq!(result.len(), 7); // Validity should be preserved when unpacking. - assert!(!result.scalar_at(0).unwrap().is_null()); - assert!(result.scalar_at(1).unwrap().is_null()); - assert!(!result.scalar_at(2).unwrap().is_null()); + let mut ctx = SESSION.create_execution_ctx(); + assert!(!result.execute_scalar(0, &mut ctx).unwrap().is_null()); + assert!(result.execute_scalar(1, &mut ctx).unwrap().is_null()); + assert!(!result.execute_scalar(2, &mut ctx).unwrap().is_null()); // Test combining patches with nullability. let patch_values = Buffer::from_iter([10u16, 0, 2000, 0, 30, 3000, 0]); diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 900284e926d..94f3feea46f 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -34,7 +34,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { &self, aggregate_fn: &AggregateFnRef, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { if !aggregate_fn.is::() { return Ok(None); @@ -48,7 +48,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { bitpacked_is_constant::() }>(array)? }); - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 5eb03415c88..4ff2b484818 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -234,17 +234,18 @@ mod test { let taken = packed.take(random_indices.clone().into_array()).unwrap(); // sanity check + let mut ctx = LEGACY_SESSION.create_execution_ctx(); random_indices .as_slice::() .iter() .enumerate() .for_each(|(ti, i)| { assert_eq!( - u32::try_from(&packed.scalar_at(*i as usize).unwrap()).unwrap(), + u32::try_from(&packed.execute_scalar(*i as usize, &mut ctx).unwrap()).unwrap(), values[*i as usize] ); assert_eq!( - u32::try_from(&taken.scalar_at(ti).unwrap()).unwrap(), + u32::try_from(&taken.execute_scalar(ti, &mut ctx).unwrap()).unwrap(), values[*i as usize] ); }); diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 13e22a2f11c..74c5baa53a4 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -34,6 +34,8 @@ mod test { use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::SliceArray; use vortex_array::assert_arrays_eq; @@ -202,7 +204,9 @@ mod test { .unwrap() .into_array(); assert_eq!( - packed_array.scalar_at(1).unwrap(), + packed_array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::null(DType::Primitive(PType::U32, Nullability::Nullable)) ); } @@ -216,7 +220,12 @@ mod test { let patches = packed.patches().unwrap().indices().clone(); assert_eq!( - usize::try_from(&patches.scalar_at(0).unwrap()).unwrap(), + usize::try_from( + &patches + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 256 ); diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index 731588a67bb..d0c0606ea93 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -14,10 +14,10 @@ impl OperationsVTable for Delta { fn scalar_at( array: ArrayView<'_, Delta>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let decompressed = array.array().slice(index..index + 1)?.to_primitive(); - decompressed.into_array().scalar_at(0) + decompressed.into_array().execute_scalar(0, ctx) } } @@ -215,7 +215,9 @@ mod tests { #[should_panic] fn test_scalar_at_non_jagged_array_oob() { let delta = da(&(0u32..2048).collect()).into_array(); - delta.scalar_at(2048).unwrap(); + delta + .execute_scalar(2048, &mut SESSION.create_execution_ctx()) + .unwrap(); } #[test] fn test_scalar_at_jagged_array() { @@ -229,7 +231,9 @@ mod tests { #[should_panic] fn test_scalar_at_jagged_array_oob() { let delta = da(&(0u32..2000).collect()).into_array(); - delta.scalar_at(2000).unwrap(); + delta + .execute_scalar(2000, &mut SESSION.create_execution_ctx()) + .unwrap(); } #[rstest] diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 266e6306a9c..f27379a866f 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -4,6 +4,8 @@ use num_traits::PrimInt; use num_traits::WrappingSub; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::expr::stats::Stat; @@ -19,7 +21,7 @@ impl FoRData { let array_ref = array.clone().into_array(); let min = array_ref .statistics() - .compute_stat(Stat::Min)? + .compute_stat(Stat::Min, &mut LEGACY_SESSION.create_execution_ctx())? .ok_or_else(|| vortex_err!("Min stat not found"))?; let encoded = match_each_integer_ptype!(array.ptype(), |T| { @@ -110,7 +112,10 @@ mod test { assert!(compressed.reference_scalar().dtype().is_signed_int()); assert!(compressed.encoded().dtype().is_signed_int()); - let encoded = compressed.encoded().scalar_at(0).unwrap(); + let encoded = compressed + .encoded() + .execute_scalar(0, &mut SESSION.create_execution_ctx()) + .unwrap(); assert_eq!(encoded, Scalar::from(0i32)); } @@ -175,7 +180,15 @@ mod test { .iter() .enumerate() .for_each(|(i, v)| { - assert_eq!(*v, i8::try_from(&compressed.scalar_at(i).unwrap()).unwrap()); + assert_eq!( + *v, + i8::try_from( + &compressed + .execute_scalar(i, &mut SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap() + ); }); assert_arrays_eq!(decompressed, array); Ok(()) diff --git a/encodings/fastlanes/src/for/compute/is_constant.rs b/encodings/fastlanes/src/for/compute/is_constant.rs index 246a9c93226..29393884e95 100644 --- a/encodings/fastlanes/src/for/compute/is_constant.rs +++ b/encodings/fastlanes/src/for/compute/is_constant.rs @@ -35,6 +35,6 @@ impl DynAggregateKernel for FoRIsConstantKernel { }; let result = is_constant(array.encoded(), ctx)?; - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 3b81c45c03d..92cd8b36888 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -49,7 +49,12 @@ impl DynAggregateKernel for FoRIsSortedKernel { is_sorted(&unsigned_array, ctx)? }; - Ok(Some(IsSorted::make_partial(batch, result, options.strict)?)) + Ok(Some(IsSorted::make_partial( + batch, + result, + options.strict, + ctx, + )?)) } } diff --git a/encodings/fastlanes/src/for/vtable/operations.rs b/encodings/fastlanes/src/for/vtable/operations.rs index e90ff7f4ad2..5fea6f4bdaa 100644 --- a/encodings/fastlanes/src/for/vtable/operations.rs +++ b/encodings/fastlanes/src/for/vtable/operations.rs @@ -15,9 +15,9 @@ impl OperationsVTable for FoR { fn scalar_at( array: ArrayView<'_, FoR>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let encoded_pvalue = array.encoded().scalar_at(index)?; + let encoded_pvalue = array.encoded().execute_scalar(index, ctx)?; let encoded_pvalue = encoded_pvalue.as_primitive(); let reference = array.reference_scalar(); let reference = reference.as_primitive(); diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 9857874d040..5c9ab27c33a 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -5,7 +5,9 @@ use std::fmt::Display; use std::fmt::Formatter; use vortex_array::ArrayRef; +use vortex_array::LEGACY_SESSION; use vortex_array::TypedArrayRef; +use vortex_array::VortexSessionExecute; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -111,14 +113,14 @@ pub trait RLEArrayExt: TypedArrayRef { )] fn values_idx_offset(&self, chunk_idx: usize) -> usize { self.values_idx_offsets() - .scalar_at(chunk_idx) + .execute_scalar(chunk_idx, &mut LEGACY_SESSION.create_execution_ctx()) .expect("index must be in bounds") .as_primitive() .as_::() .expect("index must be of type usize") - self .values_idx_offsets() - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .expect("index must be in bounds") .as_primitive() .as_::() @@ -137,6 +139,7 @@ impl> RLEArrayExt for T {} #[cfg(test)] mod tests { use vortex_array::ArrayContext; + use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; @@ -204,9 +207,10 @@ mod tests { assert_eq!(rle_array.len(), 3); assert_eq!(rle_array.values().len(), 2); - assert!(rle_array.is_valid(0).unwrap()); - assert!(!rle_array.is_valid(1).unwrap()); - assert!(rle_array.is_valid(2).unwrap()); + let mut ctx = SESSION.create_execution_ctx(); + assert!(rle_array.is_valid(0, &mut ctx).unwrap()); + assert!(!rle_array.is_valid(1, &mut ctx).unwrap()); + assert!(rle_array.is_valid(2, &mut ctx).unwrap()); } #[test] @@ -234,10 +238,11 @@ mod tests { let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); // TODO(joe): replace with compute null count - assert!(valid_slice.all_valid().unwrap()); + let mut ctx = SESSION.create_execution_ctx(); + assert!(valid_slice.all_valid(&mut ctx).unwrap()); let mixed_slice = rle_array.slice(1..5).unwrap(); - assert!(!mixed_slice.all_valid().unwrap()); + assert!(!mixed_slice.all_valid(&mut ctx).unwrap()); } #[test] @@ -267,13 +272,14 @@ mod tests { let invalid_slice = rle_array .slice(2..5) .unwrap() - .to_canonical() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .into_primitive(); - assert!(invalid_slice.all_invalid().unwrap()); + let mut ctx = SESSION.create_execution_ctx(); + assert!(invalid_slice.all_invalid(&mut ctx).unwrap()); let mixed_slice = rle_array.slice(1..4).unwrap(); - assert!(!mixed_slice.all_invalid().unwrap()); + assert!(!mixed_slice.all_invalid(&mut ctx).unwrap()); } #[test] diff --git a/encodings/fastlanes/src/rle/array/rle_decompress.rs b/encodings/fastlanes/src/rle/array/rle_decompress.rs index d9c31b5de60..95f2c012a40 100644 --- a/encodings/fastlanes/src/rle/array/rle_decompress.rs +++ b/encodings/fastlanes/src/rle/array/rle_decompress.rs @@ -56,7 +56,7 @@ where let indices = array.indices().clone().execute::(ctx)?; assert!(indices.len().is_multiple_of(FL_CHUNK_SIZE)); - let has_invalid = !indices.all_valid()?; + let has_invalid = !indices.all_valid(ctx)?; let (indices_sl, _) = indices.as_slice::().as_chunks::(); let chunk_start_idx = array.offset() / FL_CHUNK_SIZE; diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 4d54cd9cf32..2776418c231 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -16,10 +16,12 @@ impl OperationsVTable for RLE { fn scalar_at( array: ArrayView<'_, RLE>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let offset_in_chunk = array.offset(); - let chunk_relative_idx = array.indices().scalar_at(offset_in_chunk + index)?; + let chunk_relative_idx = array + .indices() + .execute_scalar(offset_in_chunk + index, ctx)?; let chunk_relative_idx = chunk_relative_idx .as_primitive() @@ -31,7 +33,7 @@ impl OperationsVTable for RLE { let scalar = array .values() - .scalar_at(value_idx_offset + chunk_relative_idx)?; + .execute_scalar(value_idx_offset + chunk_relative_idx, ctx)?; Scalar::try_new(array.dtype().clone(), scalar.into_value()) } @@ -40,7 +42,9 @@ impl OperationsVTable for RLE { #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; @@ -177,7 +181,7 @@ mod tests { if idx < encoded.len() { let original_value = expected[idx]; let encoded_value = encoded - .scalar_at(idx) + .execute_scalar(idx, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -191,14 +195,18 @@ mod tests { #[should_panic] fn test_scalar_at_out_of_bounds() { let array = fixture::rle_array(); - array.scalar_at(1025).unwrap(); + array + .execute_scalar(1025, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); } #[test] #[should_panic] fn test_scalar_at_slice_out_of_bounds() { let array = fixture::rle_array().slice(0..1).unwrap(); - array.scalar_at(1).unwrap(); + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); } #[test] diff --git a/encodings/fsst/benches/chunked_dict_fsst_builder.rs b/encodings/fsst/benches/chunked_dict_fsst_builder.rs index fbc959cfd41..ff1c61f0b80 100644 --- a/encodings/fsst/benches/chunked_dict_fsst_builder.rs +++ b/encodings/fsst/benches/chunked_dict_fsst_builder.rs @@ -5,6 +5,7 @@ use std::sync::LazyLock; use divan::Bencher; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; @@ -68,6 +69,6 @@ fn chunked_dict_fsst_into_canonical( let chunk = make_dict_fsst_chunks::(len, unique_values, chunk_count); bencher - .with_inputs(|| &chunk) - .bench_refs(|chunk| chunk.to_canonical()) + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| (**chunk).clone().execute::(ctx)) } diff --git a/encodings/fsst/benches/fsst_compress.rs b/encodings/fsst/benches/fsst_compress.rs index 7102e3fa98a..6e42f5a39cf 100644 --- a/encodings/fsst/benches/fsst_compress.rs +++ b/encodings/fsst/benches/fsst_compress.rs @@ -9,6 +9,7 @@ use divan::Bencher; use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::RecursiveCanonical; @@ -71,8 +72,8 @@ fn decompress_fsst(bencher: Bencher, (string_count, avg_len, unique_chars): (usi let encoded = fsst_compress(array, len, &dtype, &compressor); bencher - .with_inputs(|| &encoded) - .bench_refs(|encoded| encoded.to_canonical()) + .with_inputs(|| (&encoded, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(encoded, ctx)| (**encoded).clone().into_array().execute::(ctx)) } #[divan::bench(args = BENCH_ARGS)] @@ -128,8 +129,10 @@ fn canonicalize_compare( ) }) .bench_refs(|(fsst_array, constant, ctx)| { - fsst_array - .to_canonical() + (*fsst_array) + .clone() + .into_array() + .execute::(ctx) .unwrap() .into_array() .binary(constant.clone().into_array(), Operator::Eq) @@ -179,8 +182,8 @@ fn chunked_into_canonical( let array = generate_chunked_test_data(chunk_size, string_count, avg_len, unique_chars); bencher - .with_inputs(|| &array) - .bench_refs(|array| array.to_canonical()); + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .bench_refs(|(array, ctx)| (**array).clone().into_array().execute::(ctx)); } /// Helper function to generate random string data. diff --git a/encodings/fsst/benches/fsst_url_compare.rs b/encodings/fsst/benches/fsst_url_compare.rs index 705a8c0cef4..6514d691350 100644 --- a/encodings/fsst/benches/fsst_url_compare.rs +++ b/encodings/fsst/benches/fsst_url_compare.rs @@ -6,6 +6,7 @@ use std::sync::LazyLock; use divan::Bencher; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; @@ -105,8 +106,10 @@ fn eq_canonicalize_high_match(bencher: Bencher) { bencher .with_inputs(|| (&fsst_array, &constant, SESSION.create_execution_ctx())) .bench_refs(|(fsst_array, constant, ctx)| { - fsst_array - .to_canonical() + (*fsst_array) + .clone() + .into_array() + .execute::(ctx) .unwrap() .into_array() .binary(constant.clone().into_array(), Operator::Eq) @@ -127,8 +130,10 @@ fn eq_canonicalize_low_match(bencher: Bencher) { bencher .with_inputs(|| (&fsst_array, &constant, SESSION.create_execution_ctx())) .bench_refs(|(fsst_array, constant, ctx)| { - fsst_array - .to_canonical() + (*fsst_array) + .clone() + .into_array() + .execute::(ctx) .unwrap() .into_array() .binary(constant.clone().into_array(), Operator::Eq) diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 024701dad3b..412fa8302af 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -48,7 +48,7 @@ pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex impl vortex_array::array::vtable::operations::OperationsVTable for vortex_fsst::FSST -pub fn vortex_fsst::FSST::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fsst::FSST>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fsst::FSST::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_fsst::FSST>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fsst::FSST diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index ed605f47818..b510c0412ec 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -23,8 +23,10 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::Precision; use vortex_array::TypedArrayRef; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::varbin::VarBinArrayExt; @@ -580,7 +582,10 @@ impl FSSTData { // Validate that last offset doesn't exceed bytes length (when host-resident). if codes_bytes.is_on_host() && codes_offsets.is_host() && !codes_offsets.is_empty() { let last_offset: usize = (&codes_offsets - .scalar_at(codes_offsets.len() - 1) + .execute_scalar( + codes_offsets.len() - 1, + &mut LEGACY_SESSION.create_execution_ctx(), + ) .vortex_expect("offsets must support scalar_at")) .try_into() .vortex_expect("Failed to convert offset to usize"); diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index 53772e82636..23f7490529a 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -109,6 +109,8 @@ where #[cfg(test)] mod tests { use fsst::CompressorBuilder; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; @@ -133,7 +135,9 @@ mod tests { &compressor, ); - let decoded = compressed.scalar_at(0).unwrap(); + let decoded = compressed + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let expected = Scalar::utf8(big_string, Nullability::NonNullable); diff --git a/encodings/fsst/src/ops.rs b/encodings/fsst/src/ops.rs index 86172377f7e..b630508ed9e 100644 --- a/encodings/fsst/src/ops.rs +++ b/encodings/fsst/src/ops.rs @@ -17,9 +17,9 @@ impl OperationsVTable for FSST { fn scalar_at( array: ArrayView<'_, FSST>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let compressed = array.codes().scalar_at(index)?; + let compressed = array.codes().execute_scalar(index, ctx)?; let binary_datum = compressed.as_binary().value().vortex_expect("non-null"); let decoded_buffer = ByteBuffer::from(array.decompressor().decompress(binary_datum)); diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index d47d4e31267..4743b9fc1a7 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -106,6 +106,8 @@ mod tests { use parquet_variant_compute::VariantArrayBuilder; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -145,8 +147,14 @@ mod tests { let sliced = arr.slice(1..3)?; assert_eq!(sliced.len(), 2); - assert_eq!(sliced.scalar_at(0)?, arr.scalar_at(1)?); - assert_eq!(sliced.scalar_at(1)?, arr.scalar_at(2)?); + assert_eq!( + sliced.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + ); + assert_eq!( + sliced.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + ); Ok(()) } @@ -157,9 +165,21 @@ mod tests { let sliced = arr.slice(0..3)?; assert_eq!(sliced.len(), 3); - assert!(!sliced.scalar_at(0)?.is_null()); - assert!(sliced.scalar_at(1)?.is_null()); - assert!(!sliced.scalar_at(2)?.is_null()); + assert!( + !sliced + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + sliced + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + !sliced + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); Ok(()) } @@ -171,8 +191,14 @@ mod tests { let filtered = arr.filter(mask)?; assert_eq!(filtered.len(), 2); - assert_eq!(filtered.scalar_at(0)?, arr.scalar_at(0)?); - assert_eq!(filtered.scalar_at(1)?, arr.scalar_at(2)?); + assert_eq!( + filtered.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + ); + assert_eq!( + filtered.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + ); Ok(()) } @@ -185,9 +211,21 @@ mod tests { let filtered = arr.filter(mask)?; assert_eq!(filtered.len(), 3); - assert!(!filtered.scalar_at(0)?.is_null()); - assert!(filtered.scalar_at(1)?.is_null()); - assert!(filtered.scalar_at(2)?.is_null()); + assert!( + !filtered + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + filtered + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + filtered + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); Ok(()) } @@ -199,9 +237,18 @@ mod tests { let taken = arr.take(indices.into_array())?; assert_eq!(taken.len(), 3); - assert_eq!(taken.scalar_at(0)?, arr.scalar_at(2)?); - assert_eq!(taken.scalar_at(1)?, arr.scalar_at(0)?); - assert_eq!(taken.scalar_at(2)?, arr.scalar_at(3)?); + assert_eq!( + taken.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + ); + assert_eq!( + taken.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + ); + assert_eq!( + taken.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx())? + ); Ok(()) } @@ -214,10 +261,26 @@ mod tests { let taken = arr.take(indices.into_array())?; assert_eq!(taken.len(), 4); - assert!(!taken.scalar_at(0)?.is_null()); - assert!(taken.scalar_at(1)?.is_null()); - assert!(taken.scalar_at(2)?.is_null()); - assert!(!taken.scalar_at(3)?.is_null()); + assert!( + !taken + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + taken + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + taken + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + !taken + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); Ok(()) } @@ -250,8 +313,14 @@ mod tests { let sliced = arr.slice(1..3)?; assert_eq!(sliced.len(), 2); - assert_eq!(sliced.scalar_at(0)?, arr.scalar_at(1)?); - assert_eq!(sliced.scalar_at(1)?, arr.scalar_at(2)?); + assert_eq!( + sliced.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + ); + assert_eq!( + sliced.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, + arr.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + ); Ok(()) } diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 2efad44ae6e..e059b640912 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -41,7 +41,7 @@ impl OperationsVTable for ParquetVariant { fn scalar_at( array: ArrayView<'_, ParquetVariant>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { if array.validity()?.is_null(index)? { return Ok(Scalar::null(DType::Variant(Nullability::Nullable))); @@ -49,7 +49,7 @@ impl OperationsVTable for ParquetVariant { let metadata = array .metadata_array() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_binary() .value() .cloned() @@ -59,6 +59,7 @@ impl OperationsVTable for ParquetVariant { array.value_array(), array.typed_value_array(), index, + ctx, )?; Scalar::try_new( @@ -73,17 +74,18 @@ fn scalar_from_variant_storage( value: Option<&ArrayRef>, typed_value: Option<&ArrayRef>, index: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { if let Some(typed_value) = typed_value - && typed_value.is_valid(index)? + && typed_value.is_valid(index, ctx)? { - return scalar_from_typed_value_array(metadata, value, typed_value, index); + return scalar_from_typed_value_array(metadata, value, typed_value, index, ctx); } if let Some(value) = value - && value.is_valid(index)? + && value.is_valid(index, ctx)? { - return scalar_from_unshredded_value(metadata, &value.scalar_at(index)?); + return scalar_from_unshredded_value(metadata, &value.execute_scalar(index, ctx)?); } Ok(Scalar::null(DType::Null)) @@ -94,12 +96,17 @@ fn scalar_from_typed_value_array( value: Option<&ArrayRef>, typed_value: &ArrayRef, index: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let value_scalar = match value { - Some(value) if value.is_valid(index)? => Some(value.scalar_at(index)?), + Some(value) if value.is_valid(index, ctx)? => Some(value.execute_scalar(index, ctx)?), _ => None, }; - scalar_from_typed_value_scalar(metadata, value_scalar, typed_value.scalar_at(index)?) + scalar_from_typed_value_scalar( + metadata, + value_scalar, + typed_value.execute_scalar(index, ctx)?, + ) } fn scalar_from_typed_value_scalar( @@ -347,6 +354,7 @@ mod tests { use parquet_variant::VariantBuilder; use parquet_variant_compute::VariantArray as ArrowVariantArray; use parquet_variant_compute::VariantArrayBuilder; + use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Variant; use vortex_array::arrays::variant::VariantArrayExt; @@ -383,7 +391,10 @@ mod tests { Some(ScalarValue::Variant(Box::new(expected_inner))), ) .unwrap(); - assert_eq!(vortex_arr.scalar_at(index)?, expected); + assert_eq!( + vortex_arr.execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx())?, + expected + ); } Ok(()) @@ -412,9 +423,21 @@ mod tests { let variant = vortex_arr.as_opt::().unwrap(); assert!(variant.dtype().is_nullable()); - assert!(vortex_arr.scalar_at(1)?.is_null()); - assert!(!vortex_arr.scalar_at(0)?.is_null()); - assert!(!vortex_arr.scalar_at(2)?.is_null()); + assert!( + vortex_arr + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + !vortex_arr + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + !vortex_arr + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); Ok(()) } @@ -437,13 +460,21 @@ mod tests { let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; assert_eq!(vortex_arr.dtype(), &DType::Variant(Nullability::Nullable)); - assert!(vortex_arr.scalar_at(0)?.is_null()); - assert!(vortex_arr.scalar_at(1)?.is_null()); + assert!( + vortex_arr + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + vortex_arr + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); let variant_view = vortex_arr.as_opt::().unwrap(); let child = variant_view.child(); let inner_pv = child.as_opt::().unwrap(); - let mut ctx = vortex_array::LEGACY_SESSION.create_execution_ctx(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let roundtripped = inner_pv.to_arrow(&mut ctx)?; assert_eq!(roundtripped.inner().null_count(), 2); @@ -463,8 +494,16 @@ mod tests { vortex_arr.dtype(), &DType::Variant(Nullability::NonNullable) ); - assert!(!vortex_arr.scalar_at(0)?.is_null()); - assert!(!vortex_arr.scalar_at(1)?.is_null()); + assert!( + !vortex_arr + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); + assert!( + !vortex_arr + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())? + .is_null() + ); assert_scalar_at_matches_arrow_try_value(&arrow_variant, [0, 1])?; Ok(()) @@ -579,7 +618,7 @@ mod tests { let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; - let row0 = vortex_arr.scalar_at(0)?; + let row0 = vortex_arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; let row0 = row0.as_variant().value().unwrap().as_list(); assert_eq!(row0.len(), 2); assert_eq!( @@ -603,7 +642,7 @@ mod tests { Some(20) ); - let row1 = vortex_arr.scalar_at(1)?; + let row1 = vortex_arr.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?; let row1 = row1.as_variant().value().unwrap().as_list(); assert_eq!(row1.len(), 1); assert_eq!( @@ -675,7 +714,7 @@ mod tests { let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; - let object = vortex_arr.scalar_at(0)?; + let object = vortex_arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; let object = object.as_variant().value().unwrap().as_struct(); assert_eq!( diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 1020dfa0eb1..27fe03ed57b 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -44,7 +44,7 @@ pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vort impl vortex_array::array::vtable::operations::OperationsVTable for vortex_pco::Pco -pub fn vortex_pco::Pco::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_pco::Pco>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_pco::Pco::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_pco::Pco>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_pco::Pco diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 904c6e4dc13..39a67e07ede 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -632,15 +632,14 @@ impl OperationsVTable for Pco { fn scalar_at( array: ArrayView<'_, Pco>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unsliced_validity = child_to_validity(&array.slots()[0], array.dtype().nullability()); array ._slice(index, index + 1) - .decompress(&unsliced_validity, &mut ctx)? + .decompress(&unsliced_validity, ctx)? .into_array() - .scalar_at(0) + .execute_scalar(0, ctx) } } diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index c75e3447a72..1d5d61861a6 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::vtable::child_to_validity; @@ -13,7 +15,11 @@ use crate::Pco; use crate::PcoData; impl CastReduce for Pco { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { - if !dtype.is_nullable() || !array.array().all_valid()? { + if !dtype.is_nullable() + || !array + .array() + .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? + { // TODO(joe): fixme // We cannot cast to non-nullable since the validity containing nulls is used to decode // the PCO array, this would require rewriting tables. diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 163db53f69e..80d22cb75dc 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -70,7 +70,7 @@ pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vo impl vortex_array::array::vtable::operations::OperationsVTable for vortex_runend::RunEnd -pub fn vortex_runend::RunEnd::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_runend::RunEnd>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_runend::RunEnd>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_runend::RunEnd diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 5844f2c9cf4..be353bf6a2d 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -18,8 +18,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::Precision; use vortex_array::TypedArrayRef; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinViewArray; use vortex_array::buffer::BufferHandle; @@ -313,7 +315,9 @@ impl RunEndData { if ends.is_empty() { Ok(0) } else { - usize::try_from(&ends.scalar_at(ends.len() - 1)?) + usize::try_from( + &ends.execute_scalar(ends.len() - 1, &mut LEGACY_SESSION.create_execution_ctx())?, + ) } } @@ -350,20 +354,22 @@ impl RunEndData { return Ok(()); } - debug_assert!({ + #[cfg(debug_assertions)] + { // Run ends must be strictly sorted for binary search to work correctly. let pre_validation = ends.statistics().to_owned(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let is_sorted = ends .statistics() - .compute_is_strict_sorted() + .compute_is_strict_sorted(&mut ctx) .unwrap_or(false); // Preserve the original statistics since compute_is_strict_sorted may have mutated them. // We don't want to run with different stats in debug mode and outside. ends.statistics().inherit(pre_validation.iter()); - is_sorted - }); + debug_assert!(is_sorted); + } // Skip host-only validation when ends are not host-resident. if !ends.is_host() { @@ -372,13 +378,17 @@ impl RunEndData { // Validate the offset and length are valid for the given ends and values if offset != 0 && length != 0 { - let first_run_end = usize::try_from(&ends.scalar_at(0)?)?; + let first_run_end = usize::try_from( + &ends.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + )?; if first_run_end < offset { vortex_bail!("First run end {first_run_end} must be >= offset {offset}"); } } - let last_run_end = usize::try_from(&ends.scalar_at(ends.len() - 1)?)?; + let last_run_end = usize::try_from( + &ends.execute_scalar(ends.len() - 1, &mut LEGACY_SESSION.create_execution_ctx())?, + )?; let min_required_end = offset + length; if last_run_end < min_required_end { vortex_bail!("Last run end {last_run_end} must be >= offset+length {min_required_end}"); @@ -399,6 +409,7 @@ impl RunEndData { /// ``` /// # use vortex_array::arrays::BoolArray; /// # use vortex_array::IntoArray; + /// # use vortex_array::{LEGACY_SESSION, VortexSessionExecute}; /// # use vortex_buffer::buffer; /// # use vortex_error::VortexResult; /// # use vortex_runend::RunEnd; @@ -408,9 +419,10 @@ impl RunEndData { /// let run_end = RunEnd::new(ends, values); /// /// // Array encodes - /// assert_eq!(run_end.scalar_at(0)?, false.into()); - /// assert_eq!(run_end.scalar_at(1)?, false.into()); - /// assert_eq!(run_end.scalar_at(2)?, true.into()); + /// let mut ctx = LEGACY_SESSION.create_execution_ctx(); + /// assert_eq!(run_end.execute_scalar(0, &mut ctx)?, false.into()); + /// assert_eq!(run_end.execute_scalar(1, &mut ctx)?, false.into()); + /// assert_eq!(run_end.execute_scalar(2, &mut ctx)?, true.into()); /// # Ok(()) /// # } /// ``` diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index 1333a8d076a..dfda6b4cce5 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -35,7 +35,9 @@ impl CastReduce for RunEnd { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -71,19 +73,39 @@ mod tests { // RunEnd encoding should expand to [100, 100, 100, 200, 200, 100, 100, 100, 300, 300] assert_eq!(decoded.len(), 10); assert_eq!( - TryInto::::try_into(&decoded.scalar_at(0).unwrap()).unwrap(), + TryInto::::try_into( + &decoded + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 100i64 ); assert_eq!( - TryInto::::try_into(&decoded.scalar_at(3).unwrap()).unwrap(), + TryInto::::try_into( + &decoded + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 200i64 ); assert_eq!( - TryInto::::try_into(&decoded.scalar_at(5).unwrap()).unwrap(), + TryInto::::try_into( + &decoded + .execute_scalar(5, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 100i64 ); assert_eq!( - TryInto::::try_into(&decoded.scalar_at(8).unwrap()).unwrap(), + TryInto::::try_into( + &decoded + .execute_scalar(8, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 300i64 ); } diff --git a/encodings/runend/src/compute/is_constant.rs b/encodings/runend/src/compute/is_constant.rs index 53418f3ef50..d21f2334018 100644 --- a/encodings/runend/src/compute/is_constant.rs +++ b/encodings/runend/src/compute/is_constant.rs @@ -35,6 +35,6 @@ impl DynAggregateKernel for RunEndIsConstantKernel { }; let result = is_constant(array.values(), ctx)?; - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/encodings/runend/src/compute/is_sorted.rs b/encodings/runend/src/compute/is_sorted.rs index e26a3d8cb57..65de2a174f2 100644 --- a/encodings/runend/src/compute/is_sorted.rs +++ b/encodings/runend/src/compute/is_sorted.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray as _; use vortex_array::aggregate_fn::AggregateFnRef; @@ -40,11 +41,23 @@ impl DynAggregateKernel for RunEndIsSortedKernel { let result = if options.strict { // Strict sort with run-end encoding means we need to canonicalize // since run-end encoding repeats values. - is_strict_sorted(&array.array().to_canonical()?.into_array(), ctx)? + is_strict_sorted( + &array + .array() + .clone() + .execute::(ctx)? + .into_array(), + ctx, + )? } else { is_sorted(array.values(), ctx)? }; - Ok(Some(IsSorted::make_partial(batch, result, options.strict)?)) + Ok(Some(IsSorted::make_partial( + batch, + result, + options.strict, + ctx, + )?)) } } diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 432453ee609..9480e9e6f04 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -7,6 +7,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Slice; use vortex_array::arrays::dict::TakeExecuteAdaptor; @@ -57,7 +59,9 @@ fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult for RunEnd { fn scalar_at( array: ArrayView<'_, RunEnd>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - array.values().scalar_at(array.find_physical_index(index)?) + array + .values() + .execute_scalar(array.find_physical_index(index)?, ctx) } } @@ -150,7 +152,7 @@ mod tests { fn ree_scalar_at_end() { let scalar = RunEnd::encode(buffer![1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5].into_array()) .unwrap() - .scalar_at(11) + .execute_scalar(11, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(); assert_eq!(scalar, 5.into()); } diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 413862a6c30..4a85f3a4c0b 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -451,6 +451,8 @@ impl Sequence { #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; @@ -488,7 +490,7 @@ mod tests { fn test_sequence_scalar_at() { let scalar = Sequence::try_new_typed(2i64, 3, Nullability::NonNullable, 4) .unwrap() - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(); assert_eq!( diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index 0ab08cf057f..af92cd1ce53 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -8,6 +8,8 @@ use num_traits::CheckedSub; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; @@ -94,7 +96,10 @@ pub fn sequence_encode( return Ok(None); } - if !primitive_array.array().all_valid()? { + if !primitive_array + .array() + .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? + { return Ok(None); } diff --git a/encodings/sequence/src/compute/is_sorted.rs b/encodings/sequence/src/compute/is_sorted.rs index c930246508f..7df01aded17 100644 --- a/encodings/sequence/src/compute/is_sorted.rs +++ b/encodings/sequence/src/compute/is_sorted.rs @@ -27,7 +27,6 @@ impl DynAggregateKernel for SequenceIsSortedKernel { batch: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let _ = ctx; let Some(options) = aggregate_fn.as_opt::() else { return Ok(None); }; @@ -47,6 +46,11 @@ impl DynAggregateKernel for SequenceIsSortedKernel { }) })?; - Ok(Some(IsSorted::make_partial(batch, result, options.strict)?)) + Ok(Some(IsSorted::make_partial( + batch, + result, + options.strict, + ctx, + )?)) } } diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 04ba70398f6..a182aa4c781 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -8,6 +8,8 @@ use num_traits::NumCast; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; @@ -202,7 +204,7 @@ fn execute_sparse_lists_inner( builder .append_value( patch_values - .scalar_at(patch_idx) + .execute_scalar(patch_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at") .as_list(), ) @@ -298,7 +300,11 @@ fn execute_sparse_fixed_size_list_inner( .vortex_expect("fixed_size_list_elements_at"); for i in 0..list_size as usize { builder - .append_scalar(&patch_list.scalar_at(i).vortex_expect("scalar_at")) + .append_scalar( + &patch_list + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("scalar_at"), + ) .vortex_expect("element dtype must match"); } } else { @@ -557,8 +563,11 @@ mod test { use std::sync::Arc; use rstest::rstest; + use vortex_array::Canonical; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::FixedSizeListArray; @@ -1025,7 +1034,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let result_listview = actual.to_listview(); // Check the structure @@ -1079,7 +1090,10 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical().vortex_expect("no fail").into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("no fail") + .into_array(); let result_listview = actual.to_listview(); // Check the structure @@ -1122,7 +1136,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let result_listview = actual.to_listview(); // Check the structure @@ -1233,7 +1249,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); // Expected: [1,2,3], null, [4,5,6], [7,8,9], null. let expected_elements = @@ -1271,7 +1289,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); // Expected: [1,2], [99,88], [3,4], [99,88], [5,6], [99,88]. let expected_elements = buffer![1i32, 2, 99, 88, 3, 4, 99, 88, 5, 6, 99, 88].into_array(); @@ -1309,7 +1329,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); // Expected validity: [true, true, true, false, true, true]. // Expected elements: [7,8], [10,20], [7,8], [30,40], [50,60], [7,8]. @@ -1357,7 +1379,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); // Build expected: 97 copies of [99,99] with patches at positions 5, 50, 95. let mut expected_elements_vec = Vec::with_capacity(200); @@ -1414,7 +1438,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); // Expected: just [42, 43]. let expected_elements = buffer![42i32, 43].into_array(); @@ -1440,7 +1466,9 @@ mod test { .unwrap() .into_array(); - let actual = sparse.to_canonical()?.into_array(); + let actual = sparse + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let mut expected_elements = buffer_mut![1, 2, 1, 2]; expected_elements.extend(buffer![42i32; 252]); let expected = ListArray::try_new( @@ -1507,7 +1535,10 @@ mod test { .unwrap(); // Convert to canonical form - this triggers the function we're testing - let canonical = sparse.to_canonical()?.into_array(); + let canonical = sparse + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let result_listview = canonical.to_listview(); // Verify the structure @@ -1586,7 +1617,10 @@ mod test { let sparse = Sparse::try_new(indices, values, 5, Scalar::null(sliced.dtype().clone())).unwrap(); - let canonical = sparse.to_canonical()?.into_array(); + let canonical = sparse + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let result_listview = canonical.to_listview(); assert_eq!(result_listview.len(), 5); diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 736f173e894..ed5d0bf8ecc 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -16,6 +16,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -416,7 +417,10 @@ impl SparseData { } else if mask.false_count() as f64 > (0.9 * mask.len() as f64) { // Array is dominated by NULL but has non-NULL values // TODO(joe): use exe ctx? - let non_null_values = array.filter(mask.clone())?.to_canonical()?.into_array(); + let non_null_values = array + .filter(mask.clone())? + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let non_null_indices = match mask.indices() { AllOr::All => { // We already know that the mask is 90%+ false @@ -468,7 +472,7 @@ impl SparseData { let non_top_values = array .filter(non_top_mask.clone())? - .to_canonical()? + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_array(); let indices: Buffer = match non_top_mask { @@ -555,16 +559,33 @@ mod test { pub fn test_scalar_at() { let array = sparse_array(nullable_fill()); - assert_eq!(array.scalar_at(0).unwrap(), nullable_fill()); - assert_eq!(array.scalar_at(2).unwrap(), Scalar::from(Some(100_i32))); - assert_eq!(array.scalar_at(5).unwrap(), Scalar::from(Some(200_i32))); + assert_eq!( + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + nullable_fill() + ); + assert_eq!( + array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(Some(100_i32)) + ); + assert_eq!( + array + .execute_scalar(5, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(Some(200_i32)) + ); } #[test] #[should_panic(expected = "out of bounds")] fn test_scalar_at_oob() { let array = sparse_array(nullable_fill()); - array.scalar_at(10).unwrap(); + array + .execute_scalar(10, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); } #[test] @@ -578,20 +599,36 @@ mod test { .unwrap(); assert_eq!( - arr.scalar_at(10) + arr.execute_scalar(10, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .typed_value::(), Some(1234) ); - assert!(arr.scalar_at(0).unwrap().is_null()); - assert!(arr.scalar_at(99).unwrap().is_null()); + assert!( + arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + arr.execute_scalar(99, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } #[test] pub fn scalar_at_sliced() { let sliced = sparse_array(nullable_fill()).slice(2..7).unwrap(); - assert_eq!(usize::try_from(&sliced.scalar_at(0).unwrap()).unwrap(), 100); + assert_eq!( + usize::try_from( + &sliced + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), + 100 + ); } #[test] @@ -637,13 +674,23 @@ mod test { pub fn scalar_at_sliced_twice() { let sliced_once = sparse_array(nullable_fill()).slice(1..8).unwrap(); assert_eq!( - usize::try_from(&sliced_once.scalar_at(1).unwrap()).unwrap(), + usize::try_from( + &sliced_once + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 100 ); let sliced_twice = sliced_once.slice(1..6).unwrap(); assert_eq!( - usize::try_from(&sliced_twice.scalar_at(3).unwrap()).unwrap(), + usize::try_from( + &sliced_twice + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 200 ); } diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index 299344043ad..cb514c089a8 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -46,7 +46,7 @@ pub fn vortex_zigzag::ZigZag::validate(&self, _data: &Self::ArrayData, dtype: &v impl vortex_array::array::vtable::operations::OperationsVTable for vortex_zigzag::ZigZag -pub fn vortex_zigzag::ZigZag::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_zigzag::ZigZag>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZag::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_zigzag::ZigZag>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityChild for vortex_zigzag::ZigZag diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index bc10192ce78..23e6f66f3eb 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -240,9 +240,9 @@ impl OperationsVTable for ZigZag { fn scalar_at( array: ArrayView<'_, ZigZag>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let scalar = array.encoded().scalar_at(index)?; + let scalar = array.encoded().execute_scalar(index, ctx)?; if scalar.is_null() { return scalar.primitive_reinterpret_cast(ZigZagArrayExt::ptype(&array)); } @@ -270,7 +270,9 @@ impl ValidityChild for ZigZag { #[cfg(test)] mod test { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::scalar::Scalar; use vortex_buffer::buffer; @@ -283,38 +285,42 @@ mod test { .into_array() .to_primitive(); let zigzag = zigzag_encode(array.as_view())?; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_eq!( - zigzag.statistics().compute_max::(), - array.statistics().compute_max::() + zigzag.statistics().compute_max::(&mut ctx), + array.statistics().compute_max::(&mut ctx) ); assert_eq!( - zigzag.statistics().compute_null_count(), - array.statistics().compute_null_count() + zigzag.statistics().compute_null_count(&mut ctx), + array.statistics().compute_null_count(&mut ctx) ); assert_eq!( - zigzag.statistics().compute_is_constant(), - array.statistics().compute_is_constant() + zigzag.statistics().compute_is_constant(&mut ctx), + array.statistics().compute_is_constant(&mut ctx) ); let sliced = zigzag.slice(0..2).unwrap(); let sliced = sliced.as_::(); assert_eq!( - sliced.array().scalar_at(sliced.len() - 1).unwrap(), + sliced + .array() + .execute_scalar(sliced.len() - 1, &mut ctx,) + .unwrap(), Scalar::from(-5i32) ); assert_eq!( - sliced.statistics().compute_min::(), - array.statistics().compute_min::() + sliced.statistics().compute_min::(&mut ctx), + array.statistics().compute_min::(&mut ctx) ); assert_eq!( - sliced.statistics().compute_null_count(), - array.statistics().compute_null_count() + sliced.statistics().compute_null_count(&mut ctx), + array.statistics().compute_null_count(&mut ctx) ); assert_eq!( - sliced.statistics().compute_is_constant(), - array.statistics().compute_is_constant() + sliced.statistics().compute_is_constant(&mut ctx), + array.statistics().compute_is_constant(&mut ctx) ); Ok(()) } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index c038e98d629..6aa5af1f532 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -73,7 +73,9 @@ mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; @@ -94,7 +96,7 @@ mod tests { PrimitiveArray::new(buffer![-189, -160, 1], Validity::AllValid).as_view(), )?; assert_eq!( - zigzag.scalar_at(1)?, + zigzag.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, Scalar::primitive(-160, Nullability::Nullable) ); Ok(()) diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 7a4e13bb8cd..466229e9776 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -52,7 +52,7 @@ pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex impl vortex_array::array::vtable::operations::OperationsVTable for vortex_zstd::Zstd -pub fn vortex_zstd::Zstd::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_zstd::Zstd>, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::scalar_at(array: vortex_array::array::view::ArrayView<'_, vortex_zstd::Zstd>, index: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::array::vtable::validity::ValidityVTable for vortex_zstd::Zstd diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 9f3d19e1bd2..168f437c8cf 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -1029,14 +1029,13 @@ impl OperationsVTable for Zstd { fn scalar_at( array: ArrayView<'_, Zstd>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unsliced_validity = child_to_validity(&array.slots()[0], array.dtype().nullability()); let sliced = array.data().with_slice(index, index + 1); sliced - .decompress(array.dtype(), &unsliced_validity, &mut ctx)? - .scalar_at(0) + .decompress(array.dtype(), &unsliced_validity, ctx)? + .execute_scalar(0, ctx) } } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 662cb434d87..84122db22c7 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -96,7 +96,12 @@ fn test_zstd_with_validity_and_multi_frame() { let slice = compressed.slice(176..179).unwrap(); let primitive = slice.to_primitive(); assert_eq!( - i32::try_from(&primitive.scalar_at(1).unwrap()).unwrap(), + i32::try_from( + &primitive + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 177 ); assert!( diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 5d00fd5a6a4..f59a536655b 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -456,7 +456,7 @@ impl OperationsVTable for ZstdBuffers { fn scalar_at( array: ArrayView<'_, ZstdBuffers>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into @@ -465,7 +465,7 @@ impl OperationsVTable for ZstdBuffers { &array.into_owned(), &vortex_array::LEGACY_SESSION, )?; - inner_array.scalar_at(index) + inner_array.execute_scalar(index, ctx) } } @@ -570,11 +570,18 @@ mod tests { let input = make_nullable_primitive_array(); let compressed = ZstdBuffers::compress(&input, 3, &LEGACY_SESSION)?.into_array(); - assert_eq!(compressed.all_valid()?, input.all_valid()?); - assert_eq!(compressed.all_invalid()?, input.all_invalid()?); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + assert_eq!(compressed.all_valid(&mut ctx)?, input.all_valid(&mut ctx)?); + assert_eq!( + compressed.all_invalid(&mut ctx)?, + input.all_invalid(&mut ctx)? + ); for i in 0..input.len() { - assert_eq!(compressed.is_valid(i)?, input.is_valid(i)?); + assert_eq!( + compressed.is_valid(i, &mut ctx)?, + input.is_valid(i, &mut ctx)? + ); } Ok(()) diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index f6917a0e9f2..8d503c8c590 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -115,12 +115,15 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { .vortex_expect("compare operation should succeed in fuzz test") .to_bool(); let true_count = bool_result.to_bit_buffer().true_count(); + let mut ctx = SESSION.create_execution_ctx(); if true_count != expected_array.len() && (bool_result .into_array() - .all_valid() + .all_valid(&mut ctx) .vortex_expect("all_valid") - || expected_array.all_valid().vortex_expect("all_valid")) + || expected_array + .all_valid(&mut ctx) + .vortex_expect("all_valid")) { vortex_panic!( "Failed to match original array {}with{}", diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index b4c714de335..88784616eda 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -139,8 +139,9 @@ pub fn compare_canonical_array( ) }), DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let scalar_vals: Vec = (0..array.len()) - .map(|i| array.scalar_at(i).vortex_expect("scalar_at")) + .map(|i| array.execute_scalar(i, &mut ctx).vortex_expect("scalar_at")) .collect(); BoolArray::from_iter(scalar_vals.iter().map(|v| { scalar_cmp(v, value, operator) diff --git a/fuzz/src/array/fill_null.rs b/fuzz/src/array/fill_null.rs index 21430ce2b4d..4b4218cd73b 100644 --- a/fuzz/src/array/fill_null.rs +++ b/fuzz/src/array/fill_null.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::DecimalArray; @@ -207,7 +209,7 @@ fn fill_varbinview_array( .map(|i| { if validity_bits.value(i) { array_ref - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at") .as_utf8() .value() @@ -241,7 +243,7 @@ fn fill_varbinview_array( .map(|i| { if validity_bits.value(i) { array_ref - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at") .as_binary() .value() @@ -274,7 +276,10 @@ fn fill_varbinview_array( #[cfg(test)] mod tests { + use vortex_array::Canonical; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; @@ -292,12 +297,19 @@ mod tests { use super::fill_null_canonical_array; + fn canonical(array: impl IntoArray) -> Canonical { + array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + } + #[test] fn test_fill_null_primitive() { let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)]); let fill_value = Scalar::from(42i32); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = PrimitiveArray::from_iter([1i32, 42, 3, 42, 5]); assert_arrays_eq!(expected, result); @@ -310,7 +322,7 @@ mod tests { let array = BoolArray::new(data_buffer, Validity::from(validity_buffer)); let fill_value = Scalar::from(true); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = BoolArray::from(BitBuffer::from(vec![true, true, false, true])); assert_arrays_eq!(expected, result); @@ -324,7 +336,7 @@ mod tests { ); let fill_value = Scalar::utf8("default", Nullability::NonNullable); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = VarBinViewArray::from_iter_str(["hello", "default", "world"]); assert_arrays_eq!(expected, result); @@ -335,7 +347,7 @@ mod tests { let array = PrimitiveArray::from_option_iter([None::, None, None]); let fill_value = Scalar::from(100i32); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = PrimitiveArray::from_iter([100i32, 100, 100]); assert_arrays_eq!(expected, result); @@ -346,7 +358,7 @@ mod tests { let array = PrimitiveArray::from_iter([1i32, 2, 3]); let fill_value = Scalar::from(42i32); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = PrimitiveArray::from_iter([1i32, 2, 3]); assert_arrays_eq!(expected, result); @@ -358,7 +370,7 @@ mod tests { let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]); let fill_value = Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value); + let result = fill_null_canonical_array(canonical(array), &fill_value); assert!(result.is_err()); assert!( @@ -381,7 +393,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = DecimalArray::from_iter( [100i32, 999i32, 300i32, 999i32, 500i32], @@ -402,7 +414,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = DecimalArray::from_iter([1000i64, 9999i64, 3000i64], DecimalDType::new(15, 3)); @@ -421,7 +433,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = DecimalArray::from_iter( [10000i128, 99999i128, 30000i128, 99999i128], @@ -440,7 +452,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = DecimalArray::from_option_iter( [Some(777i64), Some(777i64), Some(777i64)], @@ -464,7 +476,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(array.to_canonical().unwrap(), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); let expected = DecimalArray::from_option_iter( [Some(100i32), Some(200i32), Some(300i32)], diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 4c97200fa2d..8f1aba4ed16 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::ExtensionArray; @@ -133,8 +135,14 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { // Recursively mask the storage array - let masked_storage = mask_canonical_array(array.storage_array().to_canonical()?, mask) - .vortex_expect("mask_canonical_array should succeed in fuzz test"); + let masked_storage = mask_canonical_array( + array + .storage_array() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?, + mask, + ) + .vortex_expect("mask_canonical_array should succeed in fuzz test"); let ext_dtype = array .ext_dtype() @@ -147,7 +155,10 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult Canonical { + array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + } + #[test] fn test_mask_null_array() { let array = NullArray::new(5); let mask = Mask::from_iter([true, false, true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); assert_eq!(result.len(), 5); // All values should still be null for i in 0..5 { - assert!(!result.is_valid(i).unwrap()); + assert!( + !result + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } } @@ -183,7 +205,7 @@ mod tests { let array = BoolArray::from_iter([true, false, true, false, true]); let mask = Mask::from_iter([false, true, true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = BoolArray::from_iter([None, Some(false), Some(true), None, Some(true)]); assert_arrays_eq!(result, expected); @@ -194,7 +216,7 @@ mod tests { let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::from_iter([true, false, true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)]); assert_arrays_eq!(result, expected); @@ -205,7 +227,7 @@ mod tests { let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4), None]); let mask = Mask::from_iter([false, true, true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = PrimitiveArray::from_option_iter([None, None, Some(3i32), None, None]); assert_arrays_eq!(result, expected); @@ -220,7 +242,7 @@ mod tests { ); let mask = Mask::from_iter([true, true, false, true, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = DecimalArray::from_option_iter([Some(1i128), Some(2), None, Some(4), Some(5)], dtype); @@ -232,7 +254,7 @@ mod tests { let array = VarBinViewArray::from_iter_str(["one", "two", "three", "four", "five"]); let mask = Mask::from_iter([false, true, false, true, false]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = VarBinViewArray::from_iter_nullable_str([None, Some("two"), None, Some("four"), None]); @@ -251,12 +273,24 @@ mod tests { let mask = Mask::from_iter([true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); assert_eq!(result.len(), 3); - assert!(result.is_valid(0).unwrap()); - assert!(!result.is_valid(1).unwrap()); - assert!(result.is_valid(2).unwrap()); + assert!( + result + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !result + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] @@ -267,12 +301,24 @@ mod tests { let mask = Mask::from_iter([false, true, false]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); assert_eq!(result.len(), 3); - assert!(!result.is_valid(0).unwrap()); - assert!(result.is_valid(1).unwrap()); - assert!(!result.is_valid(2).unwrap()); + assert!( + !result + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !result + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] @@ -291,12 +337,24 @@ mod tests { let mask = Mask::from_iter([true, false, true]); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); assert_eq!(result.len(), 3); - assert!(result.is_valid(0).unwrap()); - assert!(!result.is_valid(1).unwrap()); - assert!(result.is_valid(2).unwrap()); + assert!( + result + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !result + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] @@ -304,7 +362,7 @@ mod tests { let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::AllFalse(5); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = PrimitiveArray::from_option_iter([None, None, None, None, None::]); assert_arrays_eq!(result, expected); @@ -315,7 +373,7 @@ mod tests { let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::AllTrue(5); - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask).unwrap(); let expected = PrimitiveArray::from_option_iter([Some(1i32), Some(2), Some(3), Some(4), Some(5)]); @@ -326,7 +384,7 @@ mod tests { fn test_mask_empty_array() { let array = PrimitiveArray::from_iter(Vec::::new()); for mask in [Mask::AllFalse(0), Mask::AllTrue(0)] { - let result = mask_canonical_array(array.to_canonical().unwrap(), &mask).unwrap(); + let result = mask_canonical_array(canonical(array.clone()), &mask).unwrap(); assert_eq!(result.len(), 0); } } diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index 4576eb7b167..2c72dbcc7e5 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -250,7 +250,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { let scalar = if u.arbitrary()? { current_array - .scalar_at(u.choose_index(current_array.len())?) + .execute_scalar(u.choose_index(current_array.len())?, &mut ctx) .vortex_expect("scalar_at") } else { random_scalar(u, current_array.dtype())? @@ -291,7 +291,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { ActionType::Compare => { let scalar = if u.arbitrary()? { current_array - .scalar_at(u.choose_index(current_array.len())?) + .execute_scalar(u.choose_index(current_array.len())?, &mut ctx) .vortex_expect("scalar_at") } else { // We can compare arrays with different nullability @@ -354,7 +354,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { } let fill_value = if u.arbitrary()? && !current_array.is_empty() { current_array - .scalar_at(u.choose_index(current_array.len())?) + .execute_scalar(u.choose_index(current_array.len())?, &mut ctx) .vortex_expect("scalar_at") } else { random_scalar( @@ -669,7 +669,9 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { Action::ScalarAt(indices) => { let expected_scalars = expected.scalar_vec(); for (j, &idx) in indices.iter().enumerate() { - let scalar = current_array.scalar_at(idx).vortex_expect("scalar_at"); + let scalar = current_array + .execute_scalar(idx, &mut ctx) + .vortex_expect("scalar_at"); assert_scalar_eq(&expected_scalars[j], &scalar, i)?; } } @@ -726,9 +728,10 @@ pub fn assert_array_eq(lhs: &ArrayRef, rhs: &ArrayRef, step: usize) -> VortexFuz Backtrace::capture(), )); } + let mut ctx = SESSION.create_execution_ctx(); for idx in 0..lhs.len() { - let l = lhs.scalar_at(idx).vortex_expect("scalar_at"); - let r = rhs.scalar_at(idx).vortex_expect("scalar_at"); + let l = lhs.execute_scalar(idx, &mut ctx).vortex_expect("scalar_at"); + let r = rhs.execute_scalar(idx, &mut ctx).vortex_expect("scalar_at"); if l != r { return Err(VortexFuzzError::ArrayNotEqual( diff --git a/fuzz/src/array/scalar_at.rs b/fuzz/src/array/scalar_at.rs index 91898390df6..489860c7f59 100644 --- a/fuzz/src/array/scalar_at.rs +++ b/fuzz/src/array/scalar_at.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -24,7 +26,7 @@ use vortex_error::VortexResult; /// without using the scalar_at method, to serve as an independent baseline for testing. pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexResult { let canonical_ref = canonical.clone().into_array(); - if canonical_ref.is_invalid(index)? { + if canonical_ref.is_invalid(index, &mut LEGACY_SESSION.create_execution_ctx())? { return Ok(Scalar::null(canonical_ref.dtype().clone())); } Ok(match canonical { diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index 593be73c598..d7b9901cc90 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -149,8 +149,9 @@ pub fn search_sorted_canonical_array( SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side) } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let scalar_vals = (0..array.len()) - .map(|i| array.scalar_at(i)) + .map(|i| array.execute_scalar(i, &mut ctx)) .collect::>>()?; scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side) } diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index 096e5bbdc06..12fb2c9ab0c 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -104,11 +104,15 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { let mut sort_indices = (0..array.len()).collect::>(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); sort_indices.sort_by(|a, b| { - array - .scalar_at(*a) - .vortex_expect("scalar_at") - .partial_cmp(&array.scalar_at(*b).vortex_expect("scalar_at")) + let lhs = array + .execute_scalar(*a, &mut ctx) + .vortex_expect("scalar_at"); + let rhs = array + .execute_scalar(*b, &mut ctx) + .vortex_expect("scalar_at"); + lhs.partial_cmp(&rhs) .vortex_expect("must be a valid comparison") }); take_canonical_array_non_nullable_indices(array, &sort_indices) diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index 01c7eae4a39..e737b33d0cc 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -129,11 +129,12 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort &array.dtype().union_nullability(nullable), indices_slice_non_opt.len(), ); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); for idx in indices { if let Some(idx) = idx { builder.append_scalar( &array - .scalar_at(*idx)? + .execute_scalar(*idx, &mut ctx)? .cast(&array.dtype().union_nullability(nullable)) .vortex_expect("cannot cast scalar nullability"), )?; diff --git a/fuzz/src/gpu/mod.rs b/fuzz/src/gpu/mod.rs index 57b233bb686..fa6ef110759 100644 --- a/fuzz/src/gpu/mod.rs +++ b/fuzz/src/gpu/mod.rs @@ -10,7 +10,10 @@ use arbitrary::Arbitrary; use arbitrary::Result; use arbitrary::Unstructured; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::dict::ArbitraryDictArray; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; @@ -110,7 +113,10 @@ pub async fn run_compress_gpu(fuzz: FuzzCompressGpu) -> VortexFuzzResult { let original_len = array.len(); - let cpu_canonical = match array.to_canonical() { + let cpu_canonical = match array + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + { Ok(c) => c, Err(e) => { return Err(VortexFuzzError::VortexError(e, Backtrace::capture())); @@ -159,10 +165,10 @@ pub async fn run_compress_gpu(fuzz: FuzzCompressGpu) -> VortexFuzzResult { for i in 0..original_len { let cpu_scalar = cpu_array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .map_err(|e| VortexFuzzError::VortexError(e, Backtrace::capture()))?; let gpu_scalar = gpu_array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .map_err(|e| VortexFuzzError::VortexError(e, Backtrace::capture()))?; if cpu_scalar != gpu_scalar { diff --git a/vortex-array/benches/cast_primitive.rs b/vortex-array/benches/cast_primitive.rs index be200ef137d..86895fb2ce7 100644 --- a/vortex-array/benches/cast_primitive.rs +++ b/vortex-array/benches/cast_primitive.rs @@ -3,7 +3,10 @@ use divan::Bencher; use rand::prelude::*; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; @@ -30,11 +33,16 @@ fn cast_u16_to_u32(bencher: Bencher) { })) .into_array(); // Pre-compute min/max so values_fit_in is a cache hit during the benchmark. - arr.statistics().compute_all(&[Stat::Min, Stat::Max]).ok(); + arr.statistics() + .compute_all( + &[Stat::Min, Stat::Max], + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .ok(); bencher.with_inputs(|| arr.clone()).bench_refs(|a| { #[expect(clippy::unwrap_used)] a.cast(DType::Primitive(PType::U32, Nullability::Nullable)) .unwrap() - .to_canonical() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) }); } diff --git a/vortex-array/benches/chunk_array_builder.rs b/vortex-array/benches/chunk_array_builder.rs index 7d1e360c7ce..d74fd309453 100644 --- a/vortex-array/benches/chunk_array_builder.rs +++ b/vortex-array/benches/chunk_array_builder.rs @@ -8,6 +8,7 @@ use rand::RngExt; use rand::SeedableRng; use rand::prelude::StdRng; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -70,8 +71,8 @@ fn chunked_opt_bool_into_canonical(bencher: Bencher, (len, chunk_count): (usize, let chunk = make_opt_bool_chunks(len, chunk_count); bencher - .with_inputs(|| &chunk) - .bench_refs(|chunk| chunk.to_canonical()) + .with_inputs(|| (&chunk, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| (**chunk).clone().execute::(ctx)) } #[divan::bench(args = BENCH_ARGS)] @@ -97,8 +98,8 @@ fn chunked_varbinview_into_canonical(bencher: Bencher, (len, chunk_count): (usiz let chunks = make_string_chunks(false, len, chunk_count); bencher - .with_inputs(|| &chunks) - .bench_refs(|chunk| chunk.to_canonical()) + .with_inputs(|| (&chunks, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| (**chunk).clone().execute::(ctx)) } #[divan::bench(args = BENCH_ARGS)] @@ -124,8 +125,8 @@ fn chunked_varbinview_opt_into_canonical(bencher: Bencher, (len, chunk_count): ( let chunks = make_string_chunks(true, len, chunk_count); bencher - .with_inputs(|| &chunks) - .bench_refs(|chunk| chunk.to_canonical()) + .with_inputs(|| (&chunks, SESSION.create_execution_ctx())) + .bench_refs(|(chunk, ctx)| (**chunk).clone().execute::(ctx)) } #[divan::bench(args = BENCH_ARGS)] diff --git a/vortex-array/benches/dict_compress.rs b/vortex-array/benches/dict_compress.rs index 401d250f813..4f220ccaa23 100644 --- a/vortex-array/benches/dict_compress.rs +++ b/vortex-array/benches/dict_compress.rs @@ -6,7 +6,10 @@ use divan::Bencher; use rand::distr::Distribution; use rand::distr::StandardUniform; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::dict_test::gen_primitive_for_dict; @@ -75,8 +78,8 @@ where .into_array(); bencher - .with_inputs(|| &dict) - .bench_refs(|dict| dict.to_canonical()); + .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(dict, ctx)| (**dict).clone().execute::(ctx)); } #[divan::bench(args = BENCH_ARGS)] @@ -85,8 +88,8 @@ fn decode_varbin(bencher: Bencher, (len, unique_values): (usize, usize)) { let dict = dict_encode(&varbin_arr.into_array()).unwrap().into_array(); bencher - .with_inputs(|| &dict) - .bench_refs(|dict| dict.to_canonical()); + .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(dict, ctx)| (**dict).clone().execute::(ctx)); } #[divan::bench(args = BENCH_ARGS)] @@ -97,6 +100,6 @@ fn decode_varbinview(bencher: Bencher, (len, unique_values): (usize, usize)) { .into_array(); bencher - .with_inputs(|| &dict) - .bench_refs(|dict| dict.to_canonical()); + .with_inputs(|| (&dict, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(dict, ctx)| (**dict).clone().execute::(ctx)); } diff --git a/vortex-array/benches/listview_rebuild.rs b/vortex-array/benches/listview_rebuild.rs index 5ce7fea1128..00781b1a534 100644 --- a/vortex-array/benches/listview_rebuild.rs +++ b/vortex-array/benches/listview_rebuild.rs @@ -7,7 +7,10 @@ #![expect(clippy::cast_possible_truncation)] use divan::Bencher; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::ListViewArray; @@ -109,7 +112,11 @@ fn i32_small(bencher: Bencher) { let lv = make_primitive_lv(50, 32, 32); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -118,7 +125,11 @@ fn i32_small_overlapping(bencher: Bencher) { let lv = make_primitive_lv(50, 8, 1); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -127,7 +138,11 @@ fn varbinview_small(bencher: Bencher) { let lv = make_varbinview_lv(50, 32, 32); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -136,7 +151,11 @@ fn struct_small(bencher: Bencher) { let lv = make_struct_lv(50, 32, 32); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -145,7 +164,11 @@ fn i32_large(bencher: Bencher) { let lv = make_primitive_lv(50, 1_024, 1_024); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -154,7 +177,11 @@ fn varbinview_large(bencher: Bencher) { let lv = make_varbinview_lv(5, 1_024, 1_024); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -163,7 +190,11 @@ fn struct_large(bencher: Bencher) { let lv = make_struct_lv(25, 1_024, 1_024); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -185,7 +216,11 @@ fn fsl_large(bencher: Bencher) { ); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } @@ -194,6 +229,10 @@ fn list_i32_large(bencher: Bencher) { let lv = make_nested_list_lv(2, 512, 2); bencher.with_inputs(|| &lv).bench_refs(|lv| { let rebuilt = lv.rebuild(ListViewRebuildMode::MakeZeroCopyToList).unwrap(); - rebuilt.elements().to_canonical().unwrap() + rebuilt + .elements() + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() }); } diff --git a/vortex-array/benches/scalar_at_struct.rs b/vortex-array/benches/scalar_at_struct.rs index 7563a149f8c..757d6252bb9 100644 --- a/vortex-array/benches/scalar_at_struct.rs +++ b/vortex-array/benches/scalar_at_struct.rs @@ -10,6 +10,8 @@ use rand::distr::Uniform; use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; use vortex_array::dtype::FieldNames; use vortex_array::validity::Validity; @@ -23,7 +25,7 @@ const ARRAY_SIZE: usize = 100_000; const NUM_ACCESSES: usize = 1000; #[divan::bench] -fn scalar_at_struct_simple(bencher: Bencher) { +fn execute_scalar_struct_simple(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let range = Uniform::new(0i64, 100_000_000).unwrap(); @@ -49,14 +51,15 @@ fn scalar_at_struct_simple(bencher: Bencher) { bencher .with_inputs(|| (&struct_array, &indices)) .bench_refs(|(array, indices)| { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); for &idx in indices.iter() { - divan::black_box(array.scalar_at(idx).unwrap()); + divan::black_box(array.execute_scalar(idx, &mut ctx).unwrap()); } }); } #[divan::bench] -fn scalar_at_struct_wide(bencher: Bencher) { +fn execute_scalar_struct_wide(bencher: Bencher) { let mut rng = StdRng::seed_from_u64(0); let range = Uniform::new(0i64, 100_000_000).unwrap(); @@ -86,8 +89,9 @@ fn scalar_at_struct_wide(bencher: Bencher) { bencher .with_inputs(|| (&struct_array, &indices)) .bench_refs(|(array, indices)| { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); for &idx in indices.iter() { - divan::black_box(array.scalar_at(idx).unwrap()); + divan::black_box(array.execute_scalar(idx, &mut ctx).unwrap()); } }); } diff --git a/vortex-array/benches/take_primitive.rs b/vortex-array/benches/take_primitive.rs index a290aaa6fa1..4411cf1ac79 100644 --- a/vortex-array/benches/take_primitive.rs +++ b/vortex-array/benches/take_primitive.rs @@ -12,7 +12,10 @@ use divan::Bencher; use rand::distr::Uniform; use rand::prelude::*; use rand_distr::Zipf; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; @@ -38,9 +41,12 @@ fn dict_canonicalize_uniform(bencher: Bencher, num_indi let dict = DictArray::try_new(codes.into_array(), values.into_array()).unwrap(); - bencher - .with_inputs(|| &dict) - .bench_refs(|dict| dict.to_canonical()); + bencher.with_inputs(|| &dict).bench_refs(|dict| { + (*dict) + .clone() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + }); } #[divan::bench(args = NUM_INDICES, consts = VECTOR_SIZE, sample_count = 100_000)] @@ -57,7 +63,10 @@ fn dict_canonicalize_zipfian(bencher: Bencher, num_indi let dict = DictArray::try_new(codes.into_array(), values.into_array()).unwrap(); - bencher - .with_inputs(|| &dict) - .bench_refs(|dict| dict.to_canonical()); + bencher.with_inputs(|| &dict).bench_refs(|dict| { + (*dict) + .clone() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + }); } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 243c59cafa0..f5b0adb4f2c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -144,7 +144,7 @@ pub struct vortex_array::aggregate_fn::fns::is_constant::IsConstant impl vortex_array::aggregate_fn::fns::is_constant::IsConstant -pub fn vortex_array::aggregate_fn::fns::is_constant::IsConstant::make_partial(batch: &vortex_array::ArrayRef, is_constant: bool) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::is_constant::IsConstant::make_partial(batch: &vortex_array::ArrayRef, is_constant: bool, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_array::aggregate_fn::fns::is_constant::IsConstant @@ -202,7 +202,7 @@ pub struct vortex_array::aggregate_fn::fns::is_sorted::IsSorted impl vortex_array::aggregate_fn::fns::is_sorted::IsSorted -pub fn vortex_array::aggregate_fn::fns::is_sorted::IsSorted::make_partial(batch: &vortex_array::ArrayRef, is_sorted: bool, strict: bool) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::is_sorted::IsSorted::make_partial(batch: &vortex_array::ArrayRef, is_sorted: bool, strict: bool, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_array::aggregate_fn::fns::is_sorted::IsSorted @@ -1266,13 +1266,13 @@ pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, pub trait vortex_array::arrays::bool::BoolArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub fn vortex_array::arrays::bool::BoolArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::bool::BoolArrayExt::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask @@ -1280,13 +1280,13 @@ pub fn vortex_array::arrays::bool::BoolArrayExt::validity(&self) -> vortex_array impl> vortex_array::arrays::bool::BoolArrayExt for T -pub fn T::maybe_to_mask(&self) -> vortex_error::VortexResult> +pub fn T::maybe_to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer -pub fn T::to_mask(&self) -> vortex_mask::Mask +pub fn T::to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn T::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask @@ -1308,7 +1308,7 @@ pub fn vortex_array::arrays::Chunked::fmt(&self, f: &mut core::fmt::Formatter<'_ impl vortex_array::OperationsVTable for vortex_array::arrays::Chunked -pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -1868,7 +1868,7 @@ pub fn vortex_array::arrays::dict::Dict::fmt(&self, f: &mut core::fmt::Formatter impl vortex_array::OperationsVTable for vortex_array::arrays::dict::Dict -pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -1960,7 +1960,7 @@ pub fn vortex_array::arrays::dict::Dict::fmt(&self, f: &mut core::fmt::Formatter impl vortex_array::OperationsVTable for vortex_array::arrays::dict::Dict -pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -2294,7 +2294,7 @@ pub fn vortex_array::arrays::Extension::fmt(&self, f: &mut core::fmt::Formatter< impl vortex_array::OperationsVTable for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Extension @@ -2420,7 +2420,7 @@ pub fn vortex_array::arrays::Filter::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Filter -pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Filter @@ -2608,7 +2608,7 @@ pub fn vortex_array::arrays::FixedSizeList::fmt(&self, f: &mut core::fmt::Format impl vortex_array::OperationsVTable for vortex_array::arrays::FixedSizeList -pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -2748,7 +2748,7 @@ pub fn vortex_array::arrays::List::fmt(&self, f: &mut core::fmt::Formatter<'_>) impl vortex_array::OperationsVTable for vortex_array::arrays::List -pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::List @@ -2924,7 +2924,7 @@ pub fn vortex_array::arrays::ListView::fmt(&self, f: &mut core::fmt::Formatter<' impl vortex_array::OperationsVTable for vortex_array::arrays::ListView -pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::ListView @@ -3100,7 +3100,7 @@ pub fn vortex_array::arrays::Masked::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Masked -pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Masked @@ -3884,7 +3884,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt(&self, f: &mut core: impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3994,7 +3994,7 @@ pub fn vortex_array::arrays::Shared::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Shared -pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Shared @@ -4106,7 +4106,7 @@ pub fn vortex_array::arrays::slice::Slice::fmt(&self, f: &mut core::fmt::Formatt impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice -pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -4332,7 +4332,7 @@ pub fn vortex_array::arrays::Struct::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Struct @@ -5036,7 +5036,7 @@ pub fn vortex_array::arrays::Variant::fmt(&self, f: &mut core::fmt::Formatter<'_ impl vortex_array::OperationsVTable for vortex_array::arrays::Variant -pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Variant @@ -5192,7 +5192,7 @@ pub fn vortex_array::arrays::Chunked::fmt(&self, f: &mut core::fmt::Formatter<'_ impl vortex_array::OperationsVTable for vortex_array::arrays::Chunked -pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Chunked @@ -5456,7 +5456,7 @@ pub fn vortex_array::arrays::dict::Dict::fmt(&self, f: &mut core::fmt::Formatter impl vortex_array::OperationsVTable for vortex_array::arrays::dict::Dict -pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::dict::Dict @@ -5546,7 +5546,7 @@ pub fn vortex_array::arrays::Extension::fmt(&self, f: &mut core::fmt::Formatter< impl vortex_array::OperationsVTable for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Extension @@ -5628,7 +5628,7 @@ pub fn vortex_array::arrays::Filter::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Filter -pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Filter @@ -5686,7 +5686,7 @@ pub fn vortex_array::arrays::FixedSizeList::fmt(&self, f: &mut core::fmt::Format impl vortex_array::OperationsVTable for vortex_array::arrays::FixedSizeList -pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::FixedSizeList @@ -5760,7 +5760,7 @@ pub fn vortex_array::arrays::List::fmt(&self, f: &mut core::fmt::Formatter<'_>) impl vortex_array::OperationsVTable for vortex_array::arrays::List -pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::List @@ -5838,7 +5838,7 @@ pub fn vortex_array::arrays::ListView::fmt(&self, f: &mut core::fmt::Formatter<' impl vortex_array::OperationsVTable for vortex_array::arrays::ListView -pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::ListView @@ -5912,7 +5912,7 @@ pub fn vortex_array::arrays::Masked::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Masked -pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Masked @@ -6234,7 +6234,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt(&self, f: &mut core: impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6292,7 +6292,7 @@ pub fn vortex_array::arrays::Shared::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Shared -pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Shared @@ -6350,7 +6350,7 @@ pub fn vortex_array::arrays::slice::Slice::fmt(&self, f: &mut core::fmt::Formatt impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice -pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -6412,7 +6412,7 @@ pub fn vortex_array::arrays::Struct::fmt(&self, f: &mut core::fmt::Formatter<'_> impl vortex_array::OperationsVTable for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Struct @@ -6654,7 +6654,7 @@ pub fn vortex_array::arrays::Variant::fmt(&self, f: &mut core::fmt::Formatter<'_ impl vortex_array::OperationsVTable for vortex_array::arrays::Variant -pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::VTable for vortex_array::arrays::Variant @@ -18874,29 +18874,29 @@ impl vortex_array::stats::StatsSetRef<'_> pub fn vortex_array::stats::StatsSetRef<'_>::clear(&self, stat: vortex_array::expr::stats::Stat) -pub fn vortex_array::stats::StatsSetRef<'_>::compute_as core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self, stat: vortex_array::expr::stats::Stat) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_as core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self, stat: vortex_array::expr::stats::Stat, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_constant(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_constant(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_sorted(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_sorted(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_strict_sorted(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_is_strict_sorted(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_max core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_max core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_min core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_min core::convert::TryFrom<&'a vortex_array::scalar::Scalar, Error = vortex_error::VortexError>>(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_null_count(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_null_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option -pub fn vortex_array::stats::StatsSetRef<'_>::compute_uncompressed_size_in_bytes(&self) -> core::option::Option +pub fn vortex_array::stats::StatsSetRef<'_>::compute_uncompressed_size_in_bytes(&self, ctx: &mut vortex_array::ExecutionCtx) -> core::option::Option pub fn vortex_array::stats::StatsSetRef<'_>::set(&self, stat: vortex_array::expr::stats::Stat, value: vortex_array::expr::stats::Precision) impl vortex_array::stats::StatsSetRef<'_> -pub fn vortex_array::stats::StatsSetRef<'_>::compute_all(&self, stats: &[vortex_array::expr::stats::Stat]) -> vortex_error::VortexResult +pub fn vortex_array::stats::StatsSetRef<'_>::compute_all(&self, stats: &[vortex_array::expr::stats::Stat], ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::stats::StatsSetRef<'_>::compute_stat(&self, stat: vortex_array::expr::stats::Stat) -> vortex_error::VortexResult> +pub fn vortex_array::stats::StatsSetRef<'_>::compute_stat(&self, stat: vortex_array::expr::stats::Stat, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub fn vortex_array::stats::StatsSetRef<'_>::inherit<'a>(&self, iter: impl core::iter::traits::iterator::Iterator)>) @@ -20140,7 +20140,7 @@ pub fn vortex_array::arrays::Bool::scalar_at(array: vortex_array::ArrayView<'_, impl vortex_array::OperationsVTable for vortex_array::arrays::Chunked -pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Constant @@ -20152,27 +20152,27 @@ pub fn vortex_array::arrays::Decimal::scalar_at(array: vortex_array::ArrayView<' impl vortex_array::OperationsVTable for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Filter -pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::FixedSizeList -pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::List -pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::ListView -pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Masked -pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Primitive @@ -20180,11 +20180,11 @@ pub fn vortex_array::arrays::Primitive::scalar_at(array: vortex_array::ArrayView impl vortex_array::OperationsVTable for vortex_array::arrays::Shared -pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::VarBin @@ -20196,11 +20196,11 @@ pub fn vortex_array::arrays::VarBinView::scalar_at(array: vortex_array::ArrayVie impl vortex_array::OperationsVTable for vortex_array::arrays::Variant -pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::dict::Dict -pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::null::Null @@ -20212,11 +20212,11 @@ pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::Ar impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice -pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::NotSupported @@ -21746,21 +21746,23 @@ pub fn vortex_array::Array::try_new(child: v impl vortex_array::Array -pub fn vortex_array::Array::all_invalid(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::all_invalid(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::Array::all_valid(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::all_valid(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::append_to_builder(&self, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::Array::as_constant(&self) -> core::option::Option +pub fn vortex_array::Array::execute_scalar(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_array::Array::filter(&self, mask: vortex_mask::Mask) -> vortex_error::VortexResult pub fn vortex_array::Array::invalid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::Array::is_invalid(&self, index: usize) -> vortex_error::VortexResult +pub fn vortex_array::Array::is_invalid(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::Array::is_valid(&self, index: usize) -> vortex_error::VortexResult +pub fn vortex_array::Array::is_valid(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::nbuffers(&self) -> usize @@ -21934,9 +21936,9 @@ pub struct vortex_array::ArrayRef(_) impl vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::all_invalid(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::all_invalid(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::all_valid(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::all_valid(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::append_to_builder(&self, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> @@ -21968,6 +21970,8 @@ pub fn vortex_array::ArrayRef::encoding_id(&self) -> vortex_array::ArrayId pub fn vortex_array::ArrayRef::execute_parent(&self, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::ArrayRef::execute_scalar(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_array::ArrayRef::filter(&self, mask: vortex_mask::Mask) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::into_canonical(self) -> vortex_error::VortexResult @@ -21984,9 +21988,9 @@ pub fn vortex_array::ArrayRef::is_empty(&self) -> bool pub fn vortex_array::ArrayRef::is_host(&self) -> bool -pub fn vortex_array::ArrayRef::is_invalid(&self, index: usize) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::is_invalid(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::is_valid(&self, index: usize) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::is_valid(&self, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::len(&self) -> usize @@ -23988,7 +23992,7 @@ pub fn vortex_array::arrays::Bool::scalar_at(array: vortex_array::ArrayView<'_, impl vortex_array::OperationsVTable for vortex_array::arrays::Chunked -pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Chunked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Constant @@ -24000,27 +24004,27 @@ pub fn vortex_array::arrays::Decimal::scalar_at(array: vortex_array::ArrayView<' impl vortex_array::OperationsVTable for vortex_array::arrays::Extension -pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Filter -pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::FixedSizeList -pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::FixedSizeList>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::List -pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::ListView -pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Masked -pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Masked>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Primitive @@ -24028,11 +24032,11 @@ pub fn vortex_array::arrays::Primitive::scalar_at(array: vortex_array::ArrayView impl vortex_array::OperationsVTable for vortex_array::arrays::Shared -pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Shared>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::Struct -pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::VarBin @@ -24044,11 +24048,11 @@ pub fn vortex_array::arrays::VarBinView::scalar_at(array: vortex_array::ArrayVie impl vortex_array::OperationsVTable for vortex_array::arrays::Variant -pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::Variant>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::dict::Dict -pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::null::Null @@ -24060,11 +24064,11 @@ pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::Ar impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice -pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::slice::Slice>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::NotSupported diff --git a/vortex-array/src/aggregate_fn/fns/first/mod.rs b/vortex-array/src/aggregate_fn/fns/first/mod.rs index 93e1fea6043..f04f31adb8f 100644 --- a/vortex-array/src/aggregate_fn/fns/first/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/first/mod.rs @@ -100,7 +100,7 @@ impl AggregateFnVTable for First { return Ok(true); } if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.first() { - let scalar = batch.scalar_at(idx)?; + let scalar = batch.execute_scalar(idx, ctx)?; partial.value = Some(scalar.into_nullable()); } Ok(true) diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index b684ea98b17..2947ab4a100 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -114,7 +114,7 @@ pub fn is_constant(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult VortexResult VortexResult { + pub fn make_partial( + batch: &ArrayRef, + is_constant: bool, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let partial_dtype = make_is_constant_partial_dtype(batch.dtype()); if is_constant { if batch.is_empty() { return Ok(Scalar::null(partial_dtype)); } - let first_value = batch.scalar_at(0)?.into_nullable(); + let first_value = batch.execute_scalar(0, ctx)?.into_nullable(); Ok(Scalar::struct_( partial_dtype, vec![Scalar::bool(true, Nullability::NonNullable), first_value], @@ -371,13 +375,13 @@ impl AggregateFnVTable for IsConstant { // Convert to ArrayRef for DynArray methods. let array_ref = c.clone().into_array(); - let all_invalid = array_ref.all_invalid()?; + let all_invalid = array_ref.all_invalid(ctx)?; if all_invalid { partial.check_value(Scalar::null(partial.element_dtype.as_nullable())); return Ok(()); } - let all_valid = array_ref.all_valid()?; + let all_valid = array_ref.all_valid(ctx)?; // Mixed nulls → not constant. if !all_valid && !all_invalid { partial.is_constant = false; @@ -386,7 +390,7 @@ impl AggregateFnVTable for IsConstant { // All valid from here. Check batch-level constancy. if c.len() == 1 { - partial.check_value(array_ref.scalar_at(0)?.into_nullable()); + partial.check_value(array_ref.execute_scalar(0, ctx)?.into_nullable()); return Ok(()); } @@ -410,7 +414,7 @@ impl AggregateFnVTable for IsConstant { return Ok(()); } - partial.check_value(array_ref.scalar_at(0)?.into_nullable()); + partial.check_value(array_ref.execute_scalar(0, ctx)?.into_nullable()); Ok(()) } } @@ -460,11 +464,13 @@ mod tests { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = buffer![0, 1].into_array(); - arr.statistics().compute_all(&[Stat::Min, Stat::Max])?; + arr.statistics() + .compute_all(&[Stat::Min, Stat::Max], &mut ctx)?; assert!(!is_constant(&arr, &mut ctx)?); let arr = buffer![0, 0].into_array(); - arr.statistics().compute_all(&[Stat::Min, Stat::Max])?; + arr.statistics() + .compute_all(&[Stat::Min, Stat::Max], &mut ctx)?; assert!(is_constant(&arr, &mut ctx)?); let arr = PrimitiveArray::from_option_iter([Some(0), Some(0)]).into_array(); @@ -477,13 +483,15 @@ mod tests { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = PrimitiveArray::from_iter([0.0, 0.0, f32::NAN]).into_array(); - arr.statistics().compute_all(&[Stat::Min, Stat::Max])?; + arr.statistics() + .compute_all(&[Stat::Min, Stat::Max], &mut ctx)?; assert!(!is_constant(&arr, &mut ctx)?); let arr = PrimitiveArray::from_option_iter([Some(f32::NEG_INFINITY), Some(f32::NEG_INFINITY)]) .into_array(); - arr.statistics().compute_all(&[Stat::Min, Stat::Max])?; + arr.statistics() + .compute_all(&[Stat::Min, Stat::Max], &mut ctx)?; assert!(is_constant(&arr, &mut ctx)?); Ok(()) } diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index f0a016796c9..d39a700fc02 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -114,7 +114,7 @@ fn is_sorted_impl(array: &ArrayRef, strict: bool, ctx: &mut ExecutionCtx) -> Vor 0 => {} // If we have a potential null value - it has to be the first one. 1 => { - if !array.is_invalid(0)? { + if !array.is_invalid(0, ctx)? { cache_is_sorted(array, strict, false); return Ok(false); } @@ -171,13 +171,18 @@ impl IsSorted { /// Kernels that compute `is_sorted` by delegating to child arrays can call this /// to package the boolean result into the partial struct format expected by the /// accumulator, avoiding duplicated boilerplate. - pub fn make_partial(batch: &ArrayRef, is_sorted: bool, strict: bool) -> VortexResult { + pub fn make_partial( + batch: &ArrayRef, + is_sorted: bool, + strict: bool, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let partial_dtype = make_is_sorted_partial_dtype(batch.dtype()); if batch.is_empty() { return Ok(Scalar::null(partial_dtype)); } - let first_value = batch.scalar_at(0)?.into_nullable(); - let last_value = batch.scalar_at(batch.len() - 1)?.into_nullable(); + let first_value = batch.execute_scalar(0, ctx)?.into_nullable(); + let last_value = batch.execute_scalar(batch.len() - 1, ctx)?.into_nullable(); // SAFETY: We constructed partial_dtype and the children match its field dtypes exactly. Ok(unsafe { Scalar::struct_unchecked( @@ -430,7 +435,7 @@ impl AggregateFnVTable for IsSorted { let array_ref = c.clone().into_array(); // Check boundary with previous chunk. - let first_value = array_ref.scalar_at(0)?.into_nullable(); + let first_value = array_ref.execute_scalar(0, ctx)?.into_nullable(); if let Some(ref self_last) = partial.last_value { if !self_last.is_null() && !first_value.is_null() { let boundary_ok = if partial.strict { @@ -440,8 +445,11 @@ impl AggregateFnVTable for IsSorted { }; if !boundary_ok { partial.is_sorted = false; - partial.last_value = - Some(array_ref.scalar_at(array_ref.len() - 1)?.into_nullable()); + partial.last_value = Some( + array_ref + .execute_scalar(array_ref.len() - 1, ctx)? + .into_nullable(), + ); if partial.first_value.is_none() { partial.first_value = Some(first_value); } @@ -451,8 +459,11 @@ impl AggregateFnVTable for IsSorted { || (self_last.is_null() && first_value.is_null() && partial.strict) { partial.is_sorted = false; - partial.last_value = - Some(array_ref.scalar_at(array_ref.len() - 1)?.into_nullable()); + partial.last_value = Some( + array_ref + .execute_scalar(array_ref.len() - 1, ctx)? + .into_nullable(), + ); if partial.first_value.is_none() { partial.first_value = Some(first_value); } @@ -479,8 +490,11 @@ impl AggregateFnVTable for IsSorted { if partial.first_value.is_none() { partial.first_value = Some(first_value); } - partial.last_value = - Some(array_ref.scalar_at(array_ref.len() - 1)?.into_nullable()); + partial.last_value = Some( + array_ref + .execute_scalar(array_ref.len() - 1, ctx)? + .into_nullable(), + ); Ok(()) } } diff --git a/vortex-array/src/aggregate_fn/fns/last/mod.rs b/vortex-array/src/aggregate_fn/fns/last/mod.rs index 9edfce66437..5329413b4b4 100644 --- a/vortex-array/src/aggregate_fn/fns/last/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/last/mod.rs @@ -98,7 +98,7 @@ impl AggregateFnVTable for Last { ctx: &mut ExecutionCtx, ) -> VortexResult { if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.last() { - let scalar = batch.scalar_at(idx)?; + let scalar = batch.execute_scalar(idx, ctx)?; partial.value = Some(scalar.into_nullable()); } Ok(true) diff --git a/vortex-array/src/aggregate_fn/foreign.rs b/vortex-array/src/aggregate_fn/foreign.rs index 26672cb23b7..feb07e47175 100644 --- a/vortex-array/src/aggregate_fn/foreign.rs +++ b/vortex-array/src/aggregate_fn/foreign.rs @@ -111,6 +111,10 @@ impl AggregateFnVTable for ForeignAggregateFnVTable { fn finalize(&self, _states: ArrayRef) -> VortexResult { vortex_bail!("Cannot execute unknown aggregate function '{}'", self.id) } + + fn finalize_scalar(&self, _partial: &Self::Partial) -> VortexResult { + vortex_bail!("Cannot execute unknown aggregate function '{}'", self.id) + } } pub fn new_foreign_aggregate_fn(id: AggregateFnId, metadata: Vec) -> AggregateFnRef { diff --git a/vortex-array/src/aggregate_fn/proto.rs b/vortex-array/src/aggregate_fn/proto.rs index b0e7d9828ed..9bcfab5818c 100644 --- a/vortex-array/src/aggregate_fn/proto.rs +++ b/vortex-array/src/aggregate_fn/proto.rs @@ -147,6 +147,10 @@ mod tests { fn finalize(&self, partials: ArrayRef) -> VortexResult { Ok(partials) } + + fn finalize_scalar(&self, _partial: &Self::Partial) -> VortexResult { + vortex_panic!("TestAgg is for serde tests only"); + } } #[test] diff --git a/vortex-array/src/aggregate_fn/vtable.rs b/vortex-array/src/aggregate_fn/vtable.rs index 73dd2604700..da6fcbc4165 100644 --- a/vortex-array/src/aggregate_fn/vtable.rs +++ b/vortex-array/src/aggregate_fn/vtable.rs @@ -14,11 +14,9 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Columnar; use crate::ExecutionCtx; -use crate::IntoArray; use crate::aggregate_fn::AggregateFn; use crate::aggregate_fn::AggregateFnId; use crate::aggregate_fn::AggregateFnRef; -use crate::arrays::ConstantArray; use crate::dtype::DType; use crate::scalar::Scalar; @@ -138,12 +136,7 @@ pub trait AggregateFnVTable: 'static + Sized + Clone + Send + Sync { /// /// The provided `state` has dtype as specified by `state_dtype`, the result scalar must have /// dtype as specified by `return_dtype`. - fn finalize_scalar(&self, partial: &Self::Partial) -> VortexResult { - let scalar = self.to_scalar(partial)?; - let array = ConstantArray::new(scalar, 1).into_array(); - let result = self.finalize(array)?; - result.scalar_at(0) - } + fn finalize_scalar(&self, partial: &Self::Partial) -> VortexResult; } #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index b8ebee660bc..3fe72921b7c 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -206,24 +206,33 @@ impl ArrayRef { } /// Fetch the scalar at the given index. + #[deprecated( + note = "Use `execute_scalar` instead, which allows passing an execution context for more \ + efficient execution when fetching multiple scalars from the same array." + )] pub fn scalar_at(&self, index: usize) -> VortexResult { + self.execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + } + + /// Execute the array to extract a scalar at the given index. + pub fn execute_scalar(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { vortex_ensure!(index < self.len(), OutOfBounds: index, 0, self.len()); - if self.is_invalid(index)? { + if self.is_invalid(index, ctx)? { return Ok(Scalar::null(self.dtype().clone())); } - let scalar = self.0.scalar_at(self, index)?; + let scalar = self.0.execute_scalar(self, index, ctx)?; vortex_ensure!(self.dtype() == scalar.dtype(), "Scalar dtype mismatch"); Ok(scalar) } /// Returns whether the item at `index` is valid. - pub fn is_valid(&self, index: usize) -> VortexResult { + pub fn is_valid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { vortex_ensure!(index < self.len(), OutOfBounds: index, 0, self.len()); match self.validity()? { Validity::NonNullable | Validity::AllValid => Ok(true), Validity::AllInvalid => Ok(false), Validity::Array(a) => a - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_bool() .value() .ok_or_else(|| vortex_err!("validity value at index {} is null", index)), @@ -231,25 +240,25 @@ impl ArrayRef { } /// Returns whether the item at `index` is invalid. - pub fn is_invalid(&self, index: usize) -> VortexResult { - Ok(!self.is_valid(index)?) + pub fn is_invalid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(!self.is_valid(index, ctx)?) } /// Returns whether all items in the array are valid. - pub fn all_valid(&self) -> VortexResult { + pub fn all_valid(&self, ctx: &mut ExecutionCtx) -> VortexResult { match self.validity()? { Validity::NonNullable | Validity::AllValid => Ok(true), Validity::AllInvalid => Ok(false), - Validity::Array(a) => Ok(a.statistics().compute_min::().unwrap_or(false)), + Validity::Array(a) => Ok(a.statistics().compute_min::(ctx).unwrap_or(false)), } } /// Returns whether the array is all invalid. - pub fn all_invalid(&self) -> VortexResult { + pub fn all_invalid(&self, ctx: &mut ExecutionCtx) -> VortexResult { match self.validity()? { Validity::NonNullable | Validity::AllValid => Ok(false), Validity::AllInvalid => Ok(true), - Validity::Array(a) => Ok(!a.statistics().compute_max::().unwrap_or(true)), + Validity::Array(a) => Ok(!a.statistics().compute_max::(ctx).unwrap_or(true)), } } diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index eada9e79a0a..8193c128b1e 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -16,8 +16,6 @@ use vortex_error::vortex_panic; use vortex_session::registry::Id; use crate::ExecutionCtx; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; @@ -73,11 +71,6 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// Returns the encoding ID of the array. fn encoding_id(&self) -> ArrayId; - /// Fetch the scalar at the given index. - /// - /// This method panics if the index is out of bounds for the array. - fn scalar_at(&self, this: &ArrayRef, index: usize) -> VortexResult; - /// Returns the [`Validity`] of the array. fn validity(&self, this: &ArrayRef) -> VortexResult; @@ -165,6 +158,16 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { child_idx: usize, ctx: &mut ExecutionCtx, ) -> VortexResult>; + + /// Execute the scalar at the given index. + /// + /// This method panics if the index is out of bounds for the array. + fn execute_scalar( + &self, + this: &ArrayRef, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult; } /// Trait for converting a type into a Vortex [`ArrayRef`]. @@ -213,15 +216,6 @@ impl DynArray for ArrayInner { self.vtable.id() } - fn scalar_at(&self, this: &ArrayRef, index: usize) -> VortexResult { - let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - >::scalar_at( - view, - index, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - } - fn validity(&self, this: &ArrayRef) -> VortexResult { if self.dtype.is_nullable() { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; @@ -484,6 +478,16 @@ impl DynArray for ArrayInner { Ok(Some(result)) } + + fn execute_scalar( + &self, + this: &ArrayRef, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; + >::scalar_at(view, index, ctx) + } } /// Wrapper around `&mut dyn Hasher` that implements `Hasher` (and is `Sized`). diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 3cba3cb527a..7fb1248aab9 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -15,7 +15,10 @@ use std::sync::Arc; use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; @@ -344,8 +347,22 @@ impl Array { self.inner.slice(range) } + #[deprecated( + note = "Use `execute_scalar` instead, which allows passing an execution context for more \ + efficient execution when fetching multiple scalars from the same array." + )] pub fn scalar_at(&self, index: usize) -> VortexResult { - self.inner.scalar_at(index) + self.inner + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + } + + /// Execute the array to extract a scalar at the given index. + pub fn execute_scalar( + &self, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + self.inner.execute_scalar(index, ctx) } pub fn filter(&self, mask: vortex_mask::Mask) -> VortexResult { @@ -360,22 +377,23 @@ impl Array { self.inner.validity() } - pub fn is_valid(&self, index: usize) -> VortexResult { - self.inner.is_valid(index) + pub fn is_valid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { + self.inner.is_valid(index, ctx) } - pub fn is_invalid(&self, index: usize) -> VortexResult { - self.inner.is_invalid(index) + pub fn is_invalid(&self, index: usize, ctx: &mut ExecutionCtx) -> VortexResult { + self.inner.is_invalid(index, ctx) } - pub fn all_valid(&self) -> VortexResult { - self.inner.all_valid() + pub fn all_valid(&self, ctx: &mut ExecutionCtx) -> VortexResult { + self.inner.all_valid(ctx) } - pub fn all_invalid(&self) -> VortexResult { - self.inner.all_invalid() + pub fn all_invalid(&self, ctx: &mut ExecutionCtx) -> VortexResult { + self.inner.all_invalid(ctx) } + #[deprecated(note = "Use Array::::execute::() instead")] pub fn to_canonical(&self) -> VortexResult { self.inner.to_canonical() } @@ -392,18 +410,18 @@ impl Array { self.inner.as_constant() } - pub fn valid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + pub fn valid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { self.inner.valid_count(ctx) } - pub fn invalid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + pub fn invalid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { self.inner.invalid_count(ctx) } pub fn append_to_builder( &self, builder: &mut dyn crate::builders::ArrayBuilder, - ctx: &mut crate::ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { self.inner.append_to_builder(builder, ctx) } diff --git a/vortex-array/src/arrays/assertions.rs b/vortex-array/src/arrays/assertions.rs index 2dab3a23c44..389baa23857 100644 --- a/vortex-array/src/arrays/assertions.rs +++ b/vortex-array/src/arrays/assertions.rs @@ -30,8 +30,12 @@ fn execute_to_canonical(array: ArrayRef, ctx: &mut ExecutionCtx) -> ArrayRef { #[expect(clippy::unwrap_used)] fn find_mismatched_indices(left: &ArrayRef, right: &ArrayRef) -> Vec { assert_eq!(left.len(), right.len()); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); (0..left.len()) - .filter(|i| left.scalar_at(*i).unwrap() != right.scalar_at(*i).unwrap()) + .filter(|i| { + left.execute_scalar(*i, &mut ctx).unwrap() + != right.execute_scalar(*i, &mut ctx).unwrap() + }) .collect() } @@ -49,9 +53,13 @@ fn find_mismatched_indices(left: &ArrayRef, right: &ArrayRef) -> Vec { macro_rules! assert_nth_scalar { ($arr:expr, $n:expr, $expected:expr) => {{ use $crate::IntoArray as _; + use $crate::LEGACY_SESSION; + use $crate::VortexSessionExecute as _; let arr_ref: $crate::ArrayRef = $crate::IntoArray::into_array($arr.clone()); assert_eq!( - arr_ref.scalar_at($n).unwrap(), + arr_ref + .execute_scalar($n, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), $expected.try_into().unwrap() ); }}; @@ -68,12 +76,19 @@ macro_rules! assert_nth_scalar { #[macro_export] macro_rules! assert_nth_scalar_is_null { ($arr:expr, $n:expr) => {{ + use $crate::LEGACY_SESSION; + use $crate::VortexSessionExecute as _; let arr_ref: $crate::ArrayRef = $crate::IntoArray::into_array($arr.clone()); assert!( - arr_ref.scalar_at($n).unwrap().is_null(), + arr_ref + .execute_scalar($n, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null(), "expected scalar at index {} to be null, but was {:?}", $n, - arr_ref.scalar_at($n).unwrap() + arr_ref + .execute_scalar($n, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); }}; } diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 5ed85d7eab4..7f2889246a4 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -48,7 +48,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` /// # fn main() -> vortex_error::VortexResult<()> { /// use vortex_array::arrays::BoolArray; -/// use vortex_array::IntoArray; +/// use vortex_array::{IntoArray, LEGACY_SESSION, VortexSessionExecute}; /// /// // Create from iterator using FromIterator impl /// let array: BoolArray = [true, false, true, false].into_iter().collect(); @@ -58,7 +58,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// assert_eq!(sliced.len(), 2); /// /// // Access individual values -/// let value = array.scalar_at(0).unwrap(); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); +/// let value = array.execute_scalar(0, &mut ctx).unwrap(); /// assert_eq!(value, true.into()); /// # Ok(()) /// # } @@ -98,17 +99,17 @@ pub trait BoolArrayExt: TypedArrayRef { BitBuffer::new_with_offset(buffer, self.as_ref().len(), self.offset) } - fn maybe_to_mask(&self) -> VortexResult> { + fn maybe_to_mask(&self, ctx: &mut ExecutionCtx) -> VortexResult> { let all_valid = match &self.validity() { Validity::NonNullable | Validity::AllValid => true, Validity::AllInvalid => false, - Validity::Array(a) => a.statistics().compute_min::().unwrap_or(false), + Validity::Array(a) => a.statistics().compute_min::(ctx).unwrap_or(false), }; Ok(all_valid.then(|| Mask::from_buffer(self.to_bit_buffer()))) } - fn to_mask(&self) -> Mask { - self.maybe_to_mask() + fn to_mask(&self, ctx: &mut ExecutionCtx) -> Mask { + self.maybe_to_mask(ctx) .vortex_expect("failed to check validity") .vortex_expect("cannot convert nullable boolean array to mask") } @@ -369,7 +370,11 @@ mod tests { #[test] fn bool_array() { let arr = BoolArray::from_iter([true, false, true]); - let scalar = bool::try_from(&arr.scalar_at(0).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(scalar); } @@ -379,9 +384,17 @@ mod tests { assert!(matches!(arr.validity(), Ok(Validity::AllValid))); - let scalar = bool::try_from(&arr.scalar_at(0).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(scalar); - let scalar = bool::try_from(&arr.scalar_at(1).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(!scalar); } @@ -389,19 +402,35 @@ mod tests { fn test_bool_from_iter() { let arr = BoolArray::from_iter([Some(true), Some(true), None, Some(false), None]); - let scalar = bool::try_from(&arr.scalar_at(0).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(scalar); - let scalar = bool::try_from(&arr.scalar_at(1).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(scalar); - let scalar = arr.scalar_at(2).unwrap(); + let scalar = arr + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(scalar.is_null()); - let scalar = bool::try_from(&arr.scalar_at(3).unwrap()).unwrap(); + let scalar = bool::try_from( + &arr.execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + ) + .unwrap(); assert!(!scalar); - let scalar = arr.scalar_at(4).unwrap(); + let scalar = arr + .execute_scalar(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(scalar.is_null()); } diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 4537b685fa0..97be61cb524 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -269,6 +269,8 @@ mod test { use vortex_error::VortexResult; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedArrayExt; @@ -358,8 +360,17 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_valid since all non-empty chunks are all_valid - assert!(chunked.all_valid().unwrap()); - assert!(!chunked.into_array().all_invalid().unwrap()); + assert!( + chunked + .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !chunked + .into_array() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); Ok(()) } @@ -378,8 +389,17 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_invalid since all non-empty chunks are all_invalid - assert!(!chunked.all_valid().unwrap()); - assert!(chunked.into_array().all_invalid().unwrap()); + assert!( + !chunked + .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + chunked + .into_array() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); Ok(()) } @@ -398,8 +418,17 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be neither all_valid nor all_invalid - assert!(!chunked.all_valid().unwrap()); - assert!(!chunked.into_array().all_invalid().unwrap()); + assert!( + !chunked + .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !chunked + .into_array() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); Ok(()) } diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index a20e775cf5c..f3f9b45afc9 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -7,6 +7,8 @@ use vortex_buffer::Buffer; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::accessor::ArrayAccessor; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -224,6 +226,18 @@ pub fn pack_nested_lists() { let canon_values = chunked_list.unwrap().as_array().to_listview(); - assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap()); - assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap()); + assert_eq!( + l1.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert_eq!( + l2.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index 63bb4132eca..c3d7b5115e9 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -219,7 +219,9 @@ mod tests { use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::accessor::ArrayAccessor; use crate::arrays::ChunkedArray; use crate::arrays::ListArray; @@ -305,8 +307,20 @@ mod tests { let canon_values = chunked_list.unwrap().as_array().to_listview(); - assert_eq!(l1.scalar_at(0).unwrap(), canon_values.scalar_at(0).unwrap()); - assert_eq!(l2.scalar_at(0).unwrap(), canon_values.scalar_at(1).unwrap()); + assert_eq!( + l1.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert_eq!( + l2.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] diff --git a/vortex-array/src/arrays/chunked/vtable/operations.rs b/vortex-array/src/arrays/chunked/vtable/operations.rs index 7c7095a05c7..ce44b8231f8 100644 --- a/vortex-array/src/arrays/chunked/vtable/operations.rs +++ b/vortex-array/src/arrays/chunked/vtable/operations.rs @@ -14,10 +14,10 @@ impl OperationsVTable for Chunked { fn scalar_at( array: ArrayView<'_, Chunked>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let (chunk_index, chunk_offset) = array.find_chunk_idx(index)?; - array.chunk(chunk_index).scalar_at(chunk_offset) + array.chunk(chunk_index).execute_scalar(chunk_offset, ctx) } } diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 9adc39212cc..fbffbfe4576 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -348,7 +348,12 @@ mod tests { let const_null = ConstantArray::new(Scalar::null(DType::Null), 42); let actual = const_null.as_array().to_null(); assert_eq!(actual.len(), 42); - assert_eq!(actual.scalar_at(33).unwrap(), Scalar::null(DType::Null)); + assert_eq!( + actual + .execute_scalar(33, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::null(DType::Null) + ); } #[test] @@ -365,7 +370,10 @@ mod tests { let const_array = ConstantArray::new(scalar, 4).into_array(); let stats = const_array .statistics() - .compute_all(&all::().collect_vec()) + .compute_all( + &all::().collect_vec(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) .unwrap(); let canonical = const_array.to_canonical()?.into_array(); let canonical_stats = canonical.statistics(); @@ -395,7 +403,12 @@ mod tests { let canonical_const = const_array.to_primitive(); // Verify the scalar value is preserved through canonicalization - assert_eq!(canonical_const.scalar_at(0).unwrap(), f16_scalar); + assert_eq!( + canonical_const + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + f16_scalar + ); } #[test] @@ -608,10 +621,30 @@ mod tests { // Check elements are repeated correctly. let elements = canonical.elements().to_varbinview(); - assert_eq!(elements.scalar_at(0).unwrap(), "hello".into()); - assert_eq!(elements.scalar_at(1).unwrap(), "world".into()); - assert_eq!(elements.scalar_at(2).unwrap(), "hello".into()); - assert_eq!(elements.scalar_at(3).unwrap(), "world".into()); + assert_eq!( + elements + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + "hello".into() + ); + assert_eq!( + elements + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + "world".into() + ); + assert_eq!( + elements + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + "hello".into() + ); + assert_eq!( + elements + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + "world".into() + ); } #[test] @@ -655,12 +688,24 @@ mod tests { // Check elements including nulls. let elements = canonical.elements().to_primitive(); - assert_eq!(elements.scalar_at(0).unwrap(), Scalar::from(100i32)); assert_eq!( - elements.scalar_at(1).unwrap(), + elements + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(100i32) + ); + assert_eq!( + elements + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)) ); - assert_eq!(elements.scalar_at(2).unwrap(), Scalar::from(200i32)); + assert_eq!( + elements + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(200i32) + ); // Check element validity. let element_validity = elements @@ -703,11 +748,36 @@ mod tests { // Check pattern repeats correctly. for i in 0..1000 { let base = i * 5; - assert_eq!(elements.scalar_at(base).unwrap(), Scalar::from(1u8)); - assert_eq!(elements.scalar_at(base + 1).unwrap(), Scalar::from(2u8)); - assert_eq!(elements.scalar_at(base + 2).unwrap(), Scalar::from(3u8)); - assert_eq!(elements.scalar_at(base + 3).unwrap(), Scalar::from(4u8)); - assert_eq!(elements.scalar_at(base + 4).unwrap(), Scalar::from(5u8)); + assert_eq!( + elements + .execute_scalar(base, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(1u8) + ); + assert_eq!( + elements + .execute_scalar(base + 1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(2u8) + ); + assert_eq!( + elements + .execute_scalar(base + 2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(3u8) + ); + assert_eq!( + elements + .execute_scalar(base + 3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(4u8) + ); + assert_eq!( + elements + .execute_scalar(base + 4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Scalar::from(5u8) + ); } } } diff --git a/vortex-array/src/arrays/decimal/vtable/operations.rs b/vortex-array/src/arrays/decimal/vtable/operations.rs index cfd14d95d63..aca0f912583 100644 --- a/vortex-array/src/arrays/decimal/vtable/operations.rs +++ b/vortex-array/src/arrays/decimal/vtable/operations.rs @@ -32,6 +32,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::Decimal; use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; @@ -78,7 +80,9 @@ mod tests { ); assert_eq!( - array.scalar_at(0).unwrap(), + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::decimal( DecimalValue::I128(100), DecimalDType::new(3, 2), diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index 2398ed2c161..07de42dd384 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -13,6 +13,7 @@ use crate::arrays::dict::DictArraySlotsExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::scalar_fn::fns::cast::CastReduce; +use crate::validity::Validity; impl CastReduce for Dict { fn cast(array: ArrayView<'_, Dict>, dtype: &DType) -> VortexResult> { @@ -20,7 +21,10 @@ impl CastReduce for Dict { // TODO(joe): optimize this, could look at accessible values and fill_null not those? if !dtype.is_nullable() && array.values().dtype().is_nullable() - && !array.values().all_valid()? + && !matches!( + array.values().validity()?, + Validity::NonNullable | Validity::AllValid + ) { return Ok(None); } @@ -163,20 +167,9 @@ mod tests { &DType::Primitive(PType::I32, Nullability::NonNullable) ); - // Check that both codes and values are NonNullable again - let back_dict = back_to_non_nullable.as_::(); - assert_eq!( - back_dict.codes().dtype().nullability(), - Nullability::NonNullable - ); - assert_eq!( - back_dict.values().dtype().nullability(), - Nullability::NonNullable - ); - // Verify values are unchanged let original_values = dict.as_array().to_primitive(); - let final_values = back_dict.array().to_primitive(); + let final_values = back_to_non_nullable.to_primitive(); assert_arrays_eq!(original_values, final_values); } diff --git a/vortex-array/src/arrays/dict/compute/fill_null.rs b/vortex-array/src/arrays/dict/compute/fill_null.rs index fb1b4034bde..8627759256a 100644 --- a/vortex-array/src/arrays/dict/compute/fill_null.rs +++ b/vortex-array/src/arrays/dict/compute/fill_null.rs @@ -94,7 +94,9 @@ mod tests { use vortex_error::VortexExpect; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::DictArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; @@ -121,6 +123,10 @@ mod tests { .vortex_expect("operation should succeed in test"); let filled_primitive = filled.to_primitive(); assert_arrays_eq!(filled_primitive, PrimitiveArray::from_iter([10, 20, 20])); - assert!(filled_primitive.all_valid().unwrap()); + assert!( + filled_primitive + .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } } diff --git a/vortex-array/src/arrays/dict/compute/is_constant.rs b/vortex-array/src/arrays/dict/compute/is_constant.rs index f5a903a126f..82518963106 100644 --- a/vortex-array/src/arrays/dict/compute/is_constant.rs +++ b/vortex-array/src/arrays/dict/compute/is_constant.rs @@ -40,7 +40,7 @@ impl DynAggregateKernel for DictIsConstantKernel { // If codes are constant, only one dictionary value is referenced → constant. if is_constant(dict.codes(), ctx)? { - return Ok(Some(IsConstant::make_partial(batch, true)?)); + return Ok(Some(IsConstant::make_partial(batch, true, ctx)?)); } // Otherwise, check the values array. Filter to only referenced values if needed. @@ -53,6 +53,6 @@ impl DynAggregateKernel for DictIsConstantKernel { is_constant(&filtered_values, ctx)? }; - Ok(Some(IsConstant::make_partial(batch, result)?)) + Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) } } diff --git a/vortex-array/src/arrays/dict/compute/is_sorted.rs b/vortex-array/src/arrays/dict/compute/is_sorted.rs index cb20287ae83..6ab5d5727d5 100644 --- a/vortex-array/src/arrays/dict/compute/is_sorted.rs +++ b/vortex-array/src/arrays/dict/compute/is_sorted.rs @@ -44,7 +44,7 @@ impl DynAggregateKernel for DictIsSortedKernel { }; if result { - Ok(Some(IsSorted::make_partial(batch, true, strict)?)) + Ok(Some(IsSorted::make_partial(batch, true, strict, ctx)?)) } else { // We can't definitively say it's NOT sorted without canonicalizing, // so return None to let the accumulator handle it. diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index bf148004af3..23fc05d08d6 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -29,6 +29,7 @@ use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::like::LikeReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; use crate::scalar_fn::fns::pack::Pack; +use crate::validity::Validity; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&FilterReduceAdaptor(Dict)), @@ -99,7 +100,10 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { // If the scalar function is null-sensitive, then we cannot push it down to values if // we have any nulls in the codes. if array.codes().dtype().is_nullable() - && !array.codes().all_valid()? + && !matches!( + array.codes().validity()?, + Validity::NonNullable | Validity::AllValid + ) && sig.is_null_sensitive() { tracing::trace!( diff --git a/vortex-array/src/arrays/dict/take.rs b/vortex-array/src/arrays/dict/take.rs index cd98908dcb1..e38f2d3894a 100644 --- a/vortex-array/src/arrays/dict/take.rs +++ b/vortex-array/src/arrays/dict/take.rs @@ -21,6 +21,7 @@ use crate::matcher::Matcher; use crate::optimizer::rules::ArrayParentReduceRule; use crate::scalar::Scalar; use crate::stats::StatsSet; +use crate::validity::Validity; pub trait TakeReduce: VTable { /// Take elements from an array at the given indices without reading buffers. @@ -142,8 +143,12 @@ pub(crate) fn propagate_take_stats( target: &ArrayRef, indices: &ArrayRef, ) -> VortexResult<()> { + let indices_all_valid = matches!( + indices.validity()?, + Validity::NonNullable | Validity::AllValid + ); target.statistics().with_mut_typed_stats_set(|mut st| { - if indices.all_valid().unwrap_or(false) { + if indices_all_valid { let is_constant = source.statistics().get_as::(Stat::IsConstant); if is_constant == Some(Precision::Exact(true)) { // Any combination of elements from a constant array is still const diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 1fe15d5ce3d..cf1c26491b0 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -180,7 +180,7 @@ impl VTable for Dict { // TODO(joe): use stat get instead computing. // Also not the check to do here it take value validity using code validity, but this approx // is correct. - if array.codes().all_invalid()? { + if array.codes().all_invalid(ctx)? { return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), diff --git a/vortex-array/src/arrays/dict/vtable/operations.rs b/vortex-array/src/arrays/dict/vtable/operations.rs index 9c75e4f2924..1982a1e0870 100644 --- a/vortex-array/src/arrays/dict/vtable/operations.rs +++ b/vortex-array/src/arrays/dict/vtable/operations.rs @@ -15,11 +15,11 @@ impl OperationsVTable for Dict { fn scalar_at( array: ArrayView<'_, Dict>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let Some(dict_index) = array .codes() - .scalar_at(index)? + .execute_scalar(index, ctx)? .as_primitive() .as_::() else { @@ -28,7 +28,7 @@ impl OperationsVTable for Dict { Ok(array .values() - .scalar_at(dict_index)? + .execute_scalar(dict_index, ctx)? .cast(array.dtype()) .vortex_expect("Array dtype will only differ by nullability")) } diff --git a/vortex-array/src/arrays/extension/vtable/operations.rs b/vortex-array/src/arrays/extension/vtable/operations.rs index d86c7599aef..66de94b596a 100644 --- a/vortex-array/src/arrays/extension/vtable/operations.rs +++ b/vortex-array/src/arrays/extension/vtable/operations.rs @@ -14,11 +14,11 @@ impl OperationsVTable for Extension { fn scalar_at( array: ArrayView<'_, Extension>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { Ok(Scalar::extension_ref( array.ext_dtype().clone(), - array.storage_array().scalar_at(index)?, + array.storage_array().execute_scalar(index, ctx)?, )) } } diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index 1c0cd000c10..d1bea9d5128 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -85,7 +85,9 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::filter::execute::ConstantArray; @@ -333,7 +335,7 @@ mod test { let list0 = result_list.list_elements_at(0).unwrap(); assert_eq!( list0 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -342,7 +344,7 @@ mod test { ); assert_eq!( list0 - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 05f49b5c941..f6cd88410af 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -178,10 +178,10 @@ impl OperationsVTable for Filter { fn scalar_at( array: ArrayView<'_, Filter>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let rank_idx = array.mask.rank(index); - array.child().scalar_at(rank_idx) + array.child().execute_scalar(rank_idx, ctx) } } diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index 95cb101e9da..465b2817166 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -162,7 +162,7 @@ fn take_nullable_fsl( { Validity::NonNullable | Validity::AllValid => Mask::new_true(indices_len), Validity::AllInvalid => Mask::new_false(indices_len), - Validity::Array(a) => a.to_bool().to_mask(), + Validity::Array(a) => a.to_bool().to_mask(ctx), }; // We must use placeholder zeros for null lists to maintain the array length without diff --git a/vortex-array/src/arrays/fixed_size_list/tests/basic.rs b/vortex-array/src/arrays/fixed_size_list/tests/basic.rs index 3d4b6e20ce2..f910bd4a05f 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/basic.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/basic.rs @@ -6,6 +6,8 @@ use std::sync::Arc; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::dtype::DType; @@ -36,24 +38,84 @@ fn test_basic_fixed_size_list() { // Check the actual values in each list. let first_list = fsl.fixed_size_list_elements_at(0).unwrap(); - assert_eq!(first_list.scalar_at(0).unwrap(), 1i32.into()); - assert_eq!(first_list.scalar_at(1).unwrap(), 2i32.into()); - assert_eq!(first_list.scalar_at(2).unwrap(), 3i32.into()); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert_eq!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2i32.into() + ); + assert_eq!( + first_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); let second_list = fsl.fixed_size_list_elements_at(1).unwrap(); - assert_eq!(second_list.scalar_at(0).unwrap(), 4i32.into()); - assert_eq!(second_list.scalar_at(1).unwrap(), 5i32.into()); - assert_eq!(second_list.scalar_at(2).unwrap(), 6i32.into()); + assert_eq!( + second_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 4i32.into() + ); + assert_eq!( + second_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert_eq!( + second_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); let third_list = fsl.fixed_size_list_elements_at(2).unwrap(); - assert_eq!(third_list.scalar_at(0).unwrap(), 7i32.into()); - assert_eq!(third_list.scalar_at(1).unwrap(), 8i32.into()); - assert_eq!(third_list.scalar_at(2).unwrap(), 9i32.into()); + assert_eq!( + third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 7i32.into() + ); + assert_eq!( + third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 8i32.into() + ); + assert_eq!( + third_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 9i32.into() + ); let fourth_list = fsl.fixed_size_list_elements_at(3).unwrap(); - assert_eq!(fourth_list.scalar_at(0).unwrap(), 10i32.into()); - assert_eq!(fourth_list.scalar_at(1).unwrap(), 11i32.into()); - assert_eq!(fourth_list.scalar_at(2).unwrap(), 12i32.into()); + assert_eq!( + fourth_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 10i32.into() + ); + assert_eq!( + fourth_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 11i32.into() + ); + assert_eq!( + fourth_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 12i32.into() + ); } #[test] @@ -65,7 +127,9 @@ fn test_scalar_at() { let fsl = FixedSizeListArray::new(elements.into_array(), list_size, Validity::NonNullable, len); // First list: [1, 2, 3]. - let first = fsl.scalar_at(0).unwrap(); + let first = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( first, Scalar::fixed_size_list( @@ -77,12 +141,29 @@ fn test_scalar_at() { // Additionally check individual elements via fixed_size_list_at. let first_list = fsl.fixed_size_list_elements_at(0).unwrap(); - assert_eq!(first_list.scalar_at(0).unwrap(), 1i32.into()); - assert_eq!(first_list.scalar_at(1).unwrap(), 2i32.into()); - assert_eq!(first_list.scalar_at(2).unwrap(), 3i32.into()); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert_eq!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2i32.into() + ); + assert_eq!( + first_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); // Second list: [4, 5, 6]. - let second = fsl.scalar_at(1).unwrap(); + let second = fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( second, Scalar::fixed_size_list( @@ -94,9 +175,24 @@ fn test_scalar_at() { // Additionally check individual elements via fixed_size_list_at. let second_list = fsl.fixed_size_list_elements_at(1).unwrap(); - assert_eq!(second_list.scalar_at(0).unwrap(), 4i32.into()); - assert_eq!(second_list.scalar_at(1).unwrap(), 5i32.into()); - assert_eq!(second_list.scalar_at(2).unwrap(), 6i32.into()); + assert_eq!( + second_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 4i32.into() + ); + assert_eq!( + second_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert_eq!( + second_list + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); } #[test] @@ -110,14 +206,34 @@ fn test_fixed_size_list_at() { // Get the first list [1.0, 2.0]. let first_list = fsl.fixed_size_list_elements_at(0).unwrap(); assert_eq!(first_list.len(), list_size as usize); - assert_eq!(first_list.scalar_at(0).unwrap(), 1.0f64.into()); - assert_eq!(first_list.scalar_at(1).unwrap(), 2.0f64.into()); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1.0f64.into() + ); + assert_eq!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2.0f64.into() + ); // Get the third list [5.0, 6.0]. let third_list = fsl.fixed_size_list_elements_at(2).unwrap(); assert_eq!(third_list.len(), list_size as usize); - assert_eq!(third_list.scalar_at(0).unwrap(), 5.0f64.into()); - assert_eq!(third_list.scalar_at(1).unwrap(), 6.0f64.into()); + assert_eq!( + third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5.0f64.into() + ); + assert_eq!( + third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6.0f64.into() + ); } #[test] diff --git a/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs b/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs index a6fb279788d..39a7ada3af3 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/degenerate.rs @@ -6,6 +6,8 @@ use std::sync::Arc; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -54,7 +56,9 @@ fn test_fsl_size_0_length_1_non_nullable() { assert_eq!(fsl.elements().len(), 0); // Get the single empty list. - let scalar = fsl.scalar_at(0).unwrap(); + let scalar = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar.is_null()); assert_eq!( scalar, @@ -80,7 +84,9 @@ fn test_fsl_size_0_huge_length_non_nullable() { assert_eq!(fsl.elements().len(), 0); // Spot check a few lists. - let scalar_first = fsl.scalar_at(0).unwrap(); + let scalar_first = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar_first.is_null()); assert_eq!( scalar_first, @@ -91,7 +97,9 @@ fn test_fsl_size_0_huge_length_non_nullable() { ) ); - let scalar_middle = fsl.scalar_at(500_000_000_000).unwrap(); + let scalar_middle = fsl + .execute_scalar(500_000_000_000, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar_middle.is_null()); assert_eq!( scalar_middle, @@ -102,7 +110,9 @@ fn test_fsl_size_0_huge_length_non_nullable() { ) ); - let scalar_end = fsl.scalar_at(999_999_999_999).unwrap(); + let scalar_end = fsl + .execute_scalar(999_999_999_999, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar_end.is_null()); assert_eq!( scalar_end, @@ -152,7 +162,9 @@ fn test_fsl_size_0_length_1_nullable_valid() { assert_eq!(fsl.elements().len(), 0); // Get the single empty list (should be valid). - let scalar = fsl.scalar_at(0).unwrap(); + let scalar = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar.is_null()); assert_eq!( scalar, @@ -175,7 +187,9 @@ fn test_fsl_size_0_length_1_nullable_null() { assert_eq!(fsl.elements().len(), 0); // The single list should be null. - let scalar = fsl.scalar_at(0).unwrap(); + let scalar = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(scalar.is_null()); } @@ -200,7 +214,9 @@ fn test_fsl_size_0_length_10_nullable_mixed() { true, false, true, true, false, false, true, false, true, true, ]; for i in 0..len { - let scalar = fsl.scalar_at(i).unwrap(); + let scalar = fsl + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); if expected_valid[i] { assert!(!scalar.is_null()); assert_eq!( @@ -239,7 +255,9 @@ fn test_fsl_size_0_nullable_elements() { // All lists should be empty but valid. for i in 0..len { - let scalar = fsl.scalar_at(i).unwrap(); + let scalar = fsl + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!scalar.is_null()); } } diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs index 41447881996..81fd9afe0c5 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; @@ -72,7 +74,9 @@ fn test_fsl_of_fsl_basic() { // The first outer list should contain 3 inner lists. // We can check by slicing and examining scalars. - let first_scalar = outer_fsl.scalar_at(0).unwrap(); + let first_scalar = outer_fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first_scalar.is_null()); // Check the actual values in the nested structure. @@ -84,24 +88,54 @@ fn test_fsl_of_fsl_basic() { .to_fixed_size_list() .fixed_size_list_elements_at(0) .unwrap(); - assert_eq!(inner_list_0.scalar_at(0).unwrap(), 1i32.into()); - assert_eq!(inner_list_0.scalar_at(1).unwrap(), 2i32.into()); + assert_eq!( + inner_list_0 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert_eq!( + inner_list_0 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2i32.into() + ); // Check second inner list [3,4]. let inner_list_1 = first_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(1) .unwrap(); - assert_eq!(inner_list_1.scalar_at(0).unwrap(), 3i32.into()); - assert_eq!(inner_list_1.scalar_at(1).unwrap(), 4i32.into()); + assert_eq!( + inner_list_1 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); + assert_eq!( + inner_list_1 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 4i32.into() + ); // Check third inner list [5,6]. let inner_list_2 = first_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(2) .unwrap(); - assert_eq!(inner_list_2.scalar_at(0).unwrap(), 5i32.into()); - assert_eq!(inner_list_2.scalar_at(1).unwrap(), 6i32.into()); + assert_eq!( + inner_list_2 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert_eq!( + inner_list_2 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); // Second outer list contains: [[7,8], [9,10], [11,12]]. let second_outer_list = outer_fsl.fixed_size_list_elements_at(1).unwrap(); @@ -111,24 +145,54 @@ fn test_fsl_of_fsl_basic() { .to_fixed_size_list() .fixed_size_list_elements_at(0) .unwrap(); - assert_eq!(inner_list_0.scalar_at(0).unwrap(), 7i32.into()); - assert_eq!(inner_list_0.scalar_at(1).unwrap(), 8i32.into()); + assert_eq!( + inner_list_0 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 7i32.into() + ); + assert_eq!( + inner_list_0 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 8i32.into() + ); // Check second inner list [9,10]. let inner_list_1 = second_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(1) .unwrap(); - assert_eq!(inner_list_1.scalar_at(0).unwrap(), 9i32.into()); - assert_eq!(inner_list_1.scalar_at(1).unwrap(), 10i32.into()); + assert_eq!( + inner_list_1 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 9i32.into() + ); + assert_eq!( + inner_list_1 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 10i32.into() + ); // Check third inner list [11,12]. let inner_list_2 = second_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(2) .unwrap(); - assert_eq!(inner_list_2.scalar_at(0).unwrap(), 11i32.into()); - assert_eq!(inner_list_2.scalar_at(1).unwrap(), 12i32.into()); + assert_eq!( + inner_list_2 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 11i32.into() + ); + assert_eq!( + inner_list_2 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 12i32.into() + ); } #[test] @@ -173,13 +237,28 @@ fn test_fsl_of_fsl_with_nulls() { assert_eq!(outer_fsl.len(), outer_len); // First outer list is valid. - assert!(!outer_fsl.scalar_at(0).unwrap().is_null()); + assert!( + !outer_fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Second outer list is null. - assert!(outer_fsl.scalar_at(1).unwrap().is_null()); + assert!( + outer_fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Third outer list is valid. - assert!(!outer_fsl.scalar_at(2).unwrap().is_null()); + assert!( + !outer_fsl + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } #[test] @@ -236,30 +315,70 @@ fn test_deeply_nested_fsl() { .to_fixed_size_list() .fixed_size_list_elements_at(0) .unwrap(); - assert_eq!(level1_0_0.scalar_at(0).unwrap(), 1i32.into()); - assert_eq!(level1_0_0.scalar_at(1).unwrap(), 2i32.into()); + assert_eq!( + level1_0_0 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert_eq!( + level1_0_0 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2i32.into() + ); let level1_0_1 = level2_0 .to_fixed_size_list() .fixed_size_list_elements_at(1) .unwrap(); - assert_eq!(level1_0_1.scalar_at(0).unwrap(), 3i32.into()); - assert_eq!(level1_0_1.scalar_at(1).unwrap(), 4i32.into()); + assert_eq!( + level1_0_1 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); + assert_eq!( + level1_0_1 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 4i32.into() + ); // Second level-2 list: [[5,6],[7,8]]. let level1_1_0 = level2_1 .to_fixed_size_list() .fixed_size_list_elements_at(0) .unwrap(); - assert_eq!(level1_1_0.scalar_at(0).unwrap(), 5i32.into()); - assert_eq!(level1_1_0.scalar_at(1).unwrap(), 6i32.into()); + assert_eq!( + level1_1_0 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert_eq!( + level1_1_0 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); let level1_1_1 = level2_1 .to_fixed_size_list() .fixed_size_list_elements_at(1) .unwrap(); - assert_eq!(level1_1_1.scalar_at(0).unwrap(), 7i32.into()); - assert_eq!(level1_1_1.scalar_at(1).unwrap(), 8i32.into()); + assert_eq!( + level1_1_1 + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 7i32.into() + ); + assert_eq!( + level1_1_1 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 8i32.into() + ); } //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs b/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs index 13894ae3ec4..20deef502ed 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nullability.rs @@ -6,6 +6,8 @@ use std::sync::Arc; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; @@ -30,7 +32,9 @@ fn test_nullable_fsl_with_nulls() { assert_eq!(fsl.list_size(), list_size); // First list is valid: [1, 2]. - let first = fsl.scalar_at(0).unwrap(); + let first = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first.is_null()); assert_eq!( first, @@ -43,15 +47,29 @@ fn test_nullable_fsl_with_nulls() { // Check individual elements of the first list. let first_list = fsl.fixed_size_list_elements_at(0).unwrap(); - assert_eq!(first_list.scalar_at(0).unwrap(), 1i32.into()); - assert_eq!(first_list.scalar_at(1).unwrap(), 2i32.into()); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert_eq!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 2i32.into() + ); // Second list is null. - let second = fsl.scalar_at(1).unwrap(); + let second = fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second.is_null()); // Third list is valid: [5, 6]. - let third = fsl.scalar_at(2).unwrap(); + let third = fsl + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!third.is_null()); assert_eq!( third, @@ -64,11 +82,23 @@ fn test_nullable_fsl_with_nulls() { // Check individual elements of the third list. let third_list = fsl.fixed_size_list_elements_at(2).unwrap(); - assert_eq!(third_list.scalar_at(0).unwrap(), 5i32.into()); - assert_eq!(third_list.scalar_at(1).unwrap(), 6i32.into()); + assert_eq!( + third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert_eq!( + third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); // Fourth list is null. - let fourth = fsl.scalar_at(3).unwrap(); + let fourth = fsl + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(fourth.is_null()); } @@ -92,7 +122,9 @@ fn test_nullable_elements_non_nullable_lists() { )); // First list: [Some(1), None, Some(3)]. - let first = fsl.scalar_at(0).unwrap(); + let first = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first.is_null()); assert_eq!( first, @@ -104,7 +136,9 @@ fn test_nullable_elements_non_nullable_lists() { ); // Second list: [Some(4), Some(5), None]. - let second = fsl.scalar_at(1).unwrap(); + let second = fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!second.is_null()); assert_eq!( second, @@ -130,7 +164,9 @@ fn test_nullable_elements_and_nullable_lists() { assert_eq!(fsl.len(), len); // First list is valid: [Some(10), None]. - let first = fsl.scalar_at(0).unwrap(); + let first = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first.is_null()); assert_eq!( first, @@ -143,15 +179,29 @@ fn test_nullable_elements_and_nullable_lists() { // Check individual elements of the first list. let first_list = fsl.fixed_size_list_elements_at(0).unwrap(); - assert_eq!(first_list.scalar_at(0).unwrap(), Some(10u16).into()); - assert_eq!(first_list.scalar_at(1).unwrap(), None::.into()); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + Some(10u16).into() + ); + assert_eq!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + None::.into() + ); // Second list is null (but elements would be [Some(20), Some(30)]). - let second = fsl.scalar_at(1).unwrap(); + let second = fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second.is_null()); // Third list is valid: [None, None]. - let third = fsl.scalar_at(2).unwrap(); + let third = fsl + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!third.is_null()); assert_eq!( third, @@ -164,8 +214,18 @@ fn test_nullable_elements_and_nullable_lists() { // Check individual elements of the third list. let third_list = fsl.fixed_size_list_elements_at(2).unwrap(); - assert_eq!(third_list.scalar_at(0).unwrap(), None::.into()); - assert_eq!(third_list.scalar_at(1).unwrap(), None::.into()); + assert_eq!( + third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + None::.into() + ); + assert_eq!( + third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + None::.into() + ); } #[test] @@ -182,7 +242,9 @@ fn test_alternating_nulls() { // Check alternating pattern. for i in 0..len { - let scalar = fsl.scalar_at(i).unwrap(); + let scalar = fsl + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); if i % 2 == 0 { assert!(!scalar.is_null()); let expected_value = u8::try_from(i + 1).unwrap(); @@ -212,7 +274,11 @@ fn test_validity_types() { { let fsl = FixedSizeListArray::new(elements.clone(), list_size, Validity::AllInvalid, len); for i in 0..len { - assert!(fsl.scalar_at(i).unwrap().is_null()); + assert!( + fsl.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } } @@ -226,10 +292,26 @@ fn test_validity_types() { len, ); - assert!(!fsl.scalar_at(0).unwrap().is_null()); - assert!(!fsl.scalar_at(1).unwrap().is_null()); - assert!(fsl.scalar_at(2).unwrap().is_null()); - assert!(!fsl.scalar_at(3).unwrap().is_null()); + assert!( + !fsl.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + !fsl.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + fsl.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + !fsl.execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } } @@ -255,22 +337,32 @@ fn test_mixed_nullability_patterns() { let fsl = FixedSizeListArray::new(elements.into_array(), list_size, validity, len); // List 0: valid with [Some(1), None]. - let list0 = fsl.scalar_at(0).unwrap(); + let list0 = fsl + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!list0.is_null()); // List 1: null. - let list1 = fsl.scalar_at(1).unwrap(); + let list1 = fsl + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(list1.is_null()); // List 2: valid with [Some(5), Some(6)]. - let list2 = fsl.scalar_at(2).unwrap(); + let list2 = fsl + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!list2.is_null()); // List 3: valid with [Some(7), None]. - let list3 = fsl.scalar_at(3).unwrap(); + let list3 = fsl + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!list3.is_null()); // List 4: valid with [None, Some(10)]. - let list4 = fsl.scalar_at(4).unwrap(); + let list4 = fsl + .execute_scalar(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!list4.is_null()); } diff --git a/vortex-array/src/arrays/fixed_size_list/tests/take.rs b/vortex-array/src/arrays/fixed_size_list/tests/take.rs index 53b959d22a6..8bccb80c8cb 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/take.rs @@ -11,6 +11,8 @@ use super::common::create_nullable_fsl; use super::common::create_single_element_fsl; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; @@ -93,7 +95,13 @@ fn test_take_degenerate_lists( assert_eq!(result.len(), expected_len); for (i, expected_null) in expected_nulls.iter().enumerate() { - assert_eq!(result.scalar_at(i).unwrap().is_null(), *expected_null); + assert_eq!( + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null(), + *expected_null + ); } } @@ -221,6 +229,12 @@ fn test_take_nullable_arrays_fsl_specific( assert_eq!(result.len(), indices.len()); for (i, expected_null) in expected_nulls.iter().enumerate() { - assert_eq!(result.scalar_at(i).unwrap().is_null(), *expected_null); + assert_eq!( + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null(), + *expected_null + ); } } diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs index ebd2ed463a4..9f4cf02fbf8 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs @@ -14,12 +14,12 @@ impl OperationsVTable for FixedSizeList { fn scalar_at( array: ArrayView<'_, FixedSizeList>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // By the preconditions we know that the list scalar is not null. let list = array.fixed_size_list_elements_at(index)?; let children_elements: Vec = (0..list.len()) - .map(|i| list.scalar_at(i)) + .map(|i| list.execute_scalar(i, ctx)) .collect::>()?; debug_assert_eq!(children_elements.len(), array.list_size() as usize); diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index c941060a42d..55b143f644f 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -197,9 +197,10 @@ impl ListData { // We can safely unwrap the DType as primitive now let offsets_ptype = offsets.dtype().as_ptype(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Offsets must be sorted (but not strictly sorted, zero-length lists are allowed) - if let Some(is_sorted) = offsets.statistics().compute_is_sorted() { + if let Some(is_sorted) = offsets.statistics().compute_is_sorted(&mut ctx) { vortex_ensure!(is_sorted, InvalidArgument: "offsets must be sorted"); } else { vortex_bail!(InvalidArgument: "offsets must report is_sorted statistic"); @@ -207,7 +208,6 @@ impl ListData { // Validate that offsets min is non-negative, and max does not exceed the length of // the elements array. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); if let Some(min_max) = min_max(offsets, &mut ctx)? { match_each_integer_ptype!(offsets_ptype, |P| { #[allow(clippy::absurd_extreme_comparisons, unused_comparisons)] @@ -301,7 +301,7 @@ pub trait ListArrayExt: TypedArrayRef { })) } else { self.offsets() - .scalar_at(index)? + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx())? .as_primitive() .as_::() .ok_or_else(|| vortex_error::vortex_err!("offset value does not fit in usize")) @@ -336,7 +336,7 @@ pub trait ListArrayExt: TypedArrayRef { } let offsets = self.offsets(); - let first_offset = offsets.scalar_at(0)?; + let first_offset = offsets.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; let adjusted_offsets = offsets.clone().binary( ConstantArray::new(first_offset, offsets.len()).into_array(), Operator::Sub, diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 22cfe224580..52c2296dec8 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -199,7 +199,9 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray as _; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ListArray; use crate::arrays::PrimitiveArray; @@ -239,9 +241,15 @@ mod test { let element_dtype: Arc = Arc::new(I32.into()); - assert!(result.is_valid(0).unwrap()); + assert!( + result + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(0).unwrap(), + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list( Arc::clone(&element_dtype), vec![0i32.into(), 5.into()], @@ -249,11 +257,21 @@ mod test { ) ); - assert!(result.is_invalid(1).unwrap()); + assert!( + result + .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); - assert!(result.is_valid(2).unwrap()); + assert!( + result + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(2).unwrap(), + result + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list( Arc::clone(&element_dtype), vec![3i32.into()], @@ -261,9 +279,15 @@ mod test { ) ); - assert!(result.is_valid(3).unwrap()); + assert!( + result + .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(3).unwrap(), + result + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list(element_dtype, vec![], Nullability::Nullable) ); } @@ -319,9 +343,15 @@ mod test { let element_dtype: Arc = Arc::new(I32.into()); - assert!(result.is_valid(0).unwrap()); + assert!( + result + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(0).unwrap(), + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list( Arc::clone(&element_dtype), vec![3i32.into()], @@ -329,9 +359,15 @@ mod test { ) ); - assert!(result.is_valid(1).unwrap()); + assert!( + result + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(1).unwrap(), + result + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list( Arc::clone(&element_dtype), vec![0i32.into(), 5.into()], @@ -339,9 +375,15 @@ mod test { ) ); - assert!(result.is_valid(2).unwrap()); + assert!( + result + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); assert_eq!( - result.scalar_at(2).unwrap(), + result + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::list(element_dtype, vec![], Nullability::NonNullable) ); } @@ -427,8 +469,16 @@ mod test { let result_view = result.to_listview(); assert_eq!(result_view.len(), 2); - assert!(result_view.is_valid(0).unwrap()); - assert!(result_view.is_valid(1).unwrap()); + assert!( + result_view + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result_view + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] @@ -448,9 +498,21 @@ mod test { let result_view = result.to_listview(); assert_eq!(result_view.len(), 3); - assert!(result_view.is_valid(0).unwrap()); - assert!(result_view.is_invalid(1).unwrap()); - assert!(result_view.is_valid(2).unwrap()); + assert!( + result_view + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result_view + .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result_view + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } /// Regression test for validity length mismatch bug. diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index 3d70e17fbdb..f27e0a9f305 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -51,7 +51,8 @@ fn test_simple_list_array() { vec![1.into(), 2.into()], Nullability::Nullable ), - list.scalar_at(0).unwrap() + list.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); assert_eq!( Scalar::list( @@ -59,11 +60,13 @@ fn test_simple_list_array() { vec![3.into(), 4.into()], Nullability::Nullable ), - list.scalar_at(1).unwrap() + list.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); assert_eq!( Scalar::list(Arc::new(I32.into()), vec![5.into()], Nullability::Nullable), - list.scalar_at(2).unwrap() + list.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); } @@ -81,12 +84,18 @@ fn test_simple_list_array_from_iter() { assert_eq!(list.len(), list_from_iter.len()); assert_eq!( - list.scalar_at(0).unwrap(), - list_from_iter.scalar_at(0).unwrap() + list.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + list_from_iter + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); assert_eq!( - list.scalar_at(1).unwrap(), - list_from_iter.scalar_at(1).unwrap() + list.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + list_from_iter + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); } @@ -218,10 +227,30 @@ fn test_list_filter_with_nulls() { assert_eq!(filtered.len(), 4); // Check validity of filtered array using scalar_at (works on any array). - assert!(filtered.scalar_at(0).unwrap().is_valid()); - assert!(!filtered.scalar_at(1).unwrap().is_valid()); // Was null. - assert!(!filtered.scalar_at(2).unwrap().is_valid()); // Was null. - assert!(filtered.scalar_at(3).unwrap().is_valid()); + assert!( + filtered + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); + assert!( + !filtered + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); // Was null. + assert!( + !filtered + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); // Was null. + assert!( + filtered + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); } #[test] @@ -625,7 +654,9 @@ fn test_list_of_lists() { assert_arrays_eq!(inner, PrimitiveArray::from_iter([7])); // Test scalar conversion. - let scalar = list_of_lists.scalar_at(0).unwrap(); + let scalar = list_of_lists + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(matches!(scalar.dtype(), DType::List(_, _))); let list_scalar = scalar.as_list(); assert_eq!(list_scalar.len(), 2); @@ -670,11 +701,15 @@ fn test_list_of_lists_nullable_outer() { )); // First element should be [[1, 2], [3]]. - let first = list_of_lists.scalar_at(0).unwrap(); + let first = list_of_lists + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first.is_null()); // Second element should be null. - let second = list_of_lists.scalar_at(1).unwrap(); + let second = list_of_lists + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second.is_null()); // Third element should be [[4, 5, 6]]. @@ -727,7 +762,10 @@ fn test_list_of_lists_nullable_inner() { assert_eq!(first_list.len(), 3); // Check that second inner list is null. - let second_inner = first_list.array().scalar_at(1).unwrap(); + let second_inner = first_list + .array() + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second_inner.is_null()); } @@ -755,7 +793,9 @@ fn test_list_of_lists_both_nullable() { )); // First outer list should have 2 elements, second is null inner list. - let first_outer = list_of_lists.scalar_at(0).unwrap(); + let first_outer = list_of_lists + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first_outer.is_null()); let first_outer_array = list_of_lists.list_elements_at(0).unwrap(); let first_list = first_outer_array.as_::(); @@ -766,11 +806,16 @@ fn test_list_of_lists_both_nullable() { assert_eq!(first_inner.len(), 2); // Second inner list should be null. - let second_inner = first_list.array().scalar_at(1).unwrap(); + let second_inner = first_list + .array() + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second_inner.is_null()); // Second outer list should be null. - let second_outer = list_of_lists.scalar_at(1).unwrap(); + let second_outer = list_of_lists + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second_outer.is_null()); // Third outer list should have [3]. @@ -784,7 +829,10 @@ fn test_list_of_lists_both_nullable() { let fourth_outer = list_of_lists.list_elements_at(3).unwrap(); let fourth_list = fourth_outer.as_::(); assert_eq!(fourth_list.len(), 1); - let inner = fourth_list.array().scalar_at(0).unwrap(); + let inner = fourth_list + .array() + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(inner.is_null()); } @@ -916,8 +964,12 @@ fn test_recursive_compact_list_of_lists() { let non_recursive_array = non_recursive.into_array(); let recursive_array = recursive.into_array(); assert_eq!( - non_recursive_array.scalar_at(0).unwrap(), - recursive_array.scalar_at(0).unwrap() + non_recursive_array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + recursive_array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); } diff --git a/vortex-array/src/arrays/list/vtable/operations.rs b/vortex-array/src/arrays/list/vtable/operations.rs index c9fc69c7bbe..02c686cd1f1 100644 --- a/vortex-array/src/arrays/list/vtable/operations.rs +++ b/vortex-array/src/arrays/list/vtable/operations.rs @@ -16,12 +16,12 @@ impl OperationsVTable for List { fn scalar_at( array: ArrayView<'_, List>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // By the preconditions we know that the list scalar is not null. let elems = array.list_elements_at(index)?; let scalars: Vec = (0..elems.len()) - .map(|i| elems.scalar_at(i)) + .map(|i| elems.execute_scalar(i, ctx)) .collect::>()?; Ok(Scalar::list( diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 8f9aee6406e..f440b360951 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -13,7 +13,9 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use crate::ArrayRef; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -345,8 +347,8 @@ pub trait ListViewArrayExt: TypedArrayRef { .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { self.offsets() - .scalar_at(index) - .vortex_expect("offsets must support scalar_at") + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("offsets must support execute_scalar") .as_primitive() .as_::() .vortex_expect("offset must fit in usize") @@ -365,8 +367,8 @@ pub trait ListViewArrayExt: TypedArrayRef { .map(|p| match_each_integer_ptype!(p.ptype(), |P| { p.as_slice::

()[index].as_() })) .unwrap_or_else(|| { self.sizes() - .scalar_at(index) - .vortex_expect("sizes must support scalar_at") + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("sizes must support execute_scalar") .as_primitive() .as_::() .vortex_expect("size must fit in usize") @@ -553,7 +555,8 @@ fn validate_zctl( ) -> VortexResult<()> { // Offsets must be sorted (but not strictly sorted, zero-length lists are allowed), even // if there are null views. - if let Some(is_sorted) = offsets_primitive.statistics().compute_is_sorted() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + if let Some(is_sorted) = offsets_primitive.statistics().compute_is_sorted(&mut ctx) { vortex_ensure!(is_sorted, "offsets must be sorted"); } else { vortex_bail!("offsets must report is_sorted statistic"); diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index 12c2c653162..d0a88a750e6 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -285,9 +285,13 @@ impl ListViewArray { // completely fine for us to use this as a lower-bounded start of the `elements`. self.offset_at(0) } else { - self.offsets().statistics().compute_min().vortex_expect( - "[ListViewArray::rebuild]: `offsets` must report min statistic that is a `usize`", - ) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + self.offsets() + .statistics() + .compute_min(&mut ctx) + .vortex_expect( + "[ListViewArray::rebuild]: `offsets` must report min statistic that is a `usize`", + ) }; let end = if self.is_zero_copy_to_list() { @@ -374,7 +378,9 @@ mod tests { use super::ListViewRebuildMode; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; use crate::arrays::listview::ListViewArrayExt; @@ -497,7 +503,12 @@ mod tests { // Note that element at index 2 (97) is preserved as a gap. let all_elements = trimmed.elements().to_primitive(); - assert_eq!(all_elements.scalar_at(2).unwrap(), 97i32.into()); + assert_eq!( + all_elements + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 97i32.into() + ); Ok(()) } @@ -537,10 +548,26 @@ mod tests { let exact = rebuilt.rebuild(ListViewRebuildMode::MakeExact)?; // Verify the result is still valid - assert!(exact.is_valid(0).unwrap()); - assert!(exact.is_valid(1).unwrap()); - assert!(!exact.is_valid(2).unwrap()); - assert!(!exact.is_valid(3).unwrap()); + assert!( + exact + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + exact + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !exact + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !exact + .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Verify data is preserved assert_arrays_eq!( diff --git a/vortex-array/src/arrays/listview/tests/basic.rs b/vortex-array/src/arrays/listview/tests/basic.rs index 5a1bf09d028..8925d97567f 100644 --- a/vortex-array/src/arrays/listview/tests/basic.rs +++ b/vortex-array/src/arrays/listview/tests/basic.rs @@ -54,7 +54,9 @@ fn test_basic_listview_comprehensive() { ); // Test scalar_at which returns entire lists as Scalar values. - let first_scalar = listview.scalar_at(0).unwrap(); + let first_scalar = listview + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( first_scalar, Scalar::list( @@ -138,9 +140,21 @@ fn test_from_list_array() -> VortexResult<()> { ); // Check validity is preserved. - assert!(list_view.is_valid(0).unwrap()); - assert!(list_view.is_invalid(1).unwrap()); - assert!(list_view.is_valid(2).unwrap()); + assert!( + list_view + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + list_view + .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + list_view + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Check third list. assert_arrays_eq!( @@ -198,15 +212,27 @@ fn test_listview_with_constant_arrays(#[case] const_sizes: bool, #[case] const_o } else if const_offsets { // All lists start at offset 0, different sizes (overlapping). assert_eq!( - listview.list_elements_at(0).unwrap().scalar_at(0).unwrap(), + listview + .list_elements_at(0) + .unwrap() + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), 1i32.into() ); assert_eq!( - listview.list_elements_at(1).unwrap().scalar_at(0).unwrap(), + listview + .list_elements_at(1) + .unwrap() + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), 1i32.into() ); assert_eq!( - listview.list_elements_at(2).unwrap().scalar_at(0).unwrap(), + listview + .list_elements_at(2) + .unwrap() + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), 1i32.into() ); } diff --git a/vortex-array/src/arrays/listview/tests/filter.rs b/vortex-array/src/arrays/listview/tests/filter.rs index 40eef10d825..271ea746482 100644 --- a/vortex-array/src/arrays/listview/tests/filter.rs +++ b/vortex-array/src/arrays/listview/tests/filter.rs @@ -11,7 +11,9 @@ use super::common::create_large_listview; use super::common::create_nullable_listview; use super::common::create_overlapping_listview; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -184,7 +186,7 @@ fn test_filter_extreme_offsets() { let list0 = result_list.list_elements_at(0).unwrap(); assert_eq!( list0 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -193,7 +195,7 @@ fn test_filter_extreme_offsets() { ); assert_eq!( list0 - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() diff --git a/vortex-array/src/arrays/listview/tests/nested.rs b/vortex-array/src/arrays/listview/tests/nested.rs index e5ec85a690c..4aa3edd75d5 100644 --- a/vortex-array/src/arrays/listview/tests/nested.rs +++ b/vortex-array/src/arrays/listview/tests/nested.rs @@ -4,6 +4,8 @@ use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -72,7 +74,7 @@ fn test_listview_of_listview_with_overlapping() { // inner[0] should be [1, 2, 3]. assert_eq!( inner0 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -81,7 +83,7 @@ fn test_listview_of_listview_with_overlapping() { ); assert_eq!( inner0 - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -92,7 +94,7 @@ fn test_listview_of_listview_with_overlapping() { // inner[1] should be [3, 4, 5] - shares element 3 with inner[0]. assert_eq!( inner1 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -101,7 +103,7 @@ fn test_listview_of_listview_with_overlapping() { ); assert_eq!( inner1 - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -285,7 +287,7 @@ fn test_listview_zero_and_overlapping() { assert_eq!(inner1.len(), 3); // [1, 2, 3] assert_eq!( inner1 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -304,7 +306,7 @@ fn test_listview_zero_and_overlapping() { assert_eq!(inner3.len(), 3); // [2, 3, 4] assert_eq!( inner3 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -381,7 +383,12 @@ fn test_listview_of_struct_with_nulls() { assert_eq!(list1.len(), 3); // The middle element (struct[2]) should be null. - assert!(list1.scalar_at(1).unwrap().is_null()); + assert!( + list1 + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Test slicing preserves null handling. let sliced = listview.slice(1..3).unwrap(); diff --git a/vortex-array/src/arrays/listview/tests/nullability.rs b/vortex-array/src/arrays/listview/tests/nullability.rs index aa56291e233..ff3f9c44c6c 100644 --- a/vortex-array/src/arrays/listview/tests/nullability.rs +++ b/vortex-array/src/arrays/listview/tests/nullability.rs @@ -7,6 +7,8 @@ use rstest::rstest; use vortex_buffer::buffer; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -34,9 +36,21 @@ fn test_nullable_listview_comprehensive() { assert_eq!(listview.len(), 3); // Check validity. - assert!(listview.is_valid(0).unwrap()); - assert!(listview.is_invalid(1).unwrap()); - assert!(listview.is_valid(2).unwrap()); + assert!( + listview + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + listview + .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + listview + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Check dtype reflects nullability. assert!(matches!( @@ -45,7 +59,9 @@ fn test_nullable_listview_comprehensive() { )); // Test scalar_at with nulls. - let first = listview.scalar_at(0).unwrap(); + let first = listview + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!first.is_null()); assert_eq!( first, @@ -56,10 +72,14 @@ fn test_nullable_listview_comprehensive() { ) ); - let second = listview.scalar_at(1).unwrap(); + let second = listview + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(second.is_null()); - let third = listview.scalar_at(2).unwrap(); + let third = listview + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(!third.is_null()); assert_eq!( third, @@ -73,8 +93,18 @@ fn test_nullable_listview_comprehensive() { // list_elements_at still returns data even for null lists. let null_list_data = listview.list_elements_at(1).unwrap(); assert_eq!(null_list_data.len(), 2); - assert_eq!(null_list_data.scalar_at(0).unwrap(), 3i32.into()); - assert_eq!(null_list_data.scalar_at(1).unwrap(), 4i32.into()); + assert_eq!( + null_list_data + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); + assert_eq!( + null_list_data + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 4i32.into() + ); } // Parameterized tests for different null patterns. @@ -91,7 +121,12 @@ fn test_nullable_patterns(#[case] validity: Validity, #[case] expected_validity: let listview = unsafe { ListViewArray::new_unchecked(elements, offsets, sizes, validity) }; for (i, &expected) in expected_validity.iter().enumerate() { - assert_eq!(listview.is_valid(i).unwrap(), expected); + assert_eq!( + listview + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + expected + ); } } @@ -113,22 +148,72 @@ fn test_nullable_elements() { // First list: [Some(1), None]. let first_list = listview.list_elements_at(0).unwrap(); assert_eq!(first_list.len(), 2); - assert!(!first_list.scalar_at(0).unwrap().is_null()); - assert_eq!(first_list.scalar_at(0).unwrap(), 1i32.into()); - assert!(first_list.scalar_at(1).unwrap().is_null()); + assert!( + !first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert_eq!( + first_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1i32.into() + ); + assert!( + first_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Second list: [Some(3), None]. let second_list = listview.list_elements_at(1).unwrap(); - assert!(!second_list.scalar_at(0).unwrap().is_null()); - assert_eq!(second_list.scalar_at(0).unwrap(), 3i32.into()); - assert!(second_list.scalar_at(1).unwrap().is_null()); + assert!( + !second_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert_eq!( + second_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3i32.into() + ); + assert!( + second_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Third list: [Some(5), Some(6)]. let third_list = listview.list_elements_at(2).unwrap(); - assert!(!third_list.scalar_at(0).unwrap().is_null()); - assert_eq!(third_list.scalar_at(0).unwrap(), 5i32.into()); - assert!(!third_list.scalar_at(1).unwrap().is_null()); - assert_eq!(third_list.scalar_at(1).unwrap(), 6i32.into()); + assert!( + !third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert_eq!( + third_list + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 5i32.into() + ); + assert!( + !third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert_eq!( + third_list + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 6i32.into() + ); // Check dtype of elements. assert!(matches!( diff --git a/vortex-array/src/arrays/listview/tests/operations.rs b/vortex-array/src/arrays/listview/tests/operations.rs index 09cf700bb98..b890c346d87 100644 --- a/vortex-array/src/arrays/listview/tests/operations.rs +++ b/vortex-array/src/arrays/listview/tests/operations.rs @@ -59,8 +59,13 @@ fn test_slice_comprehensive() { for i in 0..4 { // Compare the sliced elements assert_eq!( - full_list.array().scalar_at(i).unwrap(), - listview.scalar_at(i).unwrap(), + full_list + .array() + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + listview + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), "Mismatch at index {}", i ); @@ -148,8 +153,18 @@ fn test_slice_with_nulls() { let sliced_list = sliced.as_::(); assert_eq!(sliced_list.len(), 2); - assert!(sliced_list.array().is_invalid(0).unwrap()); // Original index 1 was null. - assert!(sliced_list.array().is_valid(1).unwrap()); // Original index 2 was valid. + assert!( + sliced_list + .array() + .is_invalid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Original index 1 was null. + assert!( + sliced_list + .array() + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Original index 2 was valid. // Verify offsets and sizes are preserved. assert_eq!(sliced_list.offset_at(0), 2); @@ -273,8 +288,16 @@ fn test_cast_with_nulls() { assert_eq!(result.dtype(), &target_dtype); let result_list = result.to_listview(); - assert!(result_list.is_valid(0).unwrap()); - assert!(result_list.is_invalid(1).unwrap()); + assert!( + result_list + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + result_list + .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[rstest] @@ -517,10 +540,26 @@ fn test_mask_preserves_structure() { let result_list = result.to_listview(); // Check validity: true in selection means null. - assert!(!result_list.is_valid(0).unwrap()); // Masked. - assert!(result_list.is_valid(1).unwrap()); // Not masked. - assert!(!result_list.is_valid(2).unwrap()); // Masked. - assert!(!result_list.is_valid(3).unwrap()); // Masked. + assert!( + !result_list + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Masked. + assert!( + result_list + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Not masked. + assert!( + !result_list + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Masked. + assert!( + !result_list + .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Masked. // Offsets and sizes are preserved. assert_eq!(result_list.offset_at(0), 0); @@ -554,9 +593,21 @@ fn test_mask_with_existing_nulls() { let result_list = result.to_listview(); // Check combined validity: - assert!(result_list.is_valid(0).unwrap()); // Was valid, mask is false -> valid. - assert!(!result_list.is_valid(1).unwrap()); // Was invalid, mask is true -> invalid. - assert!(!result_list.is_valid(2).unwrap()); // Was valid, mask is true -> invalid. + assert!( + result_list + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Was valid, mask is false -> valid. + assert!( + !result_list + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Was invalid, mask is true -> invalid. + assert!( + !result_list + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Was valid, mask is true -> invalid. } #[test] @@ -574,9 +625,21 @@ fn test_mask_with_gaps() { let result_list = result.to_listview(); assert_eq!(result_list.len(), 3); - assert!(!result_list.is_valid(0).unwrap()); // Masked - assert!(result_list.is_valid(1).unwrap()); // Not masked - assert!(result_list.is_valid(2).unwrap()); // Not masked + assert!( + !result_list + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Masked + assert!( + result_list + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Not masked + assert!( + result_list + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Not masked // Offsets and sizes still preserved assert_eq!(result_list.offset_at(1), 4); @@ -606,9 +669,21 @@ fn test_mask_constant_arrays() { let result_list = result.to_listview(); assert_eq!(result_list.len(), 3); - assert!(result_list.is_valid(0).unwrap()); - assert!(!result_list.is_valid(1).unwrap()); // Masked - assert!(result_list.is_valid(2).unwrap()); + assert!( + result_list + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !result_list + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Masked + assert!( + result_list + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // All offsets and sizes remain constant assert_eq!(result_list.offset_at(0), 1); diff --git a/vortex-array/src/arrays/listview/tests/take.rs b/vortex-array/src/arrays/listview/tests/take.rs index 042d93dfe59..b1134f97a04 100644 --- a/vortex-array/src/arrays/listview/tests/take.rs +++ b/vortex-array/src/arrays/listview/tests/take.rs @@ -10,7 +10,9 @@ use super::common::create_large_listview; use super::common::create_nullable_listview; use super::common::create_overlapping_listview; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -173,7 +175,7 @@ fn test_take_extreme_offsets() { let list0 = result_list.list_elements_at(0).unwrap(); assert_eq!( list0 - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() @@ -182,7 +184,7 @@ fn test_take_extreme_offsets() { ); assert_eq!( list0 - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_primitive() .as_::() diff --git a/vortex-array/src/arrays/listview/vtable/operations.rs b/vortex-array/src/arrays/listview/vtable/operations.rs index c899083027a..f0cb9539cc3 100644 --- a/vortex-array/src/arrays/listview/vtable/operations.rs +++ b/vortex-array/src/arrays/listview/vtable/operations.rs @@ -16,12 +16,12 @@ impl OperationsVTable for ListView { fn scalar_at( array: ArrayView<'_, ListView>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // By the preconditions we know that the list scalar is not null. let list = array.list_elements_at(index)?; let children: Vec = (0..list.len()) - .map(|i| list.scalar_at(i)) + .map(|i| list.execute_scalar(i, ctx)) .collect::>()?; Ok(Scalar::list( diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index e902fa56a47..54924f8053f 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -9,6 +9,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::ArrayRef; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -85,7 +87,11 @@ impl Array { let dtype = child.dtype().as_nullable(); let len = child.len(); let validity_slot = validity_to_child(&validity, len); - let data = MaskedData::try_new(len, child.all_valid()?, validity)?; + let data = MaskedData::try_new( + len, + child.all_valid(&mut LEGACY_SESSION.create_execution_ctx())?, + validity, + )?; Ok(unsafe { Array::from_parts_unchecked( ArrayParts::new(Masked, dtype, len, data) diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index c02a88a08fe..958a848d180 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -12,10 +12,15 @@ use crate::arrays::dict::TakeReduce; use crate::arrays::masked::MaskedArrayExt; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; +use crate::validity::Validity; impl TakeReduce for Masked { fn take(array: ArrayView<'_, Masked>, indices: &ArrayRef) -> VortexResult> { - let taken_child = if !indices.all_valid()? { + let indices_all_valid = matches!( + indices.validity()?, + Validity::NonNullable | Validity::AllValid + ); + let taken_child = if !indices_all_valid { // This is safe because we'll mask out these positions in the validity. let fill_scalar = Scalar::zero_value(indices.dtype()); let filled_take_indices = indices.clone().fill_null(fill_scalar)?; diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 551ead31448..ed108a77bf7 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -6,6 +6,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use super::*; +use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::ToCanonical as _; @@ -46,7 +47,10 @@ fn test_canonical_dtype_matches_array_dtype() -> VortexResult<()> { let child = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); let array = MaskedArray::try_new(child, Validity::AllValid).unwrap(); - let canonical = array.to_canonical()?; + let canonical = array + .clone() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; assert_eq!(canonical.dtype(), array.dtype()); Ok(()) } @@ -63,11 +67,28 @@ fn test_masked_child_with_validity() { // Positions where validity is false should be null in masked_child. let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); - assert!(prim.is_valid(0).unwrap()); - assert!(!prim.is_valid(1).unwrap()); - assert!(prim.is_valid(2).unwrap()); - assert!(!prim.is_valid(3).unwrap()); - assert!(prim.is_valid(4).unwrap()); + assert!( + prim.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !prim + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + prim.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !prim + .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + prim.is_valid(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] diff --git a/vortex-array/src/arrays/masked/vtable/canonical.rs b/vortex-array/src/arrays/masked/vtable/canonical.rs index 397cabf768a..be94c1cfddb 100644 --- a/vortex-array/src/arrays/masked/vtable/canonical.rs +++ b/vortex-array/src/arrays/masked/vtable/canonical.rs @@ -6,6 +6,7 @@ mod tests { use rstest::rstest; use vortex_error::VortexResult; + use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; @@ -33,7 +34,10 @@ mod tests { #[case] array: MaskedArray, #[case] expected_nullability: Nullability, ) -> VortexResult<()> { - let canonical = array.to_canonical()?; + let canonical = array + .clone() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; assert_eq!(canonical.dtype().nullability(), expected_nullability); assert_eq!(canonical.dtype(), array.dtype()); Ok(()) @@ -47,17 +51,36 @@ mod tests { ) .unwrap(); - let canonical = array.to_canonical()?; + let canonical = array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; let prim = canonical.into_primitive(); // Check that null positions match validity. let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); - assert!(prim.is_valid(0).unwrap()); - assert!(!prim.is_valid(1).unwrap()); - assert!(prim.is_valid(2).unwrap()); - assert!(!prim.is_valid(3).unwrap()); - assert!(prim.is_valid(4).unwrap()); + assert!( + prim.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !prim + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + prim.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + !prim + .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + prim.is_valid(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); Ok(()) } @@ -69,7 +92,9 @@ mod tests { ) .unwrap(); - let canonical = array.to_canonical()?; + let canonical = array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; assert_eq!(canonical.dtype().nullability(), Nullability::Nullable); assert_eq!( canonical diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 2bfd91c61a8..9e3083c01e5 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -20,7 +20,9 @@ use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::Precision; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -147,7 +149,11 @@ impl VTable for Masked { }; let validity_slot = validity_to_child(&validity, len); - let data = MaskedData::try_new(len, child.all_valid()?, validity)?; + let data = MaskedData::try_new( + len, + child.all_valid(&mut LEGACY_SESSION.create_execution_ctx())?, + validity, + )?; Ok( crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data) .with_slots(vec![Some(child), validity_slot]), diff --git a/vortex-array/src/arrays/masked/vtable/operations.rs b/vortex-array/src/arrays/masked/vtable/operations.rs index eb3132cf957..294cd840376 100644 --- a/vortex-array/src/arrays/masked/vtable/operations.rs +++ b/vortex-array/src/arrays/masked/vtable/operations.rs @@ -14,9 +14,9 @@ impl OperationsVTable for Masked { fn scalar_at( array: ArrayView<'_, Masked>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { // Invalid indices are handled by the entrypoint function. - Ok(array.child().scalar_at(index)?.into_nullable()) + Ok(array.child().execute_scalar(index, ctx)?.into_nullable()) } } diff --git a/vortex-array/src/arrays/null/compute/cast.rs b/vortex-array/src/arrays/null/compute/cast.rs index b39dea5988b..be6c73ad7d5 100644 --- a/vortex-array/src/arrays/null/compute/cast.rs +++ b/vortex-array/src/arrays/null/compute/cast.rs @@ -32,6 +32,8 @@ mod tests { use rstest::rstest; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::NullArray; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; @@ -64,7 +66,12 @@ mod tests { // Verify all values are null for i in 0..5 { - assert!(result.scalar_at(i).unwrap().is_null()); + assert!( + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } } diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index e6c69beee54..1d787369d96 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -66,7 +66,9 @@ mod test { fn test_scalar_at_nulls() { let nulls = NullArray::new(10); - let scalar = nulls.scalar_at(0).unwrap(); + let scalar = nulls + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(scalar.is_null()); assert_eq!(scalar.dtype().clone(), DType::Null); } diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 6e88b02aaf0..30f68cecc51 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -123,7 +123,7 @@ impl VTable for Null { /// ``` /// # fn main() -> vortex_error::VortexResult<()> { /// use vortex_array::arrays::NullArray; -/// use vortex_array::IntoArray; +/// use vortex_array::{IntoArray, LEGACY_SESSION, VortexSessionExecute}; /// /// // Create a null array with 5 elements /// let array = NullArray::new(5); @@ -133,7 +133,8 @@ impl VTable for Null { /// assert_eq!(sliced.len(), 2); /// /// // All elements are null -/// let scalar = array.scalar_at(0).unwrap(); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); +/// let scalar = array.execute_scalar(0, &mut ctx).unwrap(); /// assert!(scalar.is_null()); /// # Ok(()) /// # } diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index b2be4205b7f..910409cb919 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -15,6 +15,8 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -111,12 +113,14 @@ pub trait PatchedArrayExt: PatchedArraySlotsExt { assert!(chunk * 1024 <= self.as_ref().len() + self.offset()); assert!(lane < self.n_lanes()); - let start = self - .lane_offsets() - .scalar_at(chunk * self.n_lanes() + lane)?; - let stop = self - .lane_offsets() - .scalar_at(chunk * self.n_lanes() + lane + 1)?; + let start = self.lane_offsets().execute_scalar( + chunk * self.n_lanes() + lane, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + let stop = self.lane_offsets().execute_scalar( + chunk * self.n_lanes() + lane + 1, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let start = start .as_primitive() @@ -186,7 +190,7 @@ impl Patched { ); vortex_ensure!( - patches.values().all_valid()?, + patches.values().all_valid(ctx)?, "PatchedArray cannot be built from Patches with nulls" ); diff --git a/vortex-array/src/arrays/patched/vtable/operations.rs b/vortex-array/src/arrays/patched/vtable/operations.rs index 95beeaaf87f..f0491666e56 100644 --- a/vortex-array/src/arrays/patched/vtable/operations.rs +++ b/vortex-array/src/arrays/patched/vtable/operations.rs @@ -43,12 +43,15 @@ impl OperationsVTable for Patched { // be slower. for (&patch_index, idx) in std::iter::zip(patch_indices.as_slice::(), range) { if patch_index == chunk_index { - return array.patch_values().scalar_at(idx)?.cast(array.dtype()); + return array + .patch_values() + .execute_scalar(idx, ctx)? + .cast(array.dtype()); } } // Otherwise, access the underlying value. - array.inner().scalar_at(index) + array.inner().execute_scalar(index, ctx) } } @@ -59,6 +62,8 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::Patched; use crate::dtype::Nullability; use crate::optimizer::ArrayOptimizer; @@ -85,19 +90,27 @@ mod tests { .into_array(); assert_eq!( - array.scalar_at(0).unwrap(), + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::primitive(0u16, Nullability::NonNullable) ); assert_eq!( - array.scalar_at(1).unwrap(), + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::primitive(1u16, Nullability::NonNullable) ); assert_eq!( - array.scalar_at(2).unwrap(), + array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::primitive(1u16, Nullability::NonNullable) ); assert_eq!( - array.scalar_at(3).unwrap(), + array + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::primitive(1u16, Nullability::NonNullable) ); } @@ -122,7 +135,9 @@ mod tests { .into_array(); for index in 0..array.len() { - let value = array.scalar_at(index).unwrap(); + let value = array + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); if [1, 2, 3].contains(&index) { assert_eq!(value, 1u16.into()); @@ -157,9 +172,19 @@ mod tests { assert!(array.is::()); - assert_eq!(array.scalar_at(0).unwrap(), 1u16.into()); + assert_eq!( + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1u16.into() + ); for index in 1..array.len() { - assert_eq!(array.scalar_at(index).unwrap(), 0u16.into()); + assert_eq!( + array + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 0u16.into() + ); } } } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index cb1f132b70f..c1bd0130eb9 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -68,6 +68,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// ``` /// # fn main() -> vortex_error::VortexResult<()> { /// use vortex_array::arrays::PrimitiveArray; +/// use vortex_array::{LEGACY_SESSION, VortexSessionExecute}; /// /// // Create from iterator using FromIterator impl /// let array: PrimitiveArray = [1i32, 2, 3, 4, 5].into_iter().collect(); @@ -76,7 +77,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; /// let sliced = array.slice(1..3)?; /// /// // Access individual values -/// let value = sliced.scalar_at(0).unwrap(); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); +/// let value = sliced.execute_scalar(0, &mut ctx).unwrap(); /// assert_eq!(value, 2i32.into()); /// /// # Ok(()) diff --git a/vortex-array/src/arrays/primitive/compute/take/mod.rs b/vortex-array/src/arrays/primitive/compute/take/mod.rs index aedda05b6b6..4023991c65d 100644 --- a/vortex-array/src/arrays/primitive/compute/take/mod.rs +++ b/vortex-array/src/arrays/primitive/compute/take/mod.rs @@ -136,6 +136,8 @@ mod test { use vortex_error::VortexExpect; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::compute::take::take_primitive_scalar; @@ -162,17 +164,23 @@ mod test { ); let actual = values.take(indices.into_array()).unwrap(); assert_eq!( - actual.scalar_at(0).vortex_expect("no fail"), + actual + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("no fail"), Scalar::from(Some(1)) ); // position 3 is null assert_eq!( - actual.scalar_at(1).vortex_expect("no fail"), + actual + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("no fail"), Scalar::null_native::() ); // the third index is null assert_eq!( - actual.scalar_at(2).vortex_expect("no fail"), + actual + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("no fail"), Scalar::null_native::() ); } diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index d0cb0efdbf4..4c0ed1acee7 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -11,6 +11,8 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; @@ -79,7 +81,9 @@ impl ArrayReduceRule for ScalarFnConstantRule { if array.is_empty() { Ok(Some(Canonical::empty(array.dtype()).into_array())) } else { - let result = array.array().scalar_at(0)?; + let result = array + .array() + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; Ok(Some(ConstantArray::new(result, array.len()).into_array())) } } diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index bcb0368772a..224a8e2f4c2 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -5,8 +5,6 @@ use vortex_error::VortexResult; use crate::ExecutionCtx; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::ConstantArray; @@ -20,25 +18,24 @@ impl OperationsVTable for ScalarFnVTable { fn scalar_at( array: ArrayView<'_, ScalarFnVTable>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let inputs: Vec<_> = array .children() .iter() - .map(|child| Ok(ConstantArray::new(child.scalar_at(index)?, 1).into_array())) + .map(|child| Ok(ConstantArray::new(child.execute_scalar(index, ctx)?, 1).into_array())) .collect::>()?; - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let args = VecExecutionArgs::new(inputs, 1); - let result = array.scalar_fn().execute(&args, &mut ctx)?; + let result = array.scalar_fn().execute(&args, ctx)?; - let scalar = match result.execute::(&mut ctx)? { + let scalar = match result.execute::(ctx)? { Columnar::Canonical(arr) => { tracing::info!( "Scalar function {} returned non-constant array from execution over all scalar inputs", array.scalar_fn(), ); - arr.into_array().scalar_at(0)? + arr.into_array().execute_scalar(0, ctx)? } Columnar::Constant(constant) => constant.scalar().clone(), }; @@ -61,7 +58,10 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexResult; + use crate::Canonical; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::arrays::ScalarFnArray; @@ -79,7 +79,10 @@ mod tests { let scalar_fn = ScalarFn::new(Binary, Operator::Add).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; - let result = scalar_fn_array.to_canonical()?.into_array(); + let result = scalar_fn_array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let expected = buffer![11i32, 22, 33].into_array(); assert_arrays_eq!(result, expected); @@ -94,7 +97,10 @@ mod tests { let scalar_fn = ScalarFn::new(Binary, Operator::Mul).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; - let result = scalar_fn_array.to_canonical()?.into_array(); + let result = scalar_fn_array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let expected = buffer![10i32, 18, 28].into_array(); assert_arrays_eq!(result, expected); @@ -113,7 +119,10 @@ mod tests { let scalar_fn = ScalarFn::new(Binary, Operator::Add).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; - let result = scalar_fn_array.to_canonical()?.into_array(); + let result = scalar_fn_array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let expected = PrimitiveArray::new( buffer![11i32, 0, 33], Validity::from_iter([true, false, true]), @@ -132,7 +141,10 @@ mod tests { let scalar_fn = ScalarFn::new(Binary, Operator::Eq).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; - let result = scalar_fn_array.to_canonical()?.into_array(); + let result = scalar_fn_array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array(); let expected = BoolArray::from_iter([false, true, false]).into_array(); assert_arrays_eq!(result, expected); diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index 3f67738684d..52245aeb198 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -118,9 +118,9 @@ impl OperationsVTable for Shared { fn scalar_at( array: ArrayView<'_, Shared>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - array.current_array_ref().scalar_at(index) + array.current_array_ref().execute_scalar(index, ctx) } } diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 911de401aa4..56dbcbcd915 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -163,9 +163,9 @@ impl OperationsVTable for Slice { fn scalar_at( array: ArrayView<'_, Slice>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - array.child().scalar_at(array.range.start + index) + array.child().execute_scalar(array.range.start + index, ctx) } } diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 3d147f9c85e..8e3192728e0 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -52,7 +52,7 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// use vortex_array::arrays::{StructArray, BoolArray}; /// use vortex_array::validity::Validity; /// use vortex_array::dtype::FieldNames; -/// use vortex_array::IntoArray; +/// use vortex_array::{IntoArray, LEGACY_SESSION, VortexSessionExecute}; /// use vortex_buffer::buffer; /// /// // Create struct with all non-null fields but struct-level nulls @@ -66,13 +66,14 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// 2, /// Validity::Array(BoolArray::from_iter([true, false]).into_array()), // row 1 is null /// ).unwrap(); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); /// /// // Row 0 is valid - returns a struct scalar with field values -/// let row0 = struct_array.scalar_at(0).unwrap(); +/// let row0 = struct_array.execute_scalar(0, &mut ctx).unwrap(); /// assert!(!row0.is_null()); /// /// // Row 1 is null at struct level - returns null even though fields have values -/// let row1 = struct_array.scalar_at(1).unwrap(); +/// let row1 = struct_array.execute_scalar(1, &mut ctx).unwrap(); /// assert!(row1.is_null()); /// ``` /// @@ -86,7 +87,7 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// use vortex_array::arrays::struct_::StructArrayExt; /// use vortex_array::validity::Validity; /// use vortex_array::dtype::FieldNames; -/// use vortex_array::IntoArray; +/// use vortex_array::{IntoArray, LEGACY_SESSION, VortexSessionExecute}; /// use vortex_buffer::buffer; /// /// // Create struct with duplicate "data" field names @@ -102,7 +103,8 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// /// // field_by_name returns the FIRST "data" field /// let first_data = struct_array.unmasked_field_by_name("data").unwrap(); -/// assert_eq!(first_data.scalar_at(0).unwrap(), 1i32.into()); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); +/// assert_eq!(first_data.execute_scalar(0, &mut ctx).unwrap(), 1i32.into()); /// ``` /// /// ## Field Operations diff --git a/vortex-array/src/arrays/struct_/compute/mod.rs b/vortex-array/src/arrays/struct_/compute/mod.rs index 6d6e8c5b788..9ddee9496aa 100644 --- a/vortex-array/src/arrays/struct_/compute/mod.rs +++ b/vortex-array/src/arrays/struct_/compute/mod.rs @@ -70,7 +70,12 @@ mod tests { .execute::(&mut LEGACY_SESSION.create_execution_ctx()) .unwrap(); assert_eq!(taken.len(), 1); - assert!(taken.into_array().all_invalid().unwrap()); + assert!( + taken + .into_array() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] diff --git a/vortex-array/src/arrays/struct_/tests.rs b/vortex-array/src/arrays/struct_/tests.rs index 38b12074900..2db9e62f4f5 100644 --- a/vortex-array/src/arrays/struct_/tests.rs +++ b/vortex-array/src/arrays/struct_/tests.rs @@ -4,7 +4,10 @@ use vortex_buffer::buffer; use vortex_error::VortexResult; +use crate::Canonical; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; @@ -140,10 +143,15 @@ fn test_uncompressed_size_in_bytes() -> VortexResult<()> { Validity::NonNullable, ); - let canonical_size = struct_array.to_canonical()?.into_array().nbytes(); + let canonical_size = struct_array + .clone() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_array() + .nbytes(); let uncompressed_size = struct_array .statistics() - .compute_uncompressed_size_in_bytes(); + .compute_uncompressed_size_in_bytes(&mut LEGACY_SESSION.create_execution_ctx()); assert_eq!(canonical_size, 2); assert_eq!(uncompressed_size, Some(4000)); diff --git a/vortex-array/src/arrays/struct_/vtable/operations.rs b/vortex-array/src/arrays/struct_/vtable/operations.rs index 021d93f3c2f..d59bf7eeca3 100644 --- a/vortex-array/src/arrays/struct_/vtable/operations.rs +++ b/vortex-array/src/arrays/struct_/vtable/operations.rs @@ -14,11 +14,11 @@ impl OperationsVTable for Struct { fn scalar_at( array: ArrayView<'_, Struct>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let field_scalars: VortexResult> = array .iter_unmasked_fields() - .map(|field| field.scalar_at(index)) + .map(|field| field.execute_scalar(index, ctx)) .collect(); // SAFETY: The vtable guarantees index is in-bounds and non-null before this is called. // Each field's scalar_at returns a scalar with the field's own dtype. diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index a09bd7495af..fb8b2d54790 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -12,7 +12,9 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_err; use crate::ArrayRef; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -210,7 +212,10 @@ impl VarBinData { // Skip host-only validation when offsets/bytes are not host-resident. if offsets.is_host() && bytes.is_on_host() { let last_offset = offsets - .scalar_at(offsets.len() - 1)? + .execute_scalar( + offsets.len() - 1, + &mut LEGACY_SESSION.create_execution_ctx(), + )? .as_primitive() .as_::() .ok_or_else( @@ -325,8 +330,8 @@ pub trait VarBinArrayExt: TypedArrayRef { (&self .offsets() - .scalar_at(index) - .vortex_expect("offsets must support scalar_at")) + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("offsets must support execute_scalar")) .try_into() .vortex_expect("Failed to convert offset to usize") } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index 9b63889e70f..2e329d5b9a6 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -7,6 +7,10 @@ use vortex_buffer::BufferMut; use vortex_error::vortex_panic; use crate::IntoArray; +#[cfg(debug_assertions)] +use crate::LEGACY_SESSION; +#[cfg(debug_assertions)] +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::dtype::DType; @@ -99,10 +103,14 @@ impl VarBinBuilder { // The builder guarantees offsets are monotonically increasing, so we can set // this stat eagerly. This avoids an O(n) recomputation when the array is // deserialized and VarBinArray::validate checks sortedness. - debug_assert!( - offsets.statistics().compute_is_sorted().unwrap_or(false), - "VarBinBuilder offsets must be sorted" - ); + #[cfg(debug_assertions)] + { + let offsets_are_sorted = offsets + .statistics() + .compute_is_sorted(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap_or(false); + debug_assert!(offsets_are_sorted, "VarBinBuilder offsets must be sorted"); + } offsets .statistics() .set(Stat::IsSorted, Precision::Exact(true.into())); @@ -122,6 +130,8 @@ impl VarBinBuilder { mod tests { use vortex_error::VortexResult; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::varbin::VarBinArrayExt; use crate::arrays::varbin::builder::VarBinBuilder; use crate::dtype::DType; @@ -142,10 +152,17 @@ mod tests { assert_eq!(array.len(), 3); assert_eq!(array.dtype().nullability(), Nullable); assert_eq!( - array.scalar_at(0).unwrap(), + array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::utf8("hello".to_string(), Nullable) ); - assert!(array.scalar_at(1).unwrap().is_null()); + assert!( + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } #[test] diff --git a/vortex-array/src/arrays/varbin/vtable/canonical.rs b/vortex-array/src/arrays/varbin/vtable/canonical.rs index 726bbfd8dcd..ecf651b3ea4 100644 --- a/vortex-array/src/arrays/varbin/vtable/canonical.rs +++ b/vortex-array/src/arrays/varbin/vtable/canonical.rs @@ -47,6 +47,8 @@ pub(crate) fn varbin_to_canonical( mod tests { use rstest::rstest; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; use crate::arrays::varbin::builder::VarBinBuilder; @@ -73,7 +75,11 @@ mod tests { let canonical = varbin.to_varbinview(); assert_eq!(canonical.dtype(), &dtype); - assert!(!canonical.is_valid(0).unwrap()); + assert!( + !canonical + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // First value is inlined (12 bytes) assert!(canonical.views()[1].is_inlined()); diff --git a/vortex-array/src/arrays/variant/vtable/operations.rs b/vortex-array/src/arrays/variant/vtable/operations.rs index fb6fc0eb499..32e4737e308 100644 --- a/vortex-array/src/arrays/variant/vtable/operations.rs +++ b/vortex-array/src/arrays/variant/vtable/operations.rs @@ -14,8 +14,8 @@ impl OperationsVTable for Variant { fn scalar_at( array: ArrayView<'_, Variant>, index: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - array.child().scalar_at(index) + array.child().execute_scalar(index, ctx) } } diff --git a/vortex-array/src/arrow/datum.rs b/vortex-array/src/arrow/datum.rs index ce77c3bb159..60a3ecacd42 100644 --- a/vortex-array/src/arrow/datum.rs +++ b/vortex-array/src/arrow/datum.rs @@ -11,6 +11,8 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrow::FromArrowArray; @@ -103,8 +105,8 @@ where Ok(ConstantArray::new( array - .scalar_at(0) - .vortex_expect("array of length 1 must support scalar_at(0)"), + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("array of length 1 must support execute_scalar(0)"), len, ) .into_array()) diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index 285ad733cfb..bb0d4fd65c7 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -147,7 +147,7 @@ fn list_view_zctl( // For ZCTL, we know that we only care about the final size. assert!(!sizes.is_empty()); let final_size = sizes - .scalar_at(sizes.len() - 1)? + .execute_scalar(sizes.len() - 1, ctx)? .cast(&DType::Primitive(O::PTYPE, Nullability::NonNullable))?; let final_size = final_size .as_primitive() diff --git a/vortex-array/src/arrow/record_batch.rs b/vortex-array/src/arrow/record_batch.rs index a7596169c15..02ec62f626a 100644 --- a/vortex-array/src/arrow/record_batch.rs +++ b/vortex-array/src/arrow/record_batch.rs @@ -17,7 +17,9 @@ use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::StructArray; use crate::arrow::ArrowArrayExecutor; +use crate::validity::Validity; +// deprecated(note = "Use ArrowArrayExecutor::execute_record_batch instead") impl TryFrom<&ArrayRef> for RecordBatch { type Error = VortexError; @@ -27,7 +29,7 @@ impl TryFrom<&ArrayRef> for RecordBatch { }; vortex_ensure!( - struct_array.all_valid()?, + matches!(struct_array.validity()?, Validity::AllValid), "RecordBatch can only be constructed from StructArray with no nulls" ); diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index 9f33722dcb7..d009fd451c9 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -308,6 +308,8 @@ impl Default for DecimalBuffer { #[cfg(test)] mod tests { + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::DecimalBuilder; @@ -329,7 +331,13 @@ mod tests { let i128s = i128s.finish(); for i in 0..i8s.len() { - assert_eq!(i8s.scalar_at(i).unwrap(), i128s.scalar_at(i).unwrap()); + assert_eq!( + i8s.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + i128s + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } } @@ -353,7 +361,9 @@ mod tests { // Test by taking a scalar from the array and appending it to a new builder. let mut builder2 = DecimalBuilder::new::(DecimalDType::new(10, 2), true.into()); for i in 0..array.len() { - let scalar = array.scalar_at(i).unwrap(); + let scalar = array + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); builder2.append_scalar(&scalar).unwrap(); } diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 87b623f5f21..b561cb3ed5f 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -281,7 +281,9 @@ mod tests { use super::FixedSizeListBuilder; use crate::IntoArray as _; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; @@ -923,7 +925,9 @@ mod tests { // Check actual values using scalar_at. - let scalar0 = array.scalar_at(0).unwrap(); + let scalar0 = array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let list0 = scalar0.as_list(); assert_eq!(list0.len(), 2); if let Some(list0_items) = list0.elements() { @@ -931,7 +935,9 @@ mod tests { assert_eq!(list0_items[1].as_primitive().typed_value::(), Some(2)); } - let scalar1 = array.scalar_at(1).unwrap(); + let scalar1 = array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let list1 = scalar1.as_list(); assert_eq!(list1.len(), 2); if let Some(list1_items) = list1.elements() { diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 4c49185d380..511f2120f1f 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -527,12 +527,20 @@ mod tests { let canon_values = chunked_list.unwrap().as_array().to_listview(); assert_eq!( - one_trailing_unused_element.scalar_at(0).unwrap(), - canon_values.scalar_at(0).unwrap() + one_trailing_unused_element + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); assert_eq!( - second_array.scalar_at(0).unwrap(), - canon_values.scalar_at(1).unwrap() + second_array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + canon_values + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() ); } @@ -563,7 +571,9 @@ mod tests { // Check actual values using scalar_at. - let scalar0 = array.scalar_at(0).unwrap(); + let scalar0 = array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let list0 = scalar0.as_list(); assert_eq!(list0.len(), 2); if let Some(list0_items) = list0.elements() { @@ -571,7 +581,9 @@ mod tests { assert_eq!(list0_items[1].as_primitive().typed_value::(), Some(2)); } - let scalar1 = array.scalar_at(1).unwrap(); + let scalar1 = array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let list1 = scalar1.as_list(); assert_eq!(list1.len(), 3); if let Some(list1_items) = list1.elements() { @@ -580,7 +592,9 @@ mod tests { assert_eq!(list1_items[2].as_primitive().typed_value::(), Some(5)); } - let scalar2 = array.scalar_at(2).unwrap(); + let scalar2 = array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let list2 = scalar2.as_list(); assert!(list2.is_null()); // This should be null. diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs index 72c6f286e42..60e96a2b806 100644 --- a/vortex-array/src/builders/mod.rs +++ b/vortex-array/src/builders/mod.rs @@ -11,6 +11,7 @@ //! ``` //! use vortex_array::builders::{builder_with_capacity, ArrayBuilder}; //! use vortex_array::dtype::{DType, Nullability}; +//! use vortex_array::{LEGACY_SESSION, VortexSessionExecute}; //! //! // Create a new builder for string data. //! let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4); @@ -21,11 +22,12 @@ //! builder.append_scalar(&"d".into()).unwrap(); //! //! let strings = builder.finish(); +//! let mut ctx = LEGACY_SESSION.create_execution_ctx(); //! -//! assert_eq!(strings.scalar_at(0).unwrap(), "a".into()); -//! assert_eq!(strings.scalar_at(1).unwrap(), "b".into()); -//! assert_eq!(strings.scalar_at(2).unwrap(), "c".into()); -//! assert_eq!(strings.scalar_at(3).unwrap(), "d".into()); +//! assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into()); +//! assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into()); +//! assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into()); +//! assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into()); //! ``` use std::any::Any; @@ -223,6 +225,7 @@ pub trait ArrayBuilder: Send { /// ``` /// use vortex_array::builders::{builder_with_capacity, ArrayBuilder}; /// use vortex_array::dtype::{DType, Nullability}; +/// use vortex_array::{LEGACY_SESSION, VortexSessionExecute}; /// /// // Create a new builder for string data. /// let mut builder = builder_with_capacity(&DType::Utf8(Nullability::NonNullable), 4); @@ -233,11 +236,12 @@ pub trait ArrayBuilder: Send { /// builder.append_scalar(&"d".into()).unwrap(); /// /// let strings = builder.finish(); +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); /// -/// assert_eq!(strings.scalar_at(0).unwrap(), "a".into()); -/// assert_eq!(strings.scalar_at(1).unwrap(), "b".into()); -/// assert_eq!(strings.scalar_at(2).unwrap(), "c".into()); -/// assert_eq!(strings.scalar_at(3).unwrap(), "d".into()); +/// assert_eq!(strings.execute_scalar(0, &mut ctx).unwrap(), "a".into()); +/// assert_eq!(strings.execute_scalar(1, &mut ctx).unwrap(), "b".into()); +/// assert_eq!(strings.execute_scalar(2, &mut ctx).unwrap(), "c".into()); +/// assert_eq!(strings.execute_scalar(3, &mut ctx).unwrap(), "d".into()); /// ``` pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box { match dtype { diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index 6952db2cfd2..93c50b130ca 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -438,9 +438,24 @@ mod tests { let array = builder.finish_into_primitive(); assert_eq!(array.len(), 3); // Check validity using scalar_at - nulls will return is_null() = true. - assert!(!array.scalar_at(0).unwrap().is_null()); - assert!(array.scalar_at(1).unwrap().is_null()); - assert!(!array.scalar_at(2).unwrap().is_null()); + assert!( + !array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + !array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); } /// REGRESSION TEST: This test verifies that `append_mask` validates the mask length. @@ -528,13 +543,38 @@ mod tests { assert_eq!(array.as_slice::(), &[100, 200, 10, 20, 30]); // Check validity - the first two should be valid (from append_value). - assert!(!array.scalar_at(0).unwrap().is_null()); // initial value 100 - assert!(!array.scalar_at(1).unwrap().is_null()); // initial value 200 + assert!( + !array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // initial value 100 + assert!( + !array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // initial value 200 // Check the range items with modified validity. - assert!(!array.scalar_at(2).unwrap().is_null()); // range index 0 - set to valid - assert!(array.scalar_at(3).unwrap().is_null()); // range index 1 - left as null - assert!(!array.scalar_at(4).unwrap().is_null()); // range index 2 - set to valid + assert!( + !array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // range index 0 - set to valid + assert!( + array + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // range index 1 - left as null + assert!( + !array + .execute_scalar(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // range index 2 - set to valid } /// Test that creating a zero-length uninit range panics. diff --git a/vortex-array/src/builders/tests.rs b/vortex-array/src/builders/tests.rs index b93a4ad2c30..749b5abdad2 100644 --- a/vortex-array/src/builders/tests.rs +++ b/vortex-array/src/builders/tests.rs @@ -6,6 +6,8 @@ use std::sync::Arc; use rstest::rstest; use vortex_error::VortexExpect; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::builders::ArrayBuilder; use crate::builders::builder_with_capacity; use crate::dtype::DType; @@ -90,8 +92,12 @@ fn test_append_zeros_matches_default_value(#[case] dtype: DType) { // Compare each element. for i in 0..num_elements { - let scalar_zeros = array_zeros.scalar_at(i).unwrap(); - let scalar_manual = array_manual.scalar_at(i).unwrap(); + let scalar_zeros = array_zeros + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); + let scalar_manual = array_manual + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( scalar_zeros, scalar_manual, @@ -188,7 +194,9 @@ fn test_append_defaults_behavior(#[case] dtype: DType, #[case] should_be_null: b assert_eq!(array.len(), 3); for i in 0..3 { - let scalar = array.scalar_at(i).unwrap(); + let scalar = array + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); if should_be_null { assert!(scalar.is_null(), "Element at index {} should be null", i); } else { @@ -243,8 +251,12 @@ where // Compare each element. for i in 0..array_direct.len() { - let scalar_direct = array_direct.scalar_at(i).unwrap(); - let scalar_indirect = array_indirect.scalar_at(i).unwrap(); + let scalar_direct = array_direct + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); + let scalar_indirect = array_indirect + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( scalar_direct, scalar_indirect, @@ -533,13 +545,17 @@ fn test_append_scalar_comprehensive(#[case] dtype: DType) { // Verify each scalar matches. for (i, expected_scalar) in scalars.iter().enumerate() { - let actual_scalar = array.scalar_at(i).unwrap(); + let actual_scalar = array + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_scalars_equal(&actual_scalar, expected_scalar, &dtype, i); } // If nullable, verify the last element is null. if dtype.is_nullable() { - let null_scalar = array.scalar_at(num_elements).unwrap(); + let null_scalar = array + .execute_scalar(num_elements, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!( null_scalar.is_null(), "Last element should be null for nullable dtype" @@ -685,16 +701,62 @@ fn test_append_scalar_mixed_nulls(#[case] dtype: DType) { assert_eq!(array.len(), 5); // Check the pattern. - assert!(!array.scalar_at(0).unwrap().is_null()); - assert!(array.scalar_at(1).unwrap().is_null()); - assert!(!array.scalar_at(2).unwrap().is_null()); - assert!(array.scalar_at(3).unwrap().is_null()); - assert!(!array.scalar_at(4).unwrap().is_null()); + assert!( + !array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + array + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + !array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + array + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); + assert!( + !array + .execute_scalar(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_null() + ); // Verify non-null values match. - assert_scalars_equal(&array.scalar_at(0).unwrap(), &test_scalars[0], &dtype, 0); - assert_scalars_equal(&array.scalar_at(2).unwrap(), &test_scalars[1], &dtype, 2); - assert_scalars_equal(&array.scalar_at(4).unwrap(), &test_scalars[2], &dtype, 4); + assert_scalars_equal( + &array + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + &test_scalars[0], + &dtype, + 0, + ); + assert_scalars_equal( + &array + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + &test_scalars[1], + &dtype, + 2, + ); + assert_scalars_equal( + &array + .execute_scalar(4, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + &test_scalars[2], + &dtype, + 4, + ); } /// Test that `append_scalar` correctly rejects scalars with wrong dtype. @@ -745,7 +807,9 @@ fn test_append_scalar_repeated_same_instance() { // All values should be 42. for i in 0..5 { - let actual = array.scalar_at(i).unwrap(); + let actual = array + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!( actual.as_primitive().typed_value::(), Some(42), diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 6d802696c74..c1e5fd6f6d6 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -1058,7 +1058,7 @@ mod tests { // Verify the value was stored correctly let retrieved = array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_binary() .value() diff --git a/vortex-array/src/compute/conformance/binary_numeric.rs b/vortex-array/src/compute/conformance/binary_numeric.rs index c36367746e7..5f5588ec273 100644 --- a/vortex-array/src/compute/conformance/binary_numeric.rs +++ b/vortex-array/src/compute/conformance/binary_numeric.rs @@ -47,7 +47,7 @@ fn to_vec_of_scalar(array: &ArrayRef) -> Vec { (0..array.len()) .map(|index| { array - .scalar_at(index) + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") }) .collect_vec() diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index b46a14d634b..01b7dc67b2a 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -74,10 +74,10 @@ fn test_cast_identity(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -109,7 +109,7 @@ fn test_cast_from_null(array: &ArrayRef) { for i in 0..array.len().min(10) { assert!( result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .is_null() ); @@ -141,10 +141,10 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), non_nullable - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -157,10 +157,10 @@ fn test_cast_to_non_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), back_to_nullable - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -190,10 +190,10 @@ fn test_cast_to_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), nullable - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -206,9 +206,9 @@ fn test_cast_to_nullable(array: &ArrayRef) { for i in 0..array.len().min(10) { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), - back.scalar_at(i) + back.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -298,10 +298,10 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip ); for i in 0..array.len().min(10) { let original = array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let casted = casted - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( original diff --git a/vortex-array/src/compute/conformance/consistency.rs b/vortex-array/src/compute/conformance/consistency.rs index 9579454d24e..1817588c1f2 100644 --- a/vortex-array/src/compute/conformance/consistency.rs +++ b/vortex-array/src/compute/conformance/consistency.rs @@ -89,10 +89,10 @@ fn test_filter_take_consistency(array: &ArrayRef) { for i in 0..filtered.len() { let filtered_val = filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let taken_val = taken - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( filtered_val, taken_val, @@ -163,10 +163,10 @@ fn test_double_mask_consistency(array: &ArrayRef) { for i in 0..double_masked.len() { let double_val = double_masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let direct_val = directly_masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( double_val, direct_val, @@ -213,10 +213,10 @@ fn test_filter_identity(array: &ArrayRef) { for i in 0..len { let original_val = array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let filtered_val = filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( filtered_val, original_val, @@ -269,10 +269,10 @@ fn test_mask_identity(array: &ArrayRef) { for i in 0..len { let original_val = array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let masked_val = masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let expected_val = original_val.clone().into_nullable(); assert_eq!( @@ -328,10 +328,10 @@ fn test_slice_filter_consistency(array: &ArrayRef) { for i in 0..filtered.len() { let filtered_val = filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let sliced_val = sliced - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( filtered_val, sliced_val, @@ -383,10 +383,10 @@ fn test_take_slice_consistency(array: &ArrayRef) { for i in 0..taken.len() { let taken_val = taken - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let sliced_val = sliced - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( taken_val, sliced_val, @@ -416,26 +416,26 @@ fn test_filter_preserves_order(array: &ArrayRef) { if len >= 4 { assert_eq!( filtered - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( filtered - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( filtered - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(3) + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -458,10 +458,10 @@ fn test_take_repeated_indices(array: &ArrayRef) { for i in 0..3 { assert_eq!( taken - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -498,10 +498,10 @@ fn test_mask_filter_null_consistency(array: &ArrayRef) { for i in 0..filtered.len() { assert_eq!( filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), direct_filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -555,10 +555,10 @@ fn test_take_preserves_properties(array: &ArrayRef) { for i in 0..len { assert_eq!( taken - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -610,11 +610,11 @@ fn test_nullable_indices_consistency(array: &ArrayRef) { // Check first element (from index 0) let expected_0 = array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable(); let actual_0 = taken - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( actual_0, expected_0, @@ -624,7 +624,7 @@ fn test_nullable_indices_consistency(array: &ArrayRef) { // Check second element (should be null) let actual_1 = taken - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert!( actual_1.is_null(), @@ -633,11 +633,11 @@ fn test_nullable_indices_consistency(array: &ArrayRef) { // Check third element (from index 2) let expected_2 = array - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable(); let actual_2 = taken - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( actual_2, expected_2, @@ -672,10 +672,10 @@ fn test_large_array_consistency(array: &ArrayRef) { for i in 0..taken.len() { assert_eq!( taken - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), filtered - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -714,7 +714,7 @@ fn test_comparison_inverse_consistency(array: &ArrayRef) { return; } else { array - .scalar_at(len / 2) + .execute_scalar(len / 2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") }; @@ -740,10 +740,10 @@ fn test_comparison_inverse_consistency(array: &ArrayRef) { for i in 0..inverted_eq.len() { let inv_val = inverted_eq - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let neq_val = neq_result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( inv_val, neq_val, @@ -768,10 +768,10 @@ fn test_comparison_inverse_consistency(array: &ArrayRef) { for i in 0..inverted_gt.len() { let inv_val = inverted_gt - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let lte_val = lte_result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( inv_val, lte_val, @@ -796,10 +796,10 @@ fn test_comparison_inverse_consistency(array: &ArrayRef) { for i in 0..inverted_lt.len() { let inv_val = inverted_lt - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let gte_val = gte_result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( inv_val, gte_val, @@ -842,7 +842,7 @@ fn test_comparison_symmetry_consistency(array: &ArrayRef) { return; } else { array - .scalar_at(len / 2) + .execute_scalar(len / 2, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") }; @@ -867,10 +867,10 @@ fn test_comparison_symmetry_consistency(array: &ArrayRef) { for i in 0..arr_gt_scalar.len() { let arr_gt = arr_gt_scalar - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let scalar_lt = scalar_lt_arr - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( arr_gt, scalar_lt, @@ -889,10 +889,10 @@ fn test_comparison_symmetry_consistency(array: &ArrayRef) { ) { for i in 0..arr_eq_scalar.len() { let arr_eq = arr_eq_scalar - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let scalar_eq = scalar_eq_arr - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( arr_eq, scalar_eq, @@ -951,10 +951,10 @@ fn test_boolean_demorgan_consistency(array: &ArrayRef) { for i in 0..not_a_and_b.len() { let left = not_a_and_b - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let right = not_a_or_not_b - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( left, right, @@ -979,10 +979,10 @@ fn test_boolean_demorgan_consistency(array: &ArrayRef) { for i in 0..not_a_or_b.len() { let left = not_a_or_b - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let right = not_a_and_not_b - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( left, right, @@ -1274,14 +1274,14 @@ fn test_cast_slice_consistency(array: &ArrayRef) { // Compare each value against the canonical form for i in 0..slice_then_cast.len() { let slice_cast_val = slice_then_cast - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); // Get the corresponding value from the canonical array (adjusted for slice offset) let canonical_val = canonical .clone() .into_array() - .scalar_at(start + i) + .execute_scalar(start + i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); // Cast the canonical scalar to the target dtype @@ -1327,10 +1327,10 @@ fn test_cast_slice_consistency(array: &ArrayRef) { for i in 0..slice_then_cast.len() { let slice_cast_val = slice_then_cast - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let cast_slice_val = cast_then_slice - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!( slice_cast_val, cast_slice_val, diff --git a/vortex-array/src/compute/conformance/filter.rs b/vortex-array/src/compute/conformance/filter.rs index 6be08dff69a..7d5f4abcad8 100644 --- a/vortex-array/src/compute/conformance/filter.rs +++ b/vortex-array/src/compute/conformance/filter.rs @@ -6,6 +6,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::assert_arrays_eq; use crate::dtype::DType; @@ -98,10 +100,10 @@ fn test_selective_filter(array: &ArrayRef) { for (filtered_idx, i) in (0..len).step_by(2).enumerate() { assert_eq!( filtered - .scalar_at(filtered_idx) + .execute_scalar(filtered_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -118,18 +120,18 @@ fn test_selective_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 2); assert_eq!( filtered - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( filtered - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(len - 1) + .execute_scalar(len - 1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -151,10 +153,10 @@ fn test_single_element_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 1); assert_eq!( filtered - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); @@ -169,10 +171,10 @@ fn test_single_element_filter(array: &ArrayRef) { assert_eq!(filtered.len(), 1); assert_eq!( filtered - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(len - 1) + .execute_scalar(len - 1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -235,10 +237,10 @@ fn test_alternating_pattern_filter(array: &ArrayRef) { if keep { assert_eq!( filtered - .scalar_at(filtered_idx) + .execute_scalar(filtered_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); filtered_idx += 1; diff --git a/vortex-array/src/compute/conformance/mask.rs b/vortex-array/src/compute/conformance/mask.rs index 5c448db126f..42d7bb9c494 100644 --- a/vortex-array/src/compute/conformance/mask.rs +++ b/vortex-array/src/compute/conformance/mask.rs @@ -54,16 +54,16 @@ fn test_heterogenous_mask(array: &ArrayRef) { if masked_out { assert!( !masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -87,10 +87,10 @@ fn test_empty_mask(array: &ArrayRef) { for i in 0..len { assert_eq!( masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -113,7 +113,7 @@ fn test_full_mask(array: &ArrayRef) { for i in 0..len { assert!( !masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); } @@ -135,16 +135,16 @@ fn test_alternating_mask(array: &ArrayRef) { if i % 2 == 0 { assert!( !masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -173,7 +173,7 @@ fn test_sparse_mask(array: &ArrayRef) { let valid_count = (0..len) .filter(|&i| { masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") }) .count(); @@ -185,7 +185,7 @@ fn test_sparse_mask(array: &ArrayRef) { .filter(|&i| { pattern[i] || !array - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") }) .count(); @@ -208,17 +208,17 @@ fn test_single_element_mask(array: &ArrayRef) { .vortex_expect("mask should succeed in conformance test"); assert!( !masked - .is_valid(0) + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); for i in 1..len { assert_eq!( masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -249,16 +249,16 @@ fn test_double_mask(array: &ArrayRef) { if mask1_pattern[i] || mask2_pattern[i] { assert!( !double_masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( double_masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); @@ -293,16 +293,16 @@ fn test_nullable_mask_input(array: &ArrayRef) { if bool_values[i] && validity_values[i] { assert!( !masked - .is_valid(i) + .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("is_valid should succeed in conformance test") ); } else { assert_eq!( masked - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .into_nullable() ); diff --git a/vortex-array/src/compute/conformance/take.rs b/vortex-array/src/compute/conformance/take.rs index 9fec410ac35..aa41da6942a 100644 --- a/vortex-array/src/compute/conformance/take.rs +++ b/vortex-array/src/compute/conformance/take.rs @@ -7,6 +7,8 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray as _; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; @@ -81,10 +83,10 @@ fn test_take_all(array: &ArrayRef) { for i in 0..len { assert_eq!( array - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -120,10 +122,13 @@ fn test_take_selective(array: &ArrayRef) { for (result_idx, &original_idx) in indices.iter().enumerate() { assert_eq!( array - .scalar_at(original_idx as usize) + .execute_scalar( + original_idx as usize, + &mut LEGACY_SESSION.create_execution_ctx() + ) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(result_idx) + .execute_scalar(result_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -139,18 +144,18 @@ fn test_take_first_and_last(array: &ArrayRef) { assert_eq!(result.len(), 2); assert_eq!( array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); assert_eq!( array - .scalar_at(len - 1) + .execute_scalar(len - 1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(1) + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -184,17 +189,17 @@ fn test_take_with_nullable_indices(array: &ArrayRef) { match idx_opt { Some(idx) => { let expected = array - .scalar_at(*idx as usize) + .execute_scalar(*idx as usize, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); let actual = result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); assert_eq!(expected, actual); } None => { assert!( result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") .is_null() ); @@ -216,12 +221,12 @@ fn test_take_repeated_indices(array: &ArrayRef) { assert_eq!(result.len(), 3); let first_elem = array - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"); for i in 0..3 { assert_eq!( result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), first_elem ); @@ -252,10 +257,10 @@ fn test_take_reverse(array: &ArrayRef) { for i in 0..len { assert_eq!( array - .scalar_at(len - 1 - i) + .execute_scalar(len - 1 - i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -273,10 +278,10 @@ fn test_take_single_middle(array: &ArrayRef) { assert_eq!(result.len(), 1); assert_eq!( array - .scalar_at(middle_idx) + .execute_scalar(middle_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -304,10 +309,10 @@ fn test_take_random_unsorted(array: &ArrayRef) { for (i, &idx) in indices.iter().enumerate() { assert_eq!( array - .scalar_at(idx as usize) + .execute_scalar(idx as usize, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -330,10 +335,10 @@ fn test_take_contiguous_range(array: &ArrayRef) { for i in 0..(end - start) { assert_eq!( array - .scalar_at(start + i) + .execute_scalar(start + i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -366,10 +371,10 @@ fn test_take_mixed_repeated(array: &ArrayRef) { for (i, &idx) in indices.iter().enumerate() { assert_eq!( array - .scalar_at(idx as usize) + .execute_scalar(idx as usize, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } @@ -398,10 +403,10 @@ fn test_take_large_indices(array: &ArrayRef) { let expected_idx = indices[i] as usize; assert_eq!( array - .scalar_at(expected_idx) + .execute_scalar(expected_idx, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test"), result - .scalar_at(i) + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at should succeed in conformance test") ); } diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index f299bb7aeeb..861d24e3538 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -19,6 +19,8 @@ use itertools::Itertools as _; pub use tree_display::TreeDisplay; use crate::ArrayRef; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; /// Describe how to convert an array to a string. /// @@ -534,7 +536,7 @@ impl ArrayRef { let is_truncated = self.len() > limit; let fmt_scalar = |i| { - self.scalar_at(i) + self.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) .map_or_else(|e| format!(""), |s| s.to_string()) }; write!( @@ -587,7 +589,7 @@ impl ArrayRef { // For non-struct arrays, simply display a single column table without header. for row_idx in 0..self.len() { let value = self - .scalar_at(row_idx) + .execute_scalar(row_idx, &mut LEGACY_SESSION.create_execution_ctx()) .map_or_else(|e| format!(""), |s| s.to_string()); builder.push_record([value]); } @@ -602,7 +604,10 @@ impl ArrayRef { builder.push_record(sf.names().iter().map(|name| name.to_string())); for row_idx in 0..self.len() { - if !self.is_valid(row_idx).unwrap_or(false) { + if !self + .is_valid(row_idx, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap_or(false) + { let null_row = vec!["null".to_string(); sf.names().len()]; builder.push_record(null_row); } else { @@ -611,7 +616,7 @@ impl ArrayRef { crate::arrays::struct_::StructArrayExt::iter_unmasked_fields(&struct_) { let value = field_array - .scalar_at(row_idx) + .execute_scalar(row_idx, &mut LEGACY_SESSION.create_execution_ctx()) .map_or_else(|e| format!(""), |s| s.to_string()); row.push(value); } @@ -628,7 +633,10 @@ impl ArrayRef { } for row_idx in 0..self.len() { - if !self.is_valid(row_idx).unwrap_or(false) { + if !self + .is_valid(row_idx, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap_or(false) + { table.modify( (1 + row_idx, 0), tabled::settings::Span::column(sf.names().len() as isize), diff --git a/vortex-array/src/expr/stats/mod.rs b/vortex-array/src/expr/stats/mod.rs index 5f998041dea..53f50bbb9e0 100644 --- a/vortex-array/src/expr/stats/mod.rs +++ b/vortex-array/src/expr/stats/mod.rs @@ -213,6 +213,8 @@ impl Display for Stat { mod test { use enum_iterator::all; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::expr::stats::Stat; @@ -220,7 +222,7 @@ mod test { fn min_of_nulls_is_not_panic() { let min = PrimitiveArray::from_option_iter::([None, None, None, None]) .statistics() - .compute_as::(Stat::Min); + .compute_as::(Stat::Min, &mut LEGACY_SESSION.create_execution_ctx()); assert_eq!(min, None); } diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 1d37d82fb75..000ea495815 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -21,7 +21,9 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -172,8 +174,11 @@ impl Patches { // Perform validation of components when they are host-resident. // This is not possible to do eagerly when the data is on GPU memory. if indices.is_host() && values.is_host() { - let max = usize::try_from(&indices.scalar_at(indices.len() - 1)?) - .map_err(|_| vortex_err!("indices must be a number"))?; + let max = usize::try_from(&indices.execute_scalar( + indices.len() - 1, + &mut LEGACY_SESSION.create_execution_ctx(), + )?) + .map_err(|_| vortex_err!("indices must be a number"))?; vortex_ensure!( max - offset < array_len, "Patch indices {max:?}, offset {offset} are longer than the array length {array_len}" @@ -182,7 +187,7 @@ impl Patches { #[cfg(debug_assertions)] { use crate::VortexSessionExecute; - let mut ctx = crate::LEGACY_SESSION.create_execution_ctx(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert!( crate::aggregate_fn::fns::is_sorted::is_sorted(&indices, &mut ctx) .unwrap_or(false), @@ -292,7 +297,7 @@ impl Patches { }; chunk_offsets - .scalar_at(idx)? + .execute_scalar(idx, &mut LEGACY_SESSION.create_execution_ctx())? .as_primitive() .as_::() .ok_or_else(|| vortex_err!("chunk offset does not fit in usize")) @@ -363,7 +368,10 @@ impl Patches { pub fn get_patched(&self, index: usize) -> VortexResult> { self.search_index(index)? .to_found() - .map(|patch_idx| self.values().scalar_at(patch_idx)) + .map(|patch_idx| { + self.values() + .execute_scalar(patch_idx, &mut LEGACY_SESSION.create_execution_ctx()) + }) .transpose() } @@ -545,7 +553,7 @@ impl Patches { pub fn min_index(&self) -> VortexResult { let first = self .indices - .scalar_at(0)? + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? .as_primitive() .as_::() .ok_or_else(|| vortex_err!("index does not fit in usize"))?; @@ -556,7 +564,10 @@ impl Patches { pub fn max_index(&self) -> VortexResult { let last = self .indices - .scalar_at(self.indices.len() - 1)? + .execute_scalar( + self.indices.len() - 1, + &mut LEGACY_SESSION.create_execution_ctx(), + )? .as_primitive() .as_::() .ok_or_else(|| vortex_err!("index does not fit in usize"))?; @@ -665,7 +676,7 @@ impl Patches { .as_ref() .map(|new_chunk_offsets| -> VortexResult { let new_chunk_base = new_chunk_offsets - .scalar_at(0)? + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())? .as_primitive() .as_::() .ok_or_else(|| vortex_err!("chunk offset does not fit in usize"))?; @@ -1387,9 +1398,24 @@ mod test { masked.values(), PrimitiveArray::from_iter([100i32, 200, 300]) ); - assert!(masked.values().is_valid(0).unwrap()); - assert!(masked.values().is_valid(1).unwrap()); - assert!(masked.values().is_valid(2).unwrap()); + assert!( + masked + .values() + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + masked + .values() + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); + assert!( + masked + .values() + .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // Indices should remain unchanged assert_arrays_eq!(masked.indices(), PrimitiveArray::from_iter([2u64, 5, 8])); @@ -1475,10 +1501,23 @@ mod test { // Values should be the null and 300 let masked_values = masked.values().to_primitive(); assert_eq!(masked_values.len(), 2); - assert!(!masked_values.is_valid(0).unwrap()); // the null value at index 5 - assert!(masked_values.is_valid(1).unwrap()); // the 300 value at index 8 + assert!( + !masked_values + .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // the null value at index 5 + assert!( + masked_values + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); // the 300 value at index 8 assert_eq!( - i32::try_from(&masked_values.scalar_at(1).unwrap()).unwrap(), + i32::try_from( + &masked_values + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 300i32 ); } @@ -1637,15 +1676,30 @@ mod test { let values = patches.values().to_primitive(); assert_eq!( - i32::try_from(&values.scalar_at(0).unwrap()).unwrap(), + i32::try_from( + &values + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 100i32 ); assert_eq!( - i32::try_from(&values.scalar_at(1).unwrap()).unwrap(), + i32::try_from( + &values + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 200i32 ); assert_eq!( - i32::try_from(&values.scalar_at(2).unwrap()).unwrap(), + i32::try_from( + &values + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ) + .unwrap(), 300i32 ); } diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index 9061b829b7c..0cf591824e5 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -106,16 +106,6 @@ pub(super) fn precondition( return Ok(Some(Canonical::empty(&return_dtype).into_array())); } - // A quick check to see if either bound is a null constant array. - if (lower.is_invalid(0)? || upper.is_invalid(0)?) - && let (Some(c_lower), Some(c_upper)) = (lower.as_constant(), upper.as_constant()) - && (c_lower.is_null() || c_upper.is_null()) - { - return Ok(Some( - ConstantArray::new(Scalar::null(return_dtype), arr.len()).into_array(), - )); - } - if lower.as_constant().is_some_and(|v| v.is_null()) || upper.as_constant().is_some_and(|v| v.is_null()) { diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/fns/binary/boolean.rs index 57face819e8..b6894fd2887 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/fns/binary/boolean.rs @@ -106,6 +106,8 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::builtins::ArrayBuiltins; use crate::canonical::ToCanonical; @@ -124,10 +126,26 @@ mod tests { let r = lhs.binary(rhs, Operator::Or).unwrap(); let r = r.to_bool().into_array(); - let v0 = r.scalar_at(0).unwrap().as_bool().value(); - let v1 = r.scalar_at(1).unwrap().as_bool().value(); - let v2 = r.scalar_at(2).unwrap().as_bool().value(); - let v3 = r.scalar_at(3).unwrap().as_bool().value(); + let v0 = r + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v1 = r + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v2 = r + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v3 = r + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); assert!(v0.unwrap()); assert!(v1.unwrap()); @@ -151,10 +169,26 @@ mod tests { .to_bool() .into_array(); - let v0 = r.scalar_at(0).unwrap().as_bool().value(); - let v1 = r.scalar_at(1).unwrap().as_bool().value(); - let v2 = r.scalar_at(2).unwrap().as_bool().value(); - let v3 = r.scalar_at(3).unwrap().as_bool().value(); + let v0 = r + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v1 = r + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v2 = r + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); + let v3 = r + .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_bool() + .value(); assert!(v0.unwrap()); assert!(!v1.unwrap()); diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 0108aac6509..6447e0c1638 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -249,7 +249,9 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ListArray; use crate::arrays::ListViewArray; @@ -348,7 +350,9 @@ mod tests { .binary(right.into_array(), Operator::Gt) .unwrap(); assert_eq!(result.len(), 10); - let scalar = result.scalar_at(0).unwrap(); + let scalar = result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert_eq!(scalar.as_bool().value(), Some(false)); } @@ -540,8 +544,23 @@ mod tests { .into_array() .binary(list.into_array(), Operator::Eq) .unwrap(); - assert!(result.scalar_at(0).unwrap().is_valid()); - assert!(result.scalar_at(1).unwrap().is_valid()); - assert!(result.scalar_at(2).unwrap().is_valid()); + assert!( + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); + assert!( + result + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); + assert!( + result + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .is_valid() + ); } } diff --git a/vortex-array/src/scalar_fn/fns/binary/mod.rs b/vortex-array/src/scalar_fn/fns/binary/mod.rs index 1dce67e6679..af81ef0ca28 100644 --- a/vortex-array/src/scalar_fn/fns/binary/mod.rs +++ b/vortex-array/src/scalar_fn/fns/binary/mod.rs @@ -314,6 +314,8 @@ mod tests { use vortex_error::VortexExpect; use super::*; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -494,7 +496,9 @@ mod tests { // Test using binary method directly let result_equal = lhs_struct.binary(rhs_struct_equal, Operator::Eq).unwrap(); assert_eq!( - result_equal.scalar_at(0).vortex_expect("value"), + result_equal + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("value"), Scalar::bool(true, Nullability::NonNullable), "Equal structs should be equal" ); @@ -503,7 +507,9 @@ mod tests { .binary(rhs_struct_different, Operator::Eq) .unwrap(); assert_eq!( - result_different.scalar_at(0).vortex_expect("value"), + result_different + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("value"), Scalar::bool(false, Nullability::NonNullable), "Different structs should not be equal" ); diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index 539614874a9..4424bdd6c2a 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -246,7 +246,7 @@ impl ScalarFnVTable for CaseWhen { return Ok(else_value); } - merge_case_branches(branches, else_value) + merge_case_branches(branches, else_value, ctx) } fn is_null_sensitive(&self, _options: &Self::Options) -> bool { @@ -268,6 +268,7 @@ const SLICE_CROSSOVER_RUN_LEN: usize = 4; fn merge_case_branches( branches: Vec<(Mask, ArrayRef)>, else_value: ArrayRef, + ctx: &mut ExecutionCtx, ) -> VortexResult { if branches.len() == 1 { let (mask, then_value) = &branches[0]; @@ -304,7 +305,14 @@ fn merge_case_branches( let fragmented = spans.len() > else_value.len() / SLICE_CROSSOVER_RUN_LEN; if fragmented { - merge_row_by_row(&branch_arrays, &else_value, &spans, &output_dtype, builder) + merge_row_by_row( + &branch_arrays, + &else_value, + &spans, + &output_dtype, + builder, + ctx, + ) } else { merge_run_by_run(&branch_arrays, &else_value, &spans, &output_dtype, builder) } @@ -318,21 +326,22 @@ fn merge_row_by_row( spans: &[(usize, usize, usize)], output_dtype: &DType, mut builder: Box, + ctx: &mut ExecutionCtx, ) -> VortexResult { let mut pos = 0; for &(start, end, branch_idx) in spans { for row in pos..start { - let scalar = else_value.scalar_at(row)?; + let scalar = else_value.execute_scalar(row, ctx)?; builder.append_scalar(&scalar.cast(output_dtype)?)?; } for row in start..end { - let scalar = branch_arrays[branch_idx].scalar_at(row)?; + let scalar = branch_arrays[branch_idx].execute_scalar(row, ctx)?; builder.append_scalar(&scalar.cast(output_dtype)?)?; } pos = end; } for row in pos..else_value.len() { - let scalar = else_value.scalar_at(row)?; + let scalar = else_value.execute_scalar(row, ctx)?; builder.append_scalar(&scalar.cast(output_dtype)?)?; } @@ -379,7 +388,8 @@ mod tests { use super::*; use crate::Canonical; use crate::IntoArray; - use crate::VortexSessionExecute as _; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; @@ -1124,19 +1134,19 @@ mod tests { let result = evaluate_expr(&expr, &test_array); assert_eq!( - result.scalar_at(0)?, + result.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, Scalar::utf8("low", Nullability::NonNullable) ); assert_eq!( - result.scalar_at(1)?, + result.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx())?, Scalar::utf8("low", Nullability::NonNullable) ); assert_eq!( - result.scalar_at(2)?, + result.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())?, Scalar::utf8("high", Nullability::NonNullable) ); assert_eq!( - result.scalar_at(3)?, + result.execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx())?, Scalar::utf8("high", Nullability::NonNullable) ); Ok(()) @@ -1194,6 +1204,7 @@ mod tests { ), ], PrimitiveArray::from_option_iter(vec![Some(99i32); n]).into_array(), + &mut SESSION.create_execution_ctx(), )?; // Even rows → 0, odd rows → 1. diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index dd2a6c9b16b..f5e8df294dd 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -21,6 +21,7 @@ use crate::kernel::ExecuteParentKernel; use crate::optimizer::rules::ArrayParentReduceRule; use crate::scalar::Scalar; use crate::scalar_fn::fns::fill_null::FillNull as FillNullExpr; +use crate::validity::Validity; /// Fill nulls in an array with a scalar value without reading buffers. /// @@ -68,12 +69,17 @@ pub(super) fn precondition( ); // If the array has no nulls, fill_null is a no-op (just cast for nullability). - if !array.dtype().is_nullable() || array.all_valid()? { + if !array.dtype().is_nullable() + || matches!( + array.validity()?, + Validity::NonNullable | Validity::AllValid + ) + { return array.clone().cast(fill_value.dtype().clone()).map(Some); } // If all values are null, replace the entire array with the fill value. - if array.all_invalid()? { + if matches!(array.validity()?, Validity::AllInvalid) { return Ok(Some( ConstantArray::new(fill_value.clone(), array.len()).into_array(), )); diff --git a/vortex-array/src/scalar_fn/fns/is_not_null.rs b/vortex-array/src/scalar_fn/fns/is_not_null.rs index b008ebb5835..f5449a20cad 100644 --- a/vortex-array/src/scalar_fn/fns/is_not_null.rs +++ b/vortex-array/src/scalar_fn/fns/is_not_null.rs @@ -129,6 +129,8 @@ mod tests { use vortex_error::VortexExpect as _; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::dtype::DType; @@ -169,7 +171,9 @@ mod tests { for (i, expected_value) in expected.iter().enumerate() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(*expected_value, Nullability::NonNullable) ); } @@ -184,7 +188,9 @@ mod tests { assert_eq!(result.len(), test_array.len()); for i in 0..result.len() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::NonNullable) ); } @@ -201,7 +207,9 @@ mod tests { assert_eq!(result.len(), test_array.len()); for i in 0..result.len() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::NonNullable) ); } @@ -228,7 +236,9 @@ mod tests { for (i, expected_value) in expected.iter().enumerate() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(*expected_value, Nullability::NonNullable) ); } diff --git a/vortex-array/src/scalar_fn/fns/is_null.rs b/vortex-array/src/scalar_fn/fns/is_null.rs index dd3ef3366c6..2d801cd6a0d 100644 --- a/vortex-array/src/scalar_fn/fns/is_null.rs +++ b/vortex-array/src/scalar_fn/fns/is_null.rs @@ -122,6 +122,8 @@ mod tests { use vortex_utils::aliases::hash_set::HashSet; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::dtype::DType; @@ -170,7 +172,9 @@ mod tests { for (i, expected_value) in expected.iter().enumerate() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(*expected_value, Nullability::NonNullable) ); } @@ -186,7 +190,9 @@ mod tests { // All values should be false (non-nullable input) for i in 0..result.len() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::NonNullable) ); } @@ -204,7 +210,9 @@ mod tests { // All values should be true (all nulls) for i in 0..result.len() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::NonNullable) ); } @@ -231,7 +239,9 @@ mod tests { for (i, expected_value) in expected.iter().enumerate() { assert_eq!( - result.scalar_at(i).unwrap(), + result + .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(*expected_value, Nullability::NonNullable) ); } diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 331f57874f0..27d60663c1d 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -219,7 +219,7 @@ fn compute_list_contains( ); } - if value.all_invalid()? || array.all_invalid()? { + if value.all_invalid(ctx)? || array.all_invalid(ctx)? { return Ok(ConstantArray::new( Scalar::null(DType::Bool(Nullability::Nullable)), array.len(), @@ -280,7 +280,7 @@ fn list_contains_scalar( // If the list array is constant, we perform a single comparison. if array.len() > 1 && array.is::() { let contains = list_contains_scalar(&array.slice(0..1)?, value, nullability, ctx)?; - return Ok(ConstantArray::new(contains.scalar_at(0)?, array.len()).into_array()); + return Ok(ConstantArray::new(contains.execute_scalar(0, ctx)?, array.len()).into_array()); } let list_array = array.clone().execute::(ctx)?; @@ -456,6 +456,8 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ListArray; use crate::arrays::VarBinArray; use crate::assert_arrays_eq; @@ -504,11 +506,13 @@ mod tests { let item = arr.apply(&expr).unwrap(); assert_eq!( - item.scalar_at(0).unwrap(), + item.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::Nullable) ); assert_eq!( - item.scalar_at(1).unwrap(), + item.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::Nullable) ); } @@ -521,11 +525,13 @@ mod tests { let item = arr.apply(&expr).unwrap(); assert_eq!( - item.scalar_at(0).unwrap(), + item.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::Nullable) ); assert_eq!( - item.scalar_at(1).unwrap(), + item.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::Nullable) ); } @@ -538,11 +544,13 @@ mod tests { let item = arr.apply(&expr).unwrap(); assert_eq!( - item.scalar_at(0).unwrap(), + item.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::Nullable) ); assert_eq!( - item.scalar_at(1).unwrap(), + item.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::Nullable) ); } @@ -561,11 +569,13 @@ mod tests { let item = arr.apply(&expr).unwrap(); assert_eq!( - item.scalar_at(0).unwrap(), + item.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::Nullable) ); assert_eq!( - item.scalar_at(1).unwrap(), + item.execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::Nullable) ); } @@ -584,10 +594,15 @@ mod tests { let item = arr.apply(&expr).unwrap(); assert_eq!( - item.scalar_at(0).unwrap(), + item.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::Nullable) ); - assert!(!item.is_valid(1).unwrap()); + assert!( + !item + .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + ); } #[test] @@ -676,7 +691,9 @@ mod tests { let expr = list_contains(lit(list_scalar.clone()), lit(2i32)); let result = arr.clone().apply(&expr).unwrap(); assert_eq!( - result.scalar_at(0).unwrap(), + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(true, Nullability::NonNullable) ); @@ -684,7 +701,9 @@ mod tests { let expr = list_contains(lit(list_scalar), lit(42i32)); let result = arr.apply(&expr).unwrap(); assert_eq!( - result.scalar_at(0).unwrap(), + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::bool(false, Nullability::NonNullable) ); } diff --git a/vortex-array/src/search_sorted.rs b/vortex-array/src/search_sorted.rs index cf096bbb8e2..14741652330 100644 --- a/vortex-array/src/search_sorted.rs +++ b/vortex-array/src/search_sorted.rs @@ -13,6 +13,8 @@ use std::hint; use vortex_error::VortexResult; use crate::ArrayRef; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::scalar::Scalar; #[derive(Debug, Copy, Clone, Eq, PartialEq)] @@ -264,7 +266,7 @@ fn search_sorted_side_idx VortexResult>( impl IndexOrd for ArrayRef { fn index_cmp(&self, idx: usize, elem: &Scalar) -> VortexResult> { - let scalar_a = self.scalar_at(idx)?; + let scalar_a = self.execute_scalar(idx, &mut LEGACY_SESSION.create_execution_ctx())?; Ok(scalar_a.partial_cmp(elem)) } diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index b9d7adc902b..248ca3587f2 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use parking_lot::RwLock; +use vortex_array::ExecutionCtx; use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_panic; @@ -15,8 +16,6 @@ use super::StatsSet; use super::StatsSetIntoIter; use super::TypedStatsSetRef; use crate::ArrayRef; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::aggregate_fn::fns::is_constant::is_constant; use crate::aggregate_fn::fns::is_sorted::is_sorted; use crate::aggregate_fn::fns::is_sorted::is_strict_sorted; @@ -154,45 +153,35 @@ impl StatsSetRef<'_> { f(&mut lock.iter()) } - pub fn compute_stat(&self, stat: Stat) -> VortexResult> { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - + pub fn compute_stat(&self, stat: Stat, ctx: &mut ExecutionCtx) -> VortexResult> { // If it's already computed and exact, we can return it. if let Some(Precision::Exact(s)) = self.get(stat) { return Ok(Some(s)); } Ok(match stat { - Stat::Min => { - min_max(self.dyn_array_ref, &mut ctx)?.map(|MinMaxResult { min, max: _ }| min) - } - Stat::Max => { - min_max(self.dyn_array_ref, &mut ctx)?.map(|MinMaxResult { min: _, max }| max) - } + Stat::Min => min_max(self.dyn_array_ref, ctx)?.map(|MinMaxResult { min, max: _ }| min), + Stat::Max => min_max(self.dyn_array_ref, ctx)?.map(|MinMaxResult { min: _, max }| max), Stat::Sum => { Stat::Sum .dtype(self.dyn_array_ref.dtype()) .is_some() .then(|| { // Sum is supported for this dtype. - sum(self.dyn_array_ref, &mut ctx) + sum(self.dyn_array_ref, ctx) }) .transpose()? } - Stat::NullCount => self - .dyn_array_ref - .invalid_count(&mut ctx) - .ok() - .map(Into::into), + Stat::NullCount => self.dyn_array_ref.invalid_count(ctx).ok().map(Into::into), Stat::IsConstant => { if self.dyn_array_ref.is_empty() { None } else { - Some(is_constant(self.dyn_array_ref, &mut ctx)?.into()) + Some(is_constant(self.dyn_array_ref, ctx)?.into()) } } - Stat::IsSorted => Some(is_sorted(self.dyn_array_ref, &mut ctx)?.into()), - Stat::IsStrictSorted => Some(is_strict_sorted(self.dyn_array_ref, &mut ctx)?.into()), + Stat::IsSorted => Some(is_sorted(self.dyn_array_ref, ctx)?.into()), + Stat::IsStrictSorted => Some(is_strict_sorted(self.dyn_array_ref, ctx)?.into()), Stat::UncompressedSizeInBytes => { let mut builder = builder_with_capacity(self.dyn_array_ref.dtype(), self.dyn_array_ref.len()); @@ -209,7 +198,7 @@ impl StatsSetRef<'_> { .is_some() .then(|| { // NaNCount is supported for this dtype. - nan_count(self.dyn_array_ref, &mut ctx) + nan_count(self.dyn_array_ref, ctx) }) .transpose()? .map(|s| s.into()) @@ -217,10 +206,10 @@ impl StatsSetRef<'_> { }) } - pub fn compute_all(&self, stats: &[Stat]) -> VortexResult { + pub fn compute_all(&self, stats: &[Stat], ctx: &mut ExecutionCtx) -> VortexResult { let mut stats_set = StatsSet::default(); for &stat in stats { - if let Some(s) = self.compute_stat(stat)? + if let Some(s) = self.compute_stat(stat, ctx)? && let Some(value) = s.into_value() { stats_set.set(stat, Precision::exact(value)); @@ -234,8 +223,9 @@ impl StatsSetRef<'_> { pub fn compute_as TryFrom<&'a Scalar, Error = VortexError>>( &self, stat: Stat, + ctx: &mut ExecutionCtx, ) -> Option { - self.compute_stat(stat) + self.compute_stat(stat, ctx) .inspect_err(|e| tracing::warn!("Failed to compute stat {stat}: {e}")) .ok() .flatten() @@ -259,32 +249,38 @@ impl StatsSetRef<'_> { self.array_stats.clear(stat); } - pub fn compute_min TryFrom<&'a Scalar, Error = VortexError>>(&self) -> Option { - self.compute_as(Stat::Min) + pub fn compute_min TryFrom<&'a Scalar, Error = VortexError>>( + &self, + ctx: &mut ExecutionCtx, + ) -> Option { + self.compute_as(Stat::Min, ctx) } - pub fn compute_max TryFrom<&'a Scalar, Error = VortexError>>(&self) -> Option { - self.compute_as(Stat::Max) + pub fn compute_max TryFrom<&'a Scalar, Error = VortexError>>( + &self, + ctx: &mut ExecutionCtx, + ) -> Option { + self.compute_as(Stat::Max, ctx) } - pub fn compute_is_sorted(&self) -> Option { - self.compute_as(Stat::IsSorted) + pub fn compute_is_sorted(&self, ctx: &mut ExecutionCtx) -> Option { + self.compute_as(Stat::IsSorted, ctx) } - pub fn compute_is_strict_sorted(&self) -> Option { - self.compute_as(Stat::IsStrictSorted) + pub fn compute_is_strict_sorted(&self, ctx: &mut ExecutionCtx) -> Option { + self.compute_as(Stat::IsStrictSorted, ctx) } - pub fn compute_is_constant(&self) -> Option { - self.compute_as(Stat::IsConstant) + pub fn compute_is_constant(&self, ctx: &mut ExecutionCtx) -> Option { + self.compute_as(Stat::IsConstant, ctx) } - pub fn compute_null_count(&self) -> Option { - self.compute_as(Stat::NullCount) + pub fn compute_null_count(&self, ctx: &mut ExecutionCtx) -> Option { + self.compute_as(Stat::NullCount, ctx) } - pub fn compute_uncompressed_size_in_bytes(&self) -> Option { - self.compute_as(Stat::UncompressedSizeInBytes) + pub fn compute_uncompressed_size_in_bytes(&self, ctx: &mut ExecutionCtx) -> Option { + self.compute_as(Stat::UncompressedSizeInBytes, ctx) } } diff --git a/vortex-array/src/stats/stats_set.rs b/vortex-array/src/stats/stats_set.rs index 5ddd0e908b1..9496720d47e 100644 --- a/vortex-array/src/stats/stats_set.rs +++ b/vortex-array/src/stats/stats_set.rs @@ -558,6 +558,8 @@ mod test { use enum_iterator::all; use itertools::Itertools; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::dtype::DType; use crate::dtype::Nullability; @@ -881,7 +883,10 @@ mod test { .filter(|s| !matches!(s, Stat::Sum)) .filter(|s| !matches!(s, Stat::NaNCount)) .collect_vec(); - array.statistics().compute_all(&all_stats).unwrap(); + array + .statistics() + .compute_all(&all_stats, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let stats = array.statistics().to_owned(); for stat in &all_stats { diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index f86256128fd..cb261a9b0b2 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -20,10 +20,10 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; -use crate::ToCanonical; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; -use crate::arrays::bool::BoolArrayExt; use crate::arrays::scalar_fn::ScalarFnFactoryExt; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; @@ -128,8 +128,8 @@ impl Validity { Self::NonNullable | Self::AllValid => true, Self::AllInvalid => false, Self::Array(a) => a - .scalar_at(index) - .vortex_expect("Validity array must support scalar_at") + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Validity array must support execute_scalar") .as_bool() .value() .vortex_expect("Validity must be non-nullable"), @@ -153,11 +153,9 @@ impl Validity { match self { Self::NonNullable => { let len = indices.len(); - let indices_mask = match indices.validity()? { - Validity::NonNullable | Validity::AllValid => Mask::new_true(len), - Validity::AllInvalid => Mask::new_false(len), - Validity::Array(a) => a.to_bool().to_mask(), - }; + let indices_mask = indices + .validity()? + .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; match indices_mask.bit_buffer() { AllOr::All => { if indices.dtype().is_nullable() { @@ -172,11 +170,9 @@ impl Validity { } Self::AllValid => { let len = indices.len(); - let indices_mask = match indices.validity()? { - Validity::NonNullable | Validity::AllValid => Mask::new_true(len), - Validity::AllInvalid => Mask::new_false(len), - Validity::Array(a) => a.to_bool().to_mask(), - }; + let indices_mask = indices + .validity()? + .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; match indices_mask.bit_buffer() { AllOr::All => Ok(Self::AllValid), AllOr::None => Ok(Self::AllInvalid), @@ -368,7 +364,7 @@ impl Validity { Self::Array(is_valid) => { is_valid .statistics() - .compute_min::() + .compute_min::(&mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("validity array must support min") .then(|| { // min true => all true @@ -393,11 +389,9 @@ impl Validity { #[inline] pub fn copy_from_array(array: &ArrayRef) -> VortexResult { let len = array.len(); - let mask = match array.validity()? { - Validity::NonNullable | Validity::AllValid => Mask::new_true(len), - Validity::AllInvalid => Mask::new_false(len), - Validity::Array(a) => a.to_bool().to_mask(), - }; + let mask = array + .validity()? + .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; Ok(Validity::from_mask(mask, array.dtype().nullability())) } diff --git a/vortex-array/src/variants.rs b/vortex-array/src/variants.rs index 257d6287b0c..ca64d1ccffb 100644 --- a/vortex-array/src/variants.rs +++ b/vortex-array/src/variants.rs @@ -135,7 +135,7 @@ impl PrimitiveTyped<'_> { /// Return the primitive value at the given index. pub fn value(&self, idx: usize) -> VortexResult> { self.0 - .is_valid(idx)? + .is_valid(idx, &mut LEGACY_SESSION.create_execution_ctx())? .then(|| self.value_unchecked(idx)) .transpose() } @@ -144,7 +144,7 @@ impl PrimitiveTyped<'_> { pub fn value_unchecked(&self, idx: usize) -> VortexResult { Ok(self .0 - .scalar_at(idx)? + .execute_scalar(idx, &mut LEGACY_SESSION.create_execution_ctx())? .as_primitive() .pvalue() .unwrap_or_else(|| PValue::zero(&self.ptype()))) @@ -165,7 +165,10 @@ impl IndexOrd> for PrimitiveTyped<'_> { // TODO(ngates): add generics to the `value` function and implement this over T. impl IndexOrd for PrimitiveTyped<'_> { fn index_cmp(&self, idx: usize, elem: &PValue) -> VortexResult> { - assert!(self.0.all_valid()?); + assert!( + self.0 + .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? + ); let value = self.value_unchecked(idx)?; Ok(value.partial_cmp(elem)) } diff --git a/vortex-btrblocks/src/schemes/patches.rs b/vortex-btrblocks/src/schemes/patches.rs index 8a82058ddb7..f094b92895e 100644 --- a/vortex-btrblocks/src/schemes/patches.rs +++ b/vortex-btrblocks/src/schemes/patches.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::patches::Patches; @@ -17,12 +19,13 @@ pub fn compress_patches(patches: Patches) -> VortexResult { // Check if the values are constant. let values = patches.values(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = if values .statistics() - .compute_is_constant() + .compute_is_constant(&mut ctx) .unwrap_or_default() { - ConstantArray::new(values.scalar_at(0)?, values.len()).into_array() + ConstantArray::new(values.execute_scalar(0, &mut ctx)?, values.len()).into_array() } else { values.clone() }; diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 47d9ae81cfb..e14de1aa9c0 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -6,7 +6,9 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::TemporalArray; @@ -83,7 +85,11 @@ impl Scheme for TemporalScheme { )?; if is_constant { - return Ok(ConstantArray::new(ext_array.scalar_at(0)?, ext_array.len()).into_array()); + return Ok(ConstantArray::new( + ext_array.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + ext_array.len(), + ) + .into_array()); } let dtype = temporal_array.dtype().clone(); diff --git a/vortex-compressor/src/builtins/constant/mod.rs b/vortex-compressor/src/builtins/constant/mod.rs index 1b177fc530b..6f51c76a646 100644 --- a/vortex-compressor/src/builtins/constant/mod.rs +++ b/vortex-compressor/src/builtins/constant/mod.rs @@ -5,6 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; use vortex_array::scalar::Scalar; @@ -43,20 +45,21 @@ mod string; /// /// If the array has any nulls, returns a [`MaskedArray`] with a [`ConstantArray`] child.` fn compress_constant_array_with_validity(source: &ArrayRef) -> VortexResult { - if source.all_invalid()? { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + if source.all_invalid(&mut ctx)? { return Ok( ConstantArray::new(Scalar::null(source.dtype().clone()), source.len()).into_array(), ); } let scalar_idx = (0..source.len()) - .position(|idx| source.is_valid(idx).unwrap_or(false)) + .position(|idx| source.is_valid(idx, &mut ctx).unwrap_or(false)) .vortex_expect("We checked that there exists a scalar that is not invalid"); - let scalar = source.scalar_at(scalar_idx)?; + let scalar = source.execute_scalar(scalar_idx, &mut ctx)?; let const_arr = ConstantArray::new(scalar, source.len()).into_array(); - if !source.all_valid()? { + if !source.all_valid(&mut ctx)? { Ok(MaskedArray::try_new(const_arr, source.validity()?)?.into_array()) } else { Ok(const_arr) diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index d1d7cc28958..db0803a6ee8 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -304,7 +304,7 @@ impl CascadingCompressor { if array.is_empty() { return Ok(array); } - if array.all_invalid()? { + if array.all_invalid(&mut LEGACY_SESSION.create_execution_ctx())? { return Ok( ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()).into_array(), ); diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index d393db58cb1..9c65be4d049 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -36,7 +36,8 @@ impl BoolStats { }); } - if input.all_invalid()? { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + if input.all_invalid(&mut ctx)? { return Ok(Self { null_count: u32::try_from(input.len())?, value_count: 0, @@ -44,10 +45,10 @@ impl BoolStats { }); } - let validity = input.as_ref().validity()?.to_mask( - input.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity = input + .as_ref() + .validity()? + .to_mask(input.as_ref().len(), &mut ctx)?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 8a1a85075ab..7a37c1b7c34 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -176,7 +176,8 @@ where }); } - if array.all_invalid()? { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + if array.all_invalid(&mut ctx)? { return Ok(FloatStats { null_count: u32::try_from(array.len())?, value_count: 0, @@ -193,7 +194,7 @@ where let null_count = array .statistics() - .compute_null_count() + .compute_null_count(&mut ctx) .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = array.len() - null_count; diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index af99efe3c0a..b75eee2d7aa 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -332,7 +332,8 @@ where }); } - if array.all_invalid()? { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + if array.all_invalid(&mut ctx)? { return Ok(IntegerStats { null_count: u32::try_from(array.len())?, value_count: 0, @@ -436,12 +437,12 @@ where let array_ref = array.as_ref(); let min = array_ref .statistics() - .compute_as::(Stat::Min) + .compute_as::(Stat::Min, &mut ctx) .vortex_expect("min should be computed"); let max = array_ref .statistics() - .compute_as::(Stat::Max) + .compute_as::(Stat::Max, &mut ctx) .vortex_expect("max should be computed"); let distinct = count_distinct_values.then(|| { diff --git a/vortex-compressor/src/stats/string.rs b/vortex-compressor/src/stats/string.rs index d35d8381611..8ed13323adc 100644 --- a/vortex-compressor/src/stats/string.rs +++ b/vortex-compressor/src/stats/string.rs @@ -3,6 +3,8 @@ //! String compression statistics. +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinViewArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -50,7 +52,7 @@ impl StringStats { ) -> VortexResult { let null_count = input .statistics() - .compute_null_count() + .compute_null_count(&mut LEGACY_SESSION.create_execution_ctx()) .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = input.len() - null_count; let estimated_distinct_count = opts diff --git a/vortex-cuda/src/canonical.rs b/vortex-cuda/src/canonical.rs index cde3841e196..591f1be404f 100644 --- a/vortex-cuda/src/canonical.rs +++ b/vortex-cuda/src/canonical.rs @@ -7,6 +7,8 @@ use async_trait::async_trait; use futures::future::try_join_all; use vortex::array::Canonical; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::ExtensionArray; @@ -50,7 +52,14 @@ impl CanonicalCudaExt for Canonical { let mut host_fields = vec![]; for field in fields.iter() { - host_fields.push(field.to_canonical()?.into_host().await?.into_array()); + host_fields.push( + field + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .into_host() + .await? + .into_array(), + ); } Ok(Canonical::Struct(StructArray::new( @@ -132,7 +141,8 @@ impl CanonicalCudaExt for Canonical { // Copy the storage array to host and rewrap in ExtensionArray. let host_storage = ext .storage_array() - .to_canonical()? + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_host() .await? .into_array(); diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 67167b57ba4..fd7190b034f 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -418,7 +418,7 @@ impl MaterializedPlan { let dtype = DType::Primitive(output_ptype, Nullability::Nullable); return ConstantArray::new(Scalar::null(dtype), len) .into_array() - .to_canonical(); + .execute::(ctx.execution_ctx()); } // The CUDA kernels are instantiated for unsigned integer types only; @@ -1853,7 +1853,7 @@ mod tests { let array = PrimitiveArray::from_option_iter( (0..2048u32).map(|i| if i % 3 == 0 { None } else { Some(i) }), ); - let cpu = array.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(array.clone())?.into_array(); let gpu = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx) .await? @@ -1887,7 +1887,7 @@ mod tests { }) .collect(); let prim = PrimitiveArray::from_option_iter(values.iter().copied()); - let cpu = prim.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(prim.clone())?.into_array(); // FoR encoding: subtract reference to get residuals [0..63]. // Null positions get 0 (from from_option_iter), which is fine — @@ -1944,7 +1944,7 @@ mod tests { let values: Vec = (0..2048).collect(); let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::AllValid); - let cpu = array.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(array.clone())?.into_array(); let gpu = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx) .await? .into_host() @@ -1983,7 +1983,7 @@ mod tests { let values = PrimitiveArray::from_option_iter([Some(10u32), None, Some(30)]); let dict = DictArray::try_new(codes.into_array(), values.into_array())?; - let cpu = dict.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(dict.clone())?.into_array(); let gpu = dict .into_array() .execute_cuda(&mut cuda_ctx) @@ -2021,7 +2021,7 @@ mod tests { let mask = Mask::from_iter((0..len).map(|i| i % 2 == 0)); let filter_array = FilterArray::try_new(prim.into_array(), mask)?; - let cpu = filter_array.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu = filter_array .into_array() .execute_cuda(&mut cuda_ctx) diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index a5ca2560d43..41e085d00b3 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -142,7 +142,7 @@ mod tests { .vortex_expect("bp"); let arr = FoR::try_new(bp.into_array(), 1000u32.into()).vortex_expect("for"); - let cpu = arr.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(arr.clone())?.into_array(); let gpu = arr .into_array() .execute_cuda(&mut ctx) @@ -177,7 +177,7 @@ mod tests { None, )?; - let cpu = alp.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(alp.clone())?.into_array(); let gpu = alp .into_array() .execute_cuda(&mut ctx) @@ -215,7 +215,7 @@ mod tests { .unwrap(); let arr = ALP::try_new(encoded, Exponents { e: 0, f: 2 }, Some(patches))?; - let cpu = arr.to_canonical()?.into_array(); + let cpu = crate::canonicalize_cpu(arr.clone())?.into_array(); let gpu = arr .into_array() .execute_cuda(&mut ctx) diff --git a/vortex-cuda/src/kernel/arrays/constant.rs b/vortex-cuda/src/kernel/arrays/constant.rs index 2164378b9e9..8e5966da243 100644 --- a/vortex-cuda/src/kernel/arrays/constant.rs +++ b/vortex-cuda/src/kernel/arrays/constant.rs @@ -227,7 +227,7 @@ mod tests { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); - let cpu_result = constant_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(constant_array.clone())?; let gpu_result = ConstantNumericExecutor .execute(constant_array.into_array(), &mut cuda_ctx) @@ -248,7 +248,7 @@ mod tests { .vortex_expect("failed to create execution context"); let constant_array = ConstantArray::new(42i32, 0); - let cpu_result = constant_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(constant_array.clone())?; let gpu_result = ConstantNumericExecutor .execute(constant_array.into_array(), &mut cuda_ctx) @@ -270,7 +270,7 @@ mod tests { // Test with array smaller than one block (< 2048 elements) let constant_array = ConstantArray::new(99i32, 100); - let cpu_result = constant_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(constant_array.clone())?; let gpu_result = ConstantNumericExecutor .execute(constant_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index ce98c989d88..442b730a037 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -339,7 +339,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -374,7 +374,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -406,7 +406,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -437,7 +437,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -470,7 +470,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -507,7 +507,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -551,7 +551,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -596,7 +596,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -629,7 +629,7 @@ mod tests { .vortex_expect("failed to create Dict array"); // Get baseline from CPU canonicalization - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; // Execute on CUDA let cuda_result = DictExecutor @@ -669,7 +669,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -697,7 +697,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -725,7 +725,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -756,7 +756,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -792,7 +792,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -825,7 +825,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -850,7 +850,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -877,7 +877,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -907,7 +907,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -932,7 +932,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -958,7 +958,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -991,7 +991,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) @@ -1027,7 +1027,7 @@ mod tests { let dict_array = DictArray::try_new(codes_array.into_array(), values.into_array()) .vortex_expect("failed to create Dict array"); - let baseline = dict_array.to_canonical()?; + let baseline = crate::canonicalize_cpu(dict_array.clone())?; let cuda_result = DictExecutor .execute(dict_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/arrays/shared.rs b/vortex-cuda/src/kernel/arrays/shared.rs index 085b30f9057..16e748d26ca 100644 --- a/vortex-cuda/src/kernel/arrays/shared.rs +++ b/vortex-cuda/src/kernel/arrays/shared.rs @@ -34,6 +34,6 @@ impl CudaExecute for SharedExecutor { shared .get_or_compute_async(|source| source.execute_cuda(ctx)) .await? - .to_canonical() + .execute::(ctx.execution_ctx()) } } diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index fae81d76ae8..068b07e7978 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -169,7 +169,7 @@ mod tests { Some(patches), )?; - let cpu_result = alp_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = ALPExecutor .execute(alp_array.into_array(), &mut cuda_ctx) @@ -213,7 +213,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), )?; - let cpu_result = alp_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = alp_array .into_array() @@ -244,7 +244,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), )?; - let cpu_result = alp_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = alp_array .into_array() diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 357d444fae4..5fb576b5089 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -202,7 +202,7 @@ mod tests { let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; assert!(bp_with_patches.patches().is_some()); - let cpu_result = bp_with_patches.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(bp_with_patches.clone())?.into_array(); let gpu_result = block_on(async { BitPackedExecutor @@ -233,7 +233,7 @@ mod tests { let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; assert!(bp_with_patches.patches().is_some()); - let cpu_result = bp_with_patches.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(bp_with_patches.clone())?.into_array(); let gpu_result = block_on(async { BitPackedExecutor @@ -274,7 +274,7 @@ mod tests { let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) .vortex_expect("operation should succeed in test"); - let cpu_result = bitpacked_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor @@ -323,7 +323,7 @@ mod tests { let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) .vortex_expect("operation should succeed in test"); - let cpu_result = bitpacked_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor @@ -388,7 +388,7 @@ mod tests { let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) .vortex_expect("operation should succeed in test"); - let cpu_result = bitpacked_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor @@ -485,7 +485,7 @@ mod tests { let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) .vortex_expect("operation should succeed in test"); - let cpu_result = bitpacked_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor .execute(bitpacked_array.into_array(), &mut cuda_ctx) @@ -520,7 +520,7 @@ mod tests { .vortex_expect("operation should succeed in test"); let sliced_array = bitpacked_array.into_array().slice(67..3969)?; assert!(sliced_array.is::()); - let cpu_result = sliced_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(sliced_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor .execute(sliced_array, &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index 53742dc11de..b719ff17aa2 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -284,7 +284,7 @@ mod tests { .vortex_expect("failed to create execution context"); let dtp_array = make_datetimeparts_array(days, seconds, subseconds, time_unit); - let cpu_result = dtp_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(dtp_array.clone())?; let gpu_result = DateTimePartsExecutor .execute(dtp_array.into_array(), &mut cuda_ctx) @@ -310,7 +310,7 @@ mod tests { let subseconds: Vec = (0..len).map(|i| (i % 1000) as i64).collect(); let dtp_array = make_datetimeparts_array(days, seconds, subseconds, TimeUnit::Milliseconds); - let cpu_result = dtp_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(dtp_array.clone())?; let gpu_result = DateTimePartsExecutor .execute(dtp_array.into_array(), &mut cuda_ctx) @@ -358,7 +358,7 @@ mod tests { ) .vortex_expect("Failed to create DateTimePartsArray"); - let cpu_result = dtp_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(dtp_array.clone())?; let gpu_result = DateTimePartsExecutor .execute(dtp_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index 3f4bdb92146..70d3c71a485 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -96,7 +96,8 @@ mod tests { ) .vortex_expect("create DecimalBytePartsArray"); - let cpu_result = dbp_array.to_canonical().vortex_expect("CPU canonicalize"); + let cpu_result = + crate::canonicalize_cpu(dbp_array.clone()).vortex_expect("CPU canonicalize"); let gpu_result = DecimalBytePartsExecutor .execute(dbp_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 490b797b9b8..0c33ee08ca4 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -74,7 +74,7 @@ impl CudaExecute for FoRExecutor { .into_primitive() .into_array() .slice(slice_range)? - .to_canonical(); + .execute::(ctx.execution_ctx()); } match_each_native_simd_ptype!(array.ptype(), |P| { decode_for::

(array, ctx).await }) @@ -162,7 +162,7 @@ mod tests { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); - let cpu_result = for_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(for_array.clone())?; let gpu_result = FoRExecutor .execute(for_array.into_array(), &mut cuda_ctx) @@ -190,7 +190,7 @@ mod tests { let packed = BitPacked::encode(&values, 3).unwrap().into_array(); let for_array = FoR::try_new(packed, (-8i8).into()).unwrap(); - let cpu_result = for_array.to_canonical().unwrap(); + let cpu_result = crate::canonicalize_cpu(for_array.clone()).unwrap(); let gpu_result = FoRExecutor .execute(for_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 428dbfea71e..1fa09702999 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -78,7 +78,7 @@ impl CudaExecute for RunEndExecutor { if matches!(values.validity()?, Validity::AllInvalid) { return ConstantArray::new(Scalar::null(values.dtype().clone()), output_len) .into_array() - .to_canonical(); + .execute::(ctx.execution_ctx()); } let ends = ends.execute_cuda(ctx).await?.into_primitive(); @@ -205,7 +205,7 @@ mod tests { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); - let cpu_result = runend_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; let gpu_result = RunEndExecutor .execute(runend_array.into_array(), &mut cuda_ctx) @@ -235,7 +235,7 @@ mod tests { let runend_array = make_runend_array(ends, values); assert_eq!(runend_array.len(), total_len); - let cpu_result = runend_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; let gpu_result = RunEndExecutor .execute(runend_array.into_array(), &mut cuda_ctx) @@ -257,7 +257,7 @@ mod tests { let runend_array = make_runend_array(vec![100u32], vec![42i32]); - let cpu_result = runend_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; let gpu_result = RunEndExecutor .execute(runend_array.into_array(), &mut cuda_ctx) @@ -284,7 +284,7 @@ mod tests { let runend_array = make_runend_array(ends, values); - let cpu_result = runend_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; let gpu_result = RunEndExecutor .execute(runend_array.into_array(), &mut cuda_ctx) @@ -314,7 +314,7 @@ mod tests { PrimitiveArray::new(Buffer::from(vec![10i32, 0, 30]), validity).into_array(); let runend_array = RunEnd::new(ends_array, values_array); - let cpu_result = runend_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(runend_array.clone())?.into_array(); // execute_cuda should fall back to CPU and still produce the correct result. let gpu_result = runend_array diff --git a/vortex-cuda/src/kernel/encodings/sequence.rs b/vortex-cuda/src/kernel/encodings/sequence.rs index bcc90ec77b7..c0721b8d920 100644 --- a/vortex-cuda/src/kernel/encodings/sequence.rs +++ b/vortex-cuda/src/kernel/encodings/sequence.rs @@ -129,7 +129,7 @@ mod tests { let array = Sequence::try_new_typed(base, multiplier, nullability, len).unwrap(); - let cpu_result = array.to_canonical().unwrap().into_array(); + let cpu_result = crate::canonicalize_cpu(array.clone()).unwrap().into_array(); let gpu_result = SequenceExecutor .execute(array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index f13a19bf0e1..98f9534be0a 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -124,7 +124,7 @@ mod tests { PrimitiveArray::new(Buffer::from(encoded_data), NonNullable).into_array(), )?; - let cpu_result = zigzag_array.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(zigzag_array.clone())?; let gpu_result = ZigZagExecutor .execute(zigzag_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index 4f09f418e4b..8b3367af6e0 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -208,7 +208,8 @@ impl CudaExecute for ZstdExecutor { dtype = %_other, "Only Binary/Utf8 ZSTD arrays supported on GPU, falling back to CPU" ); - Zstd::decompress(&zstd, ctx.execution_ctx())?.to_canonical() + Zstd::decompress(&zstd, ctx.execution_ctx())? + .execute::(ctx.execution_ctx()) } } } @@ -378,7 +379,8 @@ mod tests { let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; - let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())? + .execute::(cuda_ctx.execution_ctx())?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; @@ -413,7 +415,8 @@ mod tests { // 14 strings and 3 values per frame = ceil(14/3) = 5 frames. let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3)?; - let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; + let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())? + .execute::(cuda_ctx.execution_ctx())?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; @@ -445,7 +448,7 @@ mod tests { // Slice the array to get a subset (indices 2..7) let sliced_zstd = zstd_array.slice(2..7)?; - let cpu_result = sliced_zstd.to_canonical()?; + let cpu_result = crate::canonicalize_cpu(sliced_zstd.clone())?; let gpu_result = ZstdExecutor .execute(sliced_zstd.clone(), &mut cuda_ctx) .await?; @@ -473,7 +476,7 @@ mod tests { let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; - let cpu_result = zstd_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(zstd_array.clone())?.into_array(); // execute_cuda should fall back to CPU and still produce the correct result. let gpu_result = zstd_array diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index c93cf8aaec9..c31ab65fadb 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -239,7 +239,7 @@ mod tests { let input = PrimitiveArray::from_iter(0i64..1024).into_array(); let compressed = ZstdBuffers::compress(&input, 3, &VortexSession::empty())?; - let cpu_result = compressed.clone().into_array().to_canonical()?; + let cpu_result = crate::canonicalize_cpu(compressed.clone())?; let gpu_result = ZstdBuffersExecutor .execute(compressed.into_array(), &mut cuda_ctx) .await? @@ -266,7 +266,7 @@ mod tests { .into_array(); let compressed = ZstdBuffers::compress(&input, 3, &VortexSession::empty())?; - let cpu_result = compressed.clone().into_array().to_canonical()?; + let cpu_result = crate::canonicalize_cpu(compressed.clone())?; let gpu_result = ZstdBuffersExecutor .execute(compressed.into_array(), &mut cuda_ctx) .await? diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index cdfc4342fa6..566b1c3313d 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -94,7 +94,7 @@ mod tests { let filter_array = FilterArray::try_new(input.clone().into_array(), mask.clone())?; - let cpu_result = filter_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu_result = FilterExecutor .execute(filter_array.into_array(), &mut cuda_ctx) @@ -123,7 +123,7 @@ mod tests { let filter_array = FilterArray::try_new(input.into_array(), mask)?; - let cpu_result = filter_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu_result = FilterExecutor .execute(filter_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index c74c56e5b06..6bff0b20c70 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -88,7 +88,7 @@ mod tests { let filter_array = FilterArray::try_new(input.clone().into_array(), mask.clone())?; - let cpu_result = filter_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu_result = FilterExecutor .execute(filter_array.into_array(), &mut cuda_ctx) @@ -117,7 +117,7 @@ mod tests { let filter_array = FilterArray::try_new(input.into_array(), mask)?; - let cpu_result = filter_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu_result = FilterExecutor .execute(filter_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 84632f5cca8..23a6c91b048 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -74,7 +74,7 @@ mod tests { let filter_array = FilterArray::try_new(input.into_array(), mask.clone())?; - let cpu_result = filter_array.to_canonical()?.into_array(); + let cpu_result = crate::canonicalize_cpu(filter_array.clone())?.into_array(); let gpu_result = FilterExecutor .execute(filter_array.into_array(), &mut cuda_ctx) diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 9aed85580fd..2b1db3dd0a1 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -102,6 +102,7 @@ mod tests { use cudarc::driver::DeviceRepr; use vortex::array::IntoArray; + use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; @@ -161,10 +162,7 @@ mod tests { let cpu_result = values .clone() - .patch( - &patches, - &mut vortex::array::LEGACY_SESSION.create_execution_ctx(), - ) + .patch(&patches, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(); let PrimitiveDataParts { @@ -189,7 +187,8 @@ mod tests { Values::PTYPE, Validity::NonNullable, ) - .to_canonical() + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .into_host() .await diff --git a/vortex-cuda/src/kernel/slice/mod.rs b/vortex-cuda/src/kernel/slice/mod.rs index 647faa64fcd..30b1fdaf640 100644 --- a/vortex-cuda/src/kernel/slice/mod.rs +++ b/vortex-cuda/src/kernel/slice/mod.rs @@ -37,20 +37,30 @@ impl CudaExecute for SliceExecutor { let child = slice_array.child().clone().execute_cuda(ctx).await?; match child { - Canonical::Null(null_array) => null_array.into_array().slice(range)?.to_canonical(), - Canonical::Bool(bool_array) => bool_array.into_array().slice(range)?.to_canonical(), - Canonical::Primitive(prim_array) => { - prim_array.into_array().slice(range)?.to_canonical() - } - Canonical::Decimal(decimal_array) => { - decimal_array.into_array().slice(range)?.to_canonical() - } - Canonical::VarBinView(varbinview) => { - varbinview.into_array().slice(range)?.to_canonical() - } - Canonical::Extension(extension_array) => { - extension_array.into_array().slice(range)?.to_canonical() - } + Canonical::Null(null_array) => null_array + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), + Canonical::Bool(bool_array) => bool_array + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), + Canonical::Primitive(prim_array) => prim_array + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), + Canonical::Decimal(decimal_array) => decimal_array + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), + Canonical::VarBinView(varbinview) => varbinview + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), + Canonical::Extension(extension_array) => extension_array + .into_array() + .slice(range)? + .execute::(ctx.execution_ctx()), c => todo!("Slice kernel not implemented for {}", c.dtype()), } } diff --git a/vortex-cuda/src/lib.rs b/vortex-cuda/src/lib.rs index d23cbac1403..7ce8fa09b84 100644 --- a/vortex-cuda/src/lib.rs +++ b/vortex-cuda/src/lib.rs @@ -83,6 +83,18 @@ pub use vortex_nvcomp as nvcomp; use crate::kernel::SequenceExecutor; use crate::kernel::SliceExecutor; +#[cfg(test)] +pub(crate) fn canonicalize_cpu( + array: impl vortex::array::IntoArray, +) -> vortex::error::VortexResult { + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; + + array + .into_array() + .execute::(&mut LEGACY_SESSION.create_execution_ctx()) +} + /// Checks if CUDA is available on the system by looking for nvcc. pub fn cuda_available() -> bool { Command::new("nvcc") diff --git a/vortex-duckdb/src/exporter/run_end.rs b/vortex-duckdb/src/exporter/run_end.rs index fa62388cf57..442e612147c 100644 --- a/vortex-duckdb/src/exporter/run_end.rs +++ b/vortex-duckdb/src/exporter/run_end.rs @@ -86,7 +86,7 @@ impl ColumnExporter for RunEndExporter { if start_run_idx == end_run_idx { // NOTE(ngates): would be great if we could just export and set type == CONSTANT // self.values_exporter.export(start_run_idx, 1, vector, cache); - let constant = self.values.scalar_at(start_run_idx)?; + let constant = self.values.execute_scalar(start_run_idx, ctx)?; let value = constant.try_to_duckdb_scalar()?; vector.reference_value(&value); return Ok(()); diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 9a07028686b..f786f9ef6bf 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -56,7 +56,7 @@ pub(crate) fn new_exporter( }) .collect::>>()?; Ok(validity::new_exporter( - validity.to_mask(), + validity.to_mask(ctx), Box::new(StructExporter { children }), )) } diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 0a196598171..1eeca276eac 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -240,7 +240,7 @@ pub unsafe extern "C-unwind" fn vx_array_element_is_invalid( ) -> bool { try_or_default(error, || { vortex_ensure!(!array.is_null()); - vx_array::as_ref(array).is_invalid(index) + vx_array::as_ref(array).is_invalid(index, &mut LEGACY_SESSION.create_execution_ctx()) }) } @@ -328,7 +328,9 @@ macro_rules! ffiarray_get_ptype { pub unsafe extern "C-unwind" fn [](array: *const vx_array, index: usize) -> $ptype { let array = vx_array::as_ref(array); // TODO(joe): propagate this error up instead of expecting - let value = array.scalar_at(index).vortex_expect("scalar_at failed"); + let value = array + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("scalar_at failed"); // TODO(joe): propagate this error up instead of expecting value.as_primitive() .as_::<$ptype>() @@ -339,7 +341,9 @@ macro_rules! ffiarray_get_ptype { pub unsafe extern "C-unwind" fn [](array: *const vx_array, index: usize) -> $ptype { let array = vx_array::as_ref(array); // TODO(joe): propagate this error up instead of expecting - let value = array.scalar_at(index).vortex_expect("scalar_at failed"); + let value = array + .execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("scalar_at failed"); // TODO(joe): propagate this error up instead of expecting value.as_extension() .to_storage_scalar() @@ -373,7 +377,7 @@ pub unsafe extern "C-unwind" fn vx_array_get_utf8( let array = vx_array::as_ref(array); // TODO(joe): propagate this error up instead of expecting let value = array - .scalar_at(index as usize) + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at failed"); let utf8_scalar = value.as_utf8(); if let Some(buffer) = utf8_scalar.value() { @@ -393,7 +397,7 @@ pub unsafe extern "C-unwind" fn vx_array_get_binary( let array = vx_array::as_ref(array); // TODO(joe): propagate this error up instead of expecting let value = array - .scalar_at(index as usize) + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at failed"); let binary_scalar = value.as_binary(); if let Some(bytes) = binary_scalar.value() { diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 41dbb5c169c..e87fc4f731d 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -13,6 +13,7 @@ use futures::pin_mut; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::ConstantArray; @@ -1244,12 +1245,13 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { assert_eq!(result.len(), 3); assert_eq!(result.struct_fields().nfields(), 1); - assert!(result.all_valid()?); + let mut ctx = SESSION.create_execution_ctx(); + assert!(result.all_valid(&mut ctx)?); let nested_struct = result.unmasked_field_by_name("struct")?.to_struct(); assert_eq!(nested_struct.dtype(), &nested_dtype); assert_eq!(nested_struct.len(), 3); - assert!(nested_struct.all_invalid()?); + assert!(nested_struct.all_invalid(&mut ctx)?); Ok(()) } diff --git a/vortex-file/src/v2/file_stats_reader.rs b/vortex-file/src/v2/file_stats_reader.rs index fb31b28d61f..347670f050f 100644 --- a/vortex-file/src/v2/file_stats_reader.rs +++ b/vortex-file/src/v2/file_stats_reader.rs @@ -102,7 +102,7 @@ impl FileStatsLayoutReader { .execute::(&mut ctx)? .into_bool() .into_array() - .scalar_at(0)?; + .execute_scalar(0, &mut ctx)?; Ok(result.as_bool().value() == Some(true)) } diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index 88c7286b3b4..fe5496521b6 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -269,7 +269,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNull( ) -> jboolean { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let is_null = array_ref.inner.is_invalid(index as usize)?; + let is_null = array_ref + .inner + .is_invalid(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; if is_null { Ok(JNI_TRUE) } else { Ok(JNI_FALSE) } }) } @@ -305,9 +307,15 @@ macro_rules! get_primitive { .inner .to_extension() .storage_array() - .scalar_at(index as usize)? + .execute_scalar( + index as usize, + &mut LEGACY_SESSION.create_execution_ctx(), + )? } else { - array_ref.inner.scalar_at(index as usize)? + array_ref.inner.execute_scalar( + index as usize, + &mut LEGACY_SESSION.create_execution_ctx(), + )? }; Ok(scalar_value @@ -344,9 +352,11 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBigDecimal( .inner .to_extension() .storage_array() - .scalar_at(index as usize)? + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())? } else { - array_ref.inner.scalar_at(index as usize)? + array_ref + .inner + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())? }; let decimal_scalar = scalar_value.as_decimal(); @@ -408,7 +418,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBool( ) -> jboolean { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let value = array_ref.inner.scalar_at(index as usize)?; + let value = array_ref + .inner + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; match value.as_bool().value() { None => Ok(JNI_FALSE), Some(b) => { @@ -431,7 +443,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getUTF8<'local>( ) -> jstring { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { - let value = array_ref.inner.scalar_at(index as usize)?; + let value = array_ref + .inner + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; match value.as_utf8().value() { None => Ok(JObject::null().into_raw()), Some(buf_str) => Ok(env.new_string(buf_str.as_str())?.into_raw()), @@ -479,7 +493,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBinary<'local>( ) -> jbyteArray { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { - let value = array_ref.inner.scalar_at(index as usize)?; + let value = array_ref + .inner + .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; match value.as_binary().value() { None => Ok(JObject::null().into_raw()), Some(buf) => Ok(env.byte_array_from_slice(buf.as_slice())?.into_raw()), diff --git a/vortex-layout/src/layouts/compressed.rs b/vortex-layout/src/layouts/compressed.rs index 60f87b8efe0..fe75f66c946 100644 --- a/vortex-layout/src/layouts/compressed.rs +++ b/vortex-layout/src/layouts/compressed.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use futures::StreamExt as _; use vortex_array::ArrayContext; use vortex_array::ArrayRef; +use vortex_array::VortexSessionExecute; use vortex_array::expr::stats::Stat; use vortex_btrblocks::BtrBlocksCompressor; use vortex_error::VortexResult; @@ -97,16 +98,21 @@ impl LayoutStrategy for CompressingStrategy { let dtype = stream.dtype().clone(); let compressor = Arc::clone(&self.compressor); let stats = Arc::clone(&self.stats); + let session = session.clone(); + let compute_session = session.clone(); let handle = session.handle(); let stream = stream .map(move |chunk| { let compressor = Arc::clone(&compressor); let stats = Arc::clone(&stats); + let session = compute_session.clone(); handle.spawn_cpu(move || { let (sequence_id, chunk) = chunk?; // Compute the stats for the chunk prior to compression - chunk.statistics().compute_all(&stats)?; + chunk + .statistics() + .compute_all(&stats, &mut session.create_execution_ctx())?; Ok((sequence_id, compressor.compress_chunk(&chunk)?)) }) }) @@ -118,7 +124,7 @@ impl LayoutStrategy for CompressingStrategy { segment_sink, SequentialStreamAdapter::new(dtype, stream).sendable(), eof, - session, + &session, ) .await } diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index e8c78c2e2a7..bf6993d72b7 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -291,10 +291,11 @@ mod tests { builder.append_value("Long value to test that the statistics are actually truncated, it needs a bit of extra padding though"); builder.append_value("Another string that's meant to be smaller than the previous value, though still need extra padding"); let array = builder.finish(); + let mut stats_ctx = session.create_execution_ctx(); array.statistics().set_iter( array .statistics() - .compute_all(&Stat::all().collect::>()) + .compute_all(&Stat::all().collect::>(), &mut stats_ctx) .vortex_expect("stats computation should succeed for test array") .into_iter(), ); diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 5d0b2cf0a8d..a9ea39a6fe0 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -392,8 +392,10 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; @@ -718,7 +720,9 @@ mod tests { // ...and the result is masked with the validity of the parent StructArray assert_eq!( - result.scalar_at(0).unwrap(), + result + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Scalar::null(result.dtype().clone()), ); assert_nth_scalar!(result, 1, 2); @@ -760,7 +764,7 @@ mod tests { // Row 0: struct is valid, field "c" is 4. assert_eq!( result - .scalar_at(0) + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_struct() .field_by_idx(0) @@ -769,12 +773,18 @@ mod tests { ); // Row 1: struct is null (because root.a.b was null at this row). - assert!(result.scalar_at(1).unwrap().as_struct().is_null()); + assert!( + result + .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .as_struct() + .is_null() + ); // Row 2: struct is valid, field "c" is 6. assert_eq!( result - .scalar_at(2) + .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .as_struct() .field_by_idx(0) diff --git a/vortex-layout/src/layouts/zoned/builder.rs b/vortex-layout/src/layouts/zoned/builder.rs index c8fd3686281..6a7c7eff0e6 100644 --- a/vortex-layout/src/layouts/zoned/builder.rs +++ b/vortex-layout/src/layouts/zoned/builder.rs @@ -5,6 +5,8 @@ use std::marker::PhantomData; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::BoolBuilder; @@ -71,7 +73,7 @@ pub struct NamedArrays { impl NamedArrays { pub fn all_invalid(&self) -> VortexResult { // by convention we assume that the first array is the one we care about for logical validity - self.arrays[0].all_invalid() + self.arrays[0].all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) } } diff --git a/vortex-layout/src/layouts/zoned/writer.rs b/vortex-layout/src/layouts/zoned/writer.rs index 3e1d61098b0..09e54e97e4a 100644 --- a/vortex-layout/src/layouts/zoned/writer.rs +++ b/vortex-layout/src/layouts/zoned/writer.rs @@ -8,6 +8,7 @@ use futures::StreamExt as _; use parking_lot::Mutex; use vortex_array::ArrayContext; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; use vortex_array::expr::stats::Stat; use vortex_array::stats::PRUNING_STATS; use vortex_error::VortexResult; @@ -81,6 +82,8 @@ impl LayoutStrategy for ZonedStrategy { session: &VortexSession, ) -> VortexResult { let stats = Arc::clone(&self.options.stats); + let session = session.clone(); + let compute_session = session.clone(); let handle = session.handle(); let handle2 = handle.clone(); @@ -96,9 +99,12 @@ impl LayoutStrategy for ZonedStrategy { stream .map(move |chunk| { let stats = Arc::clone(&stats); + let session = compute_session.clone(); handle2.spawn_cpu(move || { let (sequence_id, chunk) = chunk?; - chunk.statistics().compute_all(&stats)?; + chunk + .statistics() + .compute_all(&stats, &mut session.create_execution_ctx())?; VortexResult::Ok((sequence_id, chunk)) }) }) @@ -133,7 +139,7 @@ impl LayoutStrategy for ZonedStrategy { Arc::clone(&segment_sink), stream, data_eof, - session, + &session, ) .await?; @@ -153,7 +159,7 @@ impl LayoutStrategy for ZonedStrategy { .sequenced(eof.split_off()); let zones_layout = self .stats - .write_stream(ctx, Arc::clone(&segment_sink), stats_stream, eof, session) + .write_stream(ctx, Arc::clone(&segment_sink), stats_stream, eof, &session) .await?; Ok(ZonedLayout::new( diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 39ab7179236..cb49377007e 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -7,6 +7,7 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::sum::sum; use vortex_array::arrays::StructArray; @@ -129,7 +130,7 @@ impl ZoneMap { match stat { // For stats that are associative, we can just compute them over the stat column Stat::Min | Stat::Max | Stat::Sum => { - if let Some(s) = array.statistics().compute_stat(stat)? + if let Some(s) = array.statistics().compute_stat(stat, ctx)? && let Some(v) = s.into_value() { stats_set.set(stat, Precision::exact(v)) @@ -220,7 +221,10 @@ impl StatsAccumulator { pub fn push_chunk(&mut self, array: &ArrayRef) -> VortexResult<()> { for builder in self.builders.iter_mut() { - if let Some(v) = array.statistics().compute_stat(builder.stat())? { + if let Some(v) = array + .statistics() + .compute_stat(builder.stat(), &mut LEGACY_SESSION.create_execution_ctx())? + { builder.append_scalar(v.cast(&v.dtype().as_nullable())?)?; } else { builder.append_null(); diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 6f6a0666d8c..bdcc95d7726 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -614,7 +614,10 @@ impl PyArray { )) .into()); } - Ok(PyScalar::init(py, slf.scalar_at(index)?)?) + Ok(PyScalar::init( + py, + slf.execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx())?, + )?) } /// Filter, permute, and/or repeat elements by their index. diff --git a/vortex-python/src/scan.rs b/vortex-python/src/scan.rs index 11773588add..1fd53d96131 100644 --- a/vortex-python/src/scan.rs +++ b/vortex-python/src/scan.rs @@ -4,6 +4,8 @@ use pyo3::exceptions::PyIndexError; use pyo3::prelude::*; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::layout::scan::repeated_scan::RepeatedScan; use crate::RUNTIME; @@ -68,7 +70,7 @@ impl PyRepeatedScan { if array.is_empty() { continue; } - let scalar = array.scalar_at(0)?; + let scalar = array.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; return Ok(PyScalar::init(slf.py(), scalar)?); } diff --git a/vortex-tensor/src/encodings/turboquant/tests/compute.rs b/vortex-tensor/src/encodings/turboquant/tests/compute.rs index ac0389048f4..0a9e0ab7a18 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/compute.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/compute.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; @@ -93,8 +94,9 @@ fn scalar_at_matches_decompress() -> VortexResult<()> { let full_decoded = encoded.clone().execute::(&mut ctx)?; for i in [0, 1, 5, 9] { - let expected = full_decoded.scalar_at(i)?; - let actual = encoded.scalar_at(i)?; + let expected = + full_decoded.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx())?; + let actual = encoded.execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx())?; assert_eq!(expected, actual, "scalar_at mismatch at index {i}"); } Ok(()) diff --git a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs index 8d406239019..6fc19bb93ec 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs @@ -127,7 +127,7 @@ fn nullable_l2_norm_readthrough() -> VortexResult<()> { let orig_f32 = orig_prim.as_slice::(); for row in 0..5 { if row % 2 == 0 { - assert!(norms.is_valid(row)?, "row {row} should be valid"); + assert!(norms.is_valid(row, &mut ctx)?, "row {row} should be valid"); let expected: f32 = orig_f32[row * 128..(row + 1) * 128] .iter() .map(|&v| v * v) @@ -139,7 +139,7 @@ fn nullable_l2_norm_readthrough() -> VortexResult<()> { "norm mismatch at valid row {row}: actual={actual}, expected={expected}" ); } else { - assert!(!norms.is_valid(row)?, "row {row} should be null"); + assert!(!norms.is_valid(row, &mut ctx)?, "row {row} should be null"); } } Ok(()) diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 6c06d165a0e..9f8eff0b361 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -530,8 +530,8 @@ mod tests { let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; // Row 0: self-similarity = 1.0, row 1: null. - assert!(prim.is_valid(0)?); - assert!(!prim.is_valid(1)?); + assert!(prim.is_valid(0, &mut ctx)?); + assert!(!prim.is_valid(1, &mut ctx)?); assert_close(&[prim.as_slice::()[0]], &[1.0]); Ok(()) } @@ -620,8 +620,8 @@ mod tests { let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 2)?; let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; - assert!(prim.is_valid(0)?); - assert!(!prim.is_valid(1)?); + assert!(prim.is_valid(0, &mut ctx)?); + assert!(!prim.is_valid(1, &mut ctx)?); assert_close(&[prim.as_slice::()[0]], &[1.0]); Ok(()) } diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 3f4678626bd..5928335ccf8 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -798,9 +798,9 @@ mod tests { let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; // Row 0: 1*7 + 2*8 = 23, row 1: null, row 2: 5*11 + 6*12 = 127. - assert!(prim.is_valid(0)?); - assert!(!prim.is_valid(1)?); - assert!(prim.is_valid(2)?); + assert!(prim.is_valid(0, &mut ctx)?); + assert!(!prim.is_valid(1, &mut ctx)?); + assert!(prim.is_valid(2, &mut ctx)?); assert_close(&[prim.as_slice::()[0]], &[23.0]); assert_close(&[prim.as_slice::()[2]], &[127.0]); Ok(()) @@ -901,8 +901,8 @@ mod tests { let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; // Row 0: 5.0 * 5.0 * dot([0.6, 0.8], [0.6, 0.8]) = 25.0, row 1: null. - assert!(prim.is_valid(0)?); - assert!(!prim.is_valid(1)?); + assert!(prim.is_valid(0, &mut ctx)?); + assert!(!prim.is_valid(1, &mut ctx)?); assert_close(&[prim.as_slice::()[0]], &[25.0]); Ok(()) } diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 01164a71dfe..673c454b56b 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -782,7 +782,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); let ext: ExtensionArray = array.execute(&mut ctx)?; let validity = (0..ext.len()) - .map(|i| ext.is_valid(i)) + .map(|i| ext.is_valid(i, &mut ctx)) .collect::>>()?; let storage: FixedSizeListArray = ext.storage_array().clone().execute(&mut ctx)?; let elements: PrimitiveArray = storage.elements().clone().execute(&mut ctx)?; @@ -836,9 +836,9 @@ mod tests { let storage: FixedSizeListArray = actual.storage_array().clone().execute(&mut ctx)?; let elements: PrimitiveArray = storage.elements().clone().execute(&mut ctx)?; - assert!(actual.is_valid(0)?); - assert!(!actual.is_valid(1)?); - assert!(!actual.is_valid(2)?); + assert!(actual.is_valid(0, &mut ctx)?); + assert!(!actual.is_valid(1, &mut ctx)?); + assert!(!actual.is_valid(2, &mut ctx)?); assert_close(&elements.as_slice::()[..2], &[3.0, 4.0]); Ok(()) } diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 13245dd880b..cd77da65f3c 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -320,8 +320,8 @@ mod tests { let prim: PrimitiveArray = result.into_array().execute(&mut ctx)?; // Row 0: norm = 5.0, row 1: null. - assert!(prim.is_valid(0)?); - assert!(!prim.is_valid(1)?); + assert!(prim.is_valid(0, &mut ctx)?); + assert!(!prim.is_valid(1, &mut ctx)?); assert_close(&[prim.as_slice::()[0]], &[5.0]); Ok(()) } diff --git a/vortex-test/compat-gen/src/adapter.rs b/vortex-test/compat-gen/src/adapter.rs index 9e29d6d69f0..a97399dac79 100644 --- a/vortex-test/compat-gen/src/adapter.rs +++ b/vortex-test/compat-gen/src/adapter.rs @@ -13,7 +13,9 @@ use futures::stream; use tokio::runtime::Runtime; use vortex::VortexSessionDefault; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::MaskFuture; +use vortex::array::VortexSessionExecute; use vortex::array::expr::root; use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; @@ -40,8 +42,9 @@ fn runtime() -> VortexResult { /// all stats so they are present in the serialized output. pub fn compute_all_stats(array: &ArrayRef) -> VortexResult<()> { let all_stats: Vec = Stat::all().collect(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); for node in array.depth_first_traversal() { - let computed = node.statistics().compute_all(&all_stats)?; + let computed = node.statistics().compute_all(&all_stats, &mut ctx)?; node.statistics().set_iter(computed.into_iter()); } Ok(()) diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs index 38e97d9f448..161f6d3c28e 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/constant.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::Constant; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; @@ -76,7 +78,7 @@ impl FlatLayoutFixture for ConstantFixture { Some("UTC".into()), ) .into_array() - .scalar_at(0)?; + .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; let const_timestamp = ConstantArray::new(timestamp_scalar, N); let arr = StructArray::try_new( diff --git a/vortex-tui/src/browse/ui/layouts.rs b/vortex-tui/src/browse/ui/layouts.rs index a00c363610e..d90720c5223 100644 --- a/vortex-tui/src/browse/ui/layouts.rs +++ b/vortex-tui/src/browse/ui/layouts.rs @@ -27,7 +27,9 @@ use ratatui::widgets::Table; use ratatui::widgets::Widget; use ratatui::widgets::Wrap; use vortex::array::ArrayRef; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::struct_::StructArrayExt; use vortex::error::VortexExpect; use vortex::layout::layouts::flat::Flat; @@ -163,7 +165,7 @@ fn render_array(app: &AppState, area: Rect, buf: &mut Buffer, is_stats_table: bo std::iter::once(Cell::from(Text::from(format!("{chunk_id}")))) .chain(field_arrays.iter().map(|arr| { Cell::from(Text::from( - arr.scalar_at(chunk_id) + arr.execute_scalar(chunk_id, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("scalar_at failed") .to_string(), )) diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 4a0edf99389..7fbbd80620d 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -13,6 +13,7 @@ use mimalloc::MiMalloc; use rand::RngExt; use rand::SeedableRng; use vortex::array::ArrayRef; +use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; @@ -418,6 +419,6 @@ fn decompress(bencher: Bencher, setup_fn: SetupFn) { let nbytes = compressed.nbytes(); with_byte_counter(bencher, nbytes) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| (**a).clone().execute::(ctx)); } diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 3c8777bc3d3..4ff5a8de9ef 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -14,6 +14,8 @@ use rand::RngExt; use rand::SeedableRng; use rand::prelude::IndexedRandom; use rand::rngs::StdRng; +use vortex::array::Canonical; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; @@ -39,6 +41,7 @@ use vortex::encodings::zigzag::zigzag_encode; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex_array::VortexSessionExecute; +use vortex_error::VortexResult; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -65,6 +68,10 @@ fn with_byte_counter<'a, 'b>(bencher: Bencher<'a, 'b>, bytes: u64) -> Bencher<'a } } +fn canonicalize(array: impl IntoArray, ctx: &mut ExecutionCtx) -> VortexResult { + array.into_array().execute::(ctx) +} + // Setup functions fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) { let mut rng = StdRng::seed_from_u64(0); @@ -125,8 +132,8 @@ fn bench_bitpacked_decompress_u32(bencher: Bencher) { .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "runend_compress_u32")] @@ -144,8 +151,8 @@ fn bench_runend_decompress_u32(bencher: Bencher) { let compressed = RunEnd::encode(uint_array.into_array()).unwrap(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "delta_compress_u32")] @@ -169,8 +176,8 @@ fn bench_delta_decompress_u32(bencher: Bencher) { .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "for_compress_i32")] @@ -188,8 +195,8 @@ fn bench_for_decompress_i32(bencher: Bencher) { let compressed = FoR::encode(int_array).unwrap(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "dict_compress_u32")] @@ -207,8 +214,8 @@ fn bench_dict_decompress_u32(bencher: Bencher) { let compressed = dict_encode(&uint_array.into_array()).unwrap(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "zigzag_compress_i32")] @@ -226,8 +233,8 @@ fn bench_zigzag_decompress_i32(bencher: Bencher) { let compressed = zigzag_encode(int_array.as_view()).unwrap().into_array(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[expect(clippy::cast_possible_truncation)] @@ -248,8 +255,8 @@ fn bench_sequence_decompress_u32(bencher: Bencher) { .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "alp_compress_f64")] @@ -279,8 +286,8 @@ fn bench_alp_decompress_f64(bencher: Bencher) { .unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "alp_rd_compress_f64")] @@ -302,8 +309,8 @@ fn bench_alp_rd_decompress_f64(bencher: Bencher) { let compressed = encoder.encode(float_array.as_view()); with_byte_counter(bencher, NUM_VALUES * 8) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "pcodec_compress_f64")] @@ -321,8 +328,8 @@ fn bench_pcodec_decompress_f64(bencher: Bencher) { let compressed = Pco::from_primitive(float_array.as_view(), 3, 0).unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[cfg(feature = "zstd")] @@ -351,8 +358,8 @@ fn bench_zstd_decompress_u32(bencher: Bencher) { .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } // String compression benchmarks @@ -375,8 +382,8 @@ fn bench_dict_decompress_string(bencher: Bencher) { let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes) - .with_inputs(|| &dict) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&dict, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[divan::bench(name = "fsst_compress_string")] @@ -405,8 +412,8 @@ fn bench_fsst_decompress_string(bencher: Bencher) { let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes) - .with_inputs(|| &fsst_array) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&fsst_array, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } #[cfg(feature = "zstd")] @@ -439,8 +446,8 @@ fn bench_zstd_decompress_string(bencher: Bencher) { let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes) - .with_inputs(|| &compressed) - .bench_refs(|a| a.to_canonical()); + .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| canonicalize((**a).clone(), ctx)); } // TODO(connor): Remove this. @@ -554,7 +561,7 @@ mod turboquant_benches { with_byte_counter(bencher, (NUM_VECTORS * $dim * 4) as u64) .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) .bench_refs(|(a, ctx)| { - (*a).clone() + (**a).clone() .into_array() .execute::(ctx) .unwrap() From e4e7667c8c407a659ab4fbb47405173e270ea004 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:30:49 -0400 Subject: [PATCH 066/250] Demo TurboQuant basic search with serialization (#7451) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Adds a TurboQuant demo where we convert the parquet files to a Vortex file (in-memory only now, but still serialized as bytes), and then we verify by decoding and performing a basic cosine similarity expression search with a filter pushdown. This is based on top of https://github.com/vortex-data/vortex/pull/7446, please dont merge until that has merged ## Testing The example runs! Signed-off-by: Connor Tsui --- Cargo.lock | 3 + vortex-bench/Cargo.toml | 14 +- vortex-bench/src/vector_dataset/convert.rs | 503 ++++++++++++++++++++ vortex-bench/src/vector_dataset/mod.rs | 2 + vortex/Cargo.toml | 6 + vortex/examples/tracing_vortex.rs | 7 +- vortex/examples/turboquant_vector_search.rs | 395 +++++++++++++++ 7 files changed, 922 insertions(+), 8 deletions(-) create mode 100644 vortex-bench/src/vector_dataset/convert.rs create mode 100644 vortex/examples/turboquant_vector_search.rs diff --git a/Cargo.lock b/Cargo.lock index adcab4a2864..b746598d3b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10056,6 +10056,7 @@ dependencies = [ "arrow-array 58.0.0", "codspeed-divan-compat", "fastlanes", + "futures", "mimalloc", "parquet 58.0.0", "paste", @@ -10068,6 +10069,7 @@ dependencies = [ "vortex", "vortex-alp", "vortex-array", + "vortex-bench", "vortex-btrblocks", "vortex-buffer", "vortex-bytebool", @@ -10230,6 +10232,7 @@ dependencies = [ "url", "uuid", "vortex", + "vortex-tensor", ] [[package]] diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index 8b2bb6efe25..e170a6552a8 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -17,6 +17,14 @@ version = { workspace = true } workspace = true [dependencies] +vortex = { workspace = true, features = [ + "object_store", + "files", + "tokio", + "zstd", +] } +vortex-tensor = { workspace = true } # TODO(connor): In the future, this might be inside vortex. + anyhow = { workspace = true } arrow-array = { workspace = true } arrow-schema = { workspace = true } @@ -57,12 +65,6 @@ tracing-subscriber = { workspace = true, features = [ ] } url = { workspace = true } uuid = { workspace = true, features = ["v4"] } -vortex = { workspace = true, features = [ - "object_store", - "files", - "tokio", - "zstd", -] } [features] unstable_encodings = ["vortex/unstable_encodings"] diff --git a/vortex-bench/src/vector_dataset/convert.rs b/vortex-bench/src/vector_dataset/convert.rs new file mode 100644 index 00000000000..f0c13ff33aa --- /dev/null +++ b/vortex-bench/src/vector_dataset/convert.rs @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +// TODO(connor): Should we re-export this through `conversions.rs`? + +use vortex::array::ArrayRef; +use vortex::array::IntoArray; +use vortex::array::arrays::Chunked; +use vortex::array::arrays::ChunkedArray; +use vortex::array::arrays::ExtensionArray; +use vortex::array::arrays::FixedSizeListArray; +use vortex::array::arrays::List; +use vortex::array::arrays::ListView; +use vortex::array::arrays::Primitive; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::chunked::ChunkedArrayExt; +use vortex::array::arrays::list::ListArrayExt; +use vortex::array::arrays::listview::recursive_list_from_list_view; +use vortex::array::validity::Validity; +use vortex::dtype::DType; +use vortex::dtype::extension::ExtDType; +use vortex::error::VortexExpect; +use vortex::error::VortexResult; +use vortex::error::vortex_bail; +use vortex::error::vortex_err; +use vortex::extension::EmptyMetadata; +use vortex_tensor::vector::Vector; + +/// Rewrap a list-of-float column as a [`vortex_tensor::vector::Vector`] extension array. +/// +/// Parquet has no fixed-size list logical type, so an embedding column ingested via +/// `parquet_to_vortex_chunks` arrives as `List` (or `List`) even when every row has the +/// same length. +/// +/// This helper validates that every list in `input` has the same length `D` and reconstructs the +/// column as an `Extension(FixedSizeList)` array, which is the type expected by the +/// vector search scalar functions in `vortex-tensor`. +/// +/// The input may be either a single [`ListView`] array or a [`Chunked`] array of lists (the common +/// case after `parquet_to_vortex_chunks`). Chunked inputs are converted chunk-by-chunk and +/// reassembled as a [`ChunkedArray`] of `Extension`. We also convert [`ListView`] to +/// [`List`] so that we know all elements are contiguous (this might be slow). +/// +/// # Errors +/// +/// Returns an error if: +/// - `input` is not a `ListView`, `List`, or `Chunked` array. +/// - The element type is not a float primitive (`f16`, `f32`, or `f64`). +/// - A nullable element dtype (`List`) is accepted as long as the runtime validity is +/// `NonNullable` or `AllValid` since parquet has no non-nullable-element list logical type, so +/// arrow-rs always marks list-of-float element fields as nullable on read regardless of whether +/// any element is actually missing. In that case the elements are rewrapped as non-nullable +/// before being embedded in the FSL. +/// - The element dtype is nullable *and* any element is actually null (i.e., `Validity::AllInvalid` +/// or any `Validity::Array` mask). Vector extension elements must be non-null, and that is +/// verified on construction. +/// - Any row has a different length than the first row. +/// - The list validity is nullable (vector elements cannot be null at the row level). +/// - The input has zero rows (the dimension cannot be inferred from empty input). +pub fn list_to_vector_ext(input: ArrayRef) -> VortexResult { + if let Some(chunked) = input.as_opt::() { + let converted: Vec = chunked + .iter_chunks() + .map(|chunk| list_to_vector_ext(chunk.clone())) + .collect::>()?; + + let Some(first) = converted.first() else { + vortex_bail!("list_to_vector_ext: chunked input has no chunks"); + }; + + let dtype = first.dtype().clone(); + return Ok(ChunkedArray::try_new(converted, dtype)?.into_array()); + } + + // `parquet_to_vortex_chunks` produces `ListView` arrays for list columns by default; + // materialize them into a flat `List` representation before we validate offsets. + if input.as_opt::().is_some() { + let flat = recursive_list_from_list_view(input)?; + return list_to_vector_ext(flat); + } + + let Some(list) = input.as_opt::() else { + vortex_bail!( + "list_to_vector_ext: expected a List array, got dtype {}", + input.dtype() + ); + }; + + if !matches!( + list.list_validity(), + Validity::NonNullable | Validity::AllValid + ) { + vortex_bail!( + "list_to_vector_ext: list rows must be non-nullable for Vector extension wrapping" + ); + } + + let element_dtype = list.element_dtype().clone(); + let DType::Primitive(ptype, elem_nullability) = &element_dtype else { + vortex_bail!( + "list_to_vector_ext: element dtype must be a primitive float, got {}", + element_dtype + ); + }; + if !ptype.is_float() { + vortex_bail!( + "list_to_vector_ext: element type must be float (f16/f32/f64), got {}", + ptype + ); + } + + // Extract the flat elements buffer up front: the nullable-handling branch below + // needs to inspect runtime validity before we can decide whether to rewrap it. + let raw_elements = list.sliced_elements()?; + + let num_rows = input.len(); + if num_rows == 0 { + vortex_bail!("list_to_vector_ext: cannot infer vector dimension from empty input"); + } + + // Walk the offsets array once, reusing the previous iteration's `end` as the + // next iteration's `start`. Each `offset_at` call goes through + // `ListArrayExt::offset_at`, which has a fast path when the offsets child is a + // `Primitive` array (direct slice index). That's the common case after + // `parquet_to_vortex_chunks`, so for a 100K-row column we do ~100K primitive + // slice indexes rather than 200K. The loop body is O(1) either way. + let mut prev_end = list.offset_at(0)?; + let first_end = list.offset_at(1)?; + + let dim = first_end.checked_sub(prev_end).ok_or_else(|| { + vortex_err!("list_to_vector_ext: offsets are not monotonically increasing") + })?; + if dim == 0 { + vortex_bail!("list_to_vector_ext: first row has zero elements"); + } + + prev_end = first_end; + + for i in 1..num_rows { + let end = list.offset_at(i + 1)?; + + let row_len = end + .checked_sub(prev_end) + .vortex_expect("list offsets must be monotonically increasing"); + if row_len != dim { + vortex_bail!( + "list_to_vector_ext: row {} has length {} but expected {}", + i, + row_len, + dim + ); + } + + prev_end = end; + } + + let expected_elements = num_rows + .checked_mul(dim) + .ok_or_else(|| vortex_err!("list_to_vector_ext: num_rows * dim overflows usize"))?; + if raw_elements.len() != expected_elements { + vortex_bail!( + "list_to_vector_ext: elements buffer has length {} but expected {}", + raw_elements.len(), + expected_elements + ); + } + + // Parquet has no non-nullable-element list logical type, so arrow-rs marks every + // `List`'s element field as nullable on read regardless of what the writer intended. + // That propagates through `DType::from_arrow`, so every real embedding parquet file arrives + // shaped as `List` even when every value is present. A nullable element dtype is + // losslessly convertible to a non-nullable FSL as long as the runtime validity is + // `NonNullable`/`AllValid`; we must only reject when a real null is present. + let elements = if elem_nullability.is_nullable() { + let primitive = raw_elements.as_opt::().ok_or_else(|| { + vortex_err!( + "list_to_vector_ext: expected nullable-float elements to downcast to \ + Primitive, got dtype {}", + raw_elements.dtype() + ) + })?; + match primitive.validity()? { + Validity::NonNullable | Validity::AllValid => { + // `to_host_sync` is a no-op for host-resident buffers, so this is a + // metadata change (rebuilding the array with a non-nullable dtype), + // not a data copy. + let byte_buffer = primitive.buffer_handle().to_host_sync(); + PrimitiveArray::from_byte_buffer(byte_buffer, *ptype, Validity::NonNullable) + .into_array() + } + Validity::AllInvalid => { + vortex_bail!( + "list_to_vector_ext: list has nullable element dtype with all-invalid \ + elements; Vector extension elements must be non-null" + ); + } + Validity::Array(_) => { + vortex_bail!( + "list_to_vector_ext: list has nullable element dtype with one or more \ + actual null elements; Vector extension elements must be non-null" + ); + } + } + } else { + raw_elements + }; + + let dim_u32 = u32::try_from(dim) + .map_err(|_| vortex_err!("list_to_vector_ext: dimension {dim} does not fit in u32"))?; + + // Finally, construct the `FixedSizeListArray` and wrap it in a Vector array. + let fsl = FixedSizeListArray::try_new(elements, dim_u32, Validity::NonNullable, num_rows)?; + let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) +} + +#[cfg(test)] +mod tests { + use vortex::array::Array; + use vortex::array::ArrayRef; + use vortex::array::IntoArray; + use vortex::array::arrays::BoolArray; + use vortex::array::arrays::ChunkedArray; + use vortex::array::arrays::Extension; + use vortex::array::arrays::List; + use vortex::array::arrays::ListViewArray; + use vortex::array::arrays::PrimitiveArray; + use vortex::array::arrays::extension::ExtensionArrayExt; + use vortex::array::validity::Validity; + use vortex::buffer::BufferMut; + use vortex::dtype::DType; + + use super::list_to_vector_ext; + + /// Build a `List` whose elements carry the given [`Validity`]. Passing + /// `Validity::NonNullable` produces a `List`; any other variant produces + /// a `List`, matching the shape `parquet_to_vortex_chunks` produces for + /// embedding columns after arrow-rs' canonicalization. + fn list_f32_with_element_validity( + values: &[f32], + dim: usize, + element_validity: Validity, + ) -> ArrayRef { + assert_eq!( + values.len() % dim, + 0, + "values.len() must be a multiple of dim" + ); + let num_rows = values.len() / dim; + let elements = PrimitiveArray::new::( + BufferMut::::from_iter(values.iter().copied()).freeze(), + element_validity, + ) + .into_array(); + let mut offsets_buf = BufferMut::::with_capacity(num_rows + 1); + for i in 0..=num_rows { + offsets_buf.push(i32::try_from(i * dim).unwrap()); + } + let offsets = + PrimitiveArray::new::(offsets_buf.freeze(), Validity::NonNullable).into_array(); + Array::::new(elements, offsets, Validity::NonNullable).into_array() + } + + fn list_f32(rows: &[&[f32]]) -> ArrayRef { + let mut elements = BufferMut::::with_capacity(rows.iter().map(|r| r.len()).sum()); + let mut offsets = BufferMut::::with_capacity(rows.len() + 1); + offsets.push(0); + for row in rows { + for &v in row.iter() { + elements.push(v); + } + offsets.push(i32::try_from(elements.len()).unwrap()); + } + + let elements_array = + PrimitiveArray::new::(elements.freeze(), Validity::NonNullable).into_array(); + let offsets_array = + PrimitiveArray::new::(offsets.freeze(), Validity::NonNullable).into_array(); + Array::::new(elements_array, offsets_array, Validity::NonNullable).into_array() + } + + #[test] + fn uniform_list_becomes_vector_extension() { + let list = list_f32(&[&[1.0, 2.0, 3.0], &[4.0, 5.0, 6.0], &[7.0, 8.0, 9.0]]); + let wrapped = list_to_vector_ext(list).unwrap(); + assert_eq!(wrapped.len(), 3); + let ext = wrapped.as_opt::().expect("returns Extension"); + assert!(matches!( + ext.storage_array().dtype(), + DType::FixedSizeList(_, 3, _) + )); + } + + #[test] + fn mismatched_row_length_is_rejected() { + let list = list_f32(&[&[1.0, 2.0, 3.0], &[4.0, 5.0]]); + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("row 1 has length 2 but expected 3"), + "unexpected error: {err}", + ); + } + + #[test] + fn non_list_input_is_rejected() { + let primitive = PrimitiveArray::new::( + BufferMut::::from_iter([1.0f32, 2.0, 3.0]).freeze(), + Validity::NonNullable, + ) + .into_array(); + let err = list_to_vector_ext(primitive).unwrap_err().to_string(); + assert!( + err.contains("expected a List array"), + "unexpected error: {err}" + ); + } + + #[test] + fn empty_input_is_rejected() { + let list = list_f32(&[]); + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("cannot infer vector dimension from empty input"), + "unexpected error: {err}", + ); + } + + /// Build a `ListView` whose every row is a length-`dim` slice of the flattened + /// `values` buffer. This shape matches what `parquet_to_vortex_chunks` produces for + /// embedding columns after arrow-rs' canonicalization, and exercises the + /// `list_to_vector_ext` fast-path that collapses `ListView` → `List` before + /// validating offsets. + fn list_view_f32(dim: usize, rows: &[&[f32]]) -> ArrayRef { + let mut values = BufferMut::::with_capacity(rows.len() * dim); + for row in rows { + assert_eq!(row.len(), dim); + for &v in row.iter() { + values.push(v); + } + } + let elements = + PrimitiveArray::new::(values.freeze(), Validity::NonNullable).into_array(); + + let dim_i32 = i32::try_from(dim).unwrap(); + let num_rows = rows.len(); + + let mut offsets_buf = BufferMut::::with_capacity(num_rows); + for i in 0..num_rows { + offsets_buf.push(i32::try_from(i).unwrap() * dim_i32); + } + let offsets = + PrimitiveArray::new::(offsets_buf.freeze(), Validity::NonNullable).into_array(); + + let mut sizes_buf = BufferMut::::with_capacity(num_rows); + for _ in 0..num_rows { + sizes_buf.push(dim_i32); + } + let sizes = + PrimitiveArray::new::(sizes_buf.freeze(), Validity::NonNullable).into_array(); + + ListViewArray::try_new(elements, offsets, sizes, Validity::NonNullable) + .unwrap() + .into_array() + } + + #[test] + fn list_view_input_is_rewrapped_as_vector_extension() { + // Simulates the post-parquet-ingest shape: the `emb` column arrives as a + // ListView, not a List. `list_to_vector_ext` must materialize it via + // `recursive_list_from_list_view` and then validate offsets on the flattened + // `List` form. + let list_view = list_view_f32(3, &[&[1.0, 2.0, 3.0], &[4.0, 5.0, 6.0]]); + let wrapped = list_to_vector_ext(list_view).unwrap(); + assert_eq!(wrapped.len(), 2); + let ext = wrapped.as_opt::().expect("returns Extension"); + assert!(matches!( + ext.storage_array().dtype(), + DType::FixedSizeList(_, 3, _) + )); + } + + #[test] + fn all_invalid_list_validity_is_rejected() { + // A list with `Validity::AllInvalid` means every row is null. The Vector + // extension type requires non-nullable elements at the FSL level, so we + // must reject this input rather than silently dropping the validity mask. + let elements = PrimitiveArray::new::( + BufferMut::::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0]).freeze(), + Validity::NonNullable, + ) + .into_array(); + let offsets = PrimitiveArray::new::( + BufferMut::::from_iter([0i32, 3, 6]).freeze(), + Validity::NonNullable, + ) + .into_array(); + let list = Array::::new(elements, offsets, Validity::AllInvalid).into_array(); + + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("list rows must be non-nullable"), + "unexpected error: {err}" + ); + } + + #[test] + fn non_float_element_type_is_rejected() { + // Build a List. + let elements = PrimitiveArray::new::( + BufferMut::::from_iter([1i32, 2, 3, 4]).freeze(), + Validity::NonNullable, + ) + .into_array(); + let offsets = PrimitiveArray::new::( + BufferMut::::from_iter([0i32, 2, 4]).freeze(), + Validity::NonNullable, + ) + .into_array(); + let list = Array::::new(elements, offsets, Validity::NonNullable).into_array(); + + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("element type must be float"), + "unexpected error: {err}", + ); + } + + #[test] + fn nullable_elements_with_real_nulls_are_rejected() { + // A `List` whose elements carry a real `Validity::Array` mask with + // at least one `false` bit has one or more actually-missing values. The + // rejection here is about runtime nulls, not dtype metadata: a nullable + // element dtype with all-valid runtime validity is accepted (see + // `nullable_element_dtype_with_all_valid_elements_is_accepted`), because + // parquet-ingested embeddings always arrive shaped that way even when + // every value is present. A real null, on the other hand, cannot be + // represented in the Vector extension FSL and must be rejected rather + // than silently dropped. + let element_validity = Validity::Array( + BoolArray::from_iter([true, true, false, true, true, true]).into_array(), + ); + let list = + list_f32_with_element_validity(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 3, element_validity); + + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("one or more actual null elements"), + "unexpected error: {err}" + ); + } + + #[test] + fn nullable_element_dtype_with_all_valid_elements_is_accepted() { + // This is the regression test for the Cohere parquet case: every real + // VectorDBBench parquet file arrives as `List` with + // `Validity::AllValid` elements because parquet has no non-nullable + // list-element logical type and arrow-rs propagates the nullable bit + // through `DType::from_arrow`. `list_to_vector_ext` must accept this + // shape by rewrapping the elements as non-nullable before building the + // FSL, rather than rejecting outright on the dtype metadata. + let list = + list_f32_with_element_validity(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 3, Validity::AllValid); + + let wrapped = list_to_vector_ext(list).unwrap(); + assert_eq!(wrapped.len(), 2); + let ext = wrapped.as_opt::().expect("returns Extension"); + assert!(matches!( + ext.storage_array().dtype(), + DType::FixedSizeList(_, 3, _) + )); + } + + #[test] + fn nullable_element_dtype_with_all_invalid_elements_is_rejected() { + // A `List` whose elements are `Validity::AllInvalid` means every + // value is missing. Rewrapping as non-nullable would silently drop the + // validity and produce bogus vectors, so this must be rejected. + let list = list_f32_with_element_validity( + &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + 3, + Validity::AllInvalid, + ); + + let err = list_to_vector_ext(list).unwrap_err().to_string(); + assert!( + err.contains("all-invalid elements"), + "unexpected error: {err}" + ); + } + + #[test] + fn chunked_input_with_mixed_dimensions_returns_error() { + let dim_three = list_f32(&[&[1.0, 2.0, 3.0]]); + let dim_two = list_f32(&[&[4.0, 5.0]]); + let chunked = + ChunkedArray::try_new(vec![dim_three.clone(), dim_two], dim_three.dtype().clone()) + .unwrap() + .into_array(); + + let err = list_to_vector_ext(chunked).unwrap_err().to_string(); + assert!(err.contains("Mismatched types"), "unexpected error: {err}"); + } +} diff --git a/vortex-bench/src/vector_dataset/mod.rs b/vortex-bench/src/vector_dataset/mod.rs index fe1e42a68d0..d826aa9fc5d 100644 --- a/vortex-bench/src/vector_dataset/mod.rs +++ b/vortex-bench/src/vector_dataset/mod.rs @@ -22,12 +22,14 @@ //! into per-flavor `.vortex` files, after which the scan driver re-opens those files per iteration. mod catalog; +mod convert; mod download; mod layout; mod paths; pub use catalog::ALL_VECTOR_DATASETS; pub use catalog::VectorDataset; +pub use convert::list_to_vector_ext; pub use download::DatasetPaths; pub use download::download; pub use download::neighbors_url; diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index f042f568c11..982127a4035 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -54,6 +54,7 @@ anyhow = { workspace = true } arrow-array = { workspace = true } divan = { workspace = true } fastlanes = { workspace = true } +futures = { workspace = true } mimalloc = { workspace = true } parquet = { workspace = true } paste = { workspace = true } @@ -64,6 +65,7 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } vortex = { path = ".", features = ["tokio"] } +vortex-bench = { workspace = true, features = ["unstable_encodings"] } vortex-tensor = { workspace = true } [features] @@ -83,6 +85,10 @@ unstable_encodings = [ "vortex-zstd?/unstable_encodings", ] +[[example]] +name = "turboquant_vector_search" +required-features = ["files", "tokio", "unstable_encodings"] + [[bench]] name = "single_encoding_throughput" harness = false diff --git a/vortex/examples/tracing_vortex.rs b/vortex/examples/tracing_vortex.rs index 7fd0267988e..f1227e2e017 100644 --- a/vortex/examples/tracing_vortex.rs +++ b/vortex/examples/tracing_vortex.rs @@ -91,8 +91,11 @@ async fn main() -> Result<(), Box> { Ok(()) } -/// Simulates application activity with various log levels and spans -#[expect(clippy::cognitive_complexity)] +/// Simulates application activity with various log levels and spans. +#[allow( + clippy::cognitive_complexity, + reason = "tracing sometimes triggers this" +)] async fn simulate_application_activity(user_id: u32) { // Simulate HTTP request handling let request_span = span!( diff --git a/vortex/examples/turboquant_vector_search.rs b/vortex/examples/turboquant_vector_search.rs new file mode 100644 index 00000000000..c4302e0c5ad --- /dev/null +++ b/vortex/examples/turboquant_vector_search.rs @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! TurboQuant vector-search roundtrip on a vector-embedding dataset. +//! +//! Load a parquet dataset (cohere-small), wrap the `emb` column as a `Vector` +//! extension, compress with BtrBlocks + TurboQuant, write to an in-memory Vortex file, then read +//! the file back twice: +//! +//! 1. plain scan — decode to canonical `FixedSizeList` and verify the per-element diff +//! against the original. TurboQuant is lossy, so we only check the reconstructed values are +//! within a tolerance. +//! 2. scan with a pushed-down cosine-similarity filter `cosine_similarity(emb, query) > thresh`. +//! The `CosineSimilarity` scalar fn is expressed directly as a filter `Expression`, so row +//! selection happens inside the scan rather than after materialization. +//! +//! The parquet file is cached under `vortex-bench/data//` after the first download. Run +//! with: +//! +//! ```sh +//! cargo run --example turboquant_vector_search \ +//! -p vortex --features unstable_encodings --release +//! ``` + +use std::path::PathBuf; +use std::time::Instant; + +use anyhow::Result; +use anyhow::bail; +use anyhow::ensure; +use futures::TryStreamExt; +use vortex::VortexSessionDefault; +use vortex::array::ArrayRef; +use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::ChunkedArray; +use vortex::array::arrays::ExtensionArray; +use vortex::array::arrays::FixedSizeListArray; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::StructArray; +use vortex::array::arrays::extension::ExtensionArrayExt; +use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::array::expr::col; +use vortex::array::expr::gt; +use vortex::array::expr::lit; +use vortex::array::extension::EmptyMetadata; +use vortex::array::scalar::Scalar; +use vortex::array::scalar_fn::EmptyOptions; +use vortex::array::scalar_fn::ScalarFnVTable; +use vortex::array::scalar_fn::ScalarFnVTableExt; +use vortex::buffer::ByteBuffer; +use vortex::buffer::ByteBufferMut; +use vortex::dtype::DType; +use vortex::dtype::Nullability; +use vortex::dtype::PType; +use vortex::file::ALLOWED_ENCODINGS; +use vortex::file::OpenOptionsSessionExt; +use vortex::file::WriteOptionsSessionExt; +use vortex::file::WriteStrategyBuilder; +use vortex::io::session::RuntimeSessionExt; +use vortex::session::VortexSession; +use vortex_array::ExecutionCtx; +use vortex_array::builtins::ArrayBuiltins; +use vortex_bench::conversions::parquet_to_vortex_chunks; +use vortex_bench::vector_dataset; +use vortex_bench::vector_dataset::TrainLayout; +use vortex_bench::vector_dataset::VectorDataset; +use vortex_bench::vector_dataset::list_to_vector_ext; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_error::VortexExpect; +use vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity; +use vortex_tensor::scalar_fns::l2_denorm::L2Denorm; +use vortex_tensor::scalar_fns::sorf_transform::SorfTransform; +use vortex_tensor::vector::AnyVector; +use vortex_tensor::vector::Vector; + +/// Cosine threshold for the demo filter. The query comes from the test split, so it may or may not +/// have nearby rows in the train split. +const COSINE_THRESHOLD: f32 = 0.90; + +/// Slack for checking decoded rows against a predicate that was evaluated on TurboQuant's lossy +/// readthrough representation. +const COSINE_THRESHOLD_TOL: f32 = 0.02; + +/// Regression ceiling on the decoded vs original max-abs-diff for 8-bit TurboQuant on 768-dim f32 +/// embeddings. Observed on cohere-small: ~0.10. Pinned with slack so the check catches large +/// quality regressions without flapping on normal run-to-run variation. +const MAX_ABS_DIFF_TOL: f32 = 0.2; + +#[tokio::main] +async fn main() -> Result<()> { + // Opt in to registering the tensor scalar-fn array plugins before building the session. + // Without this, the TurboQuant-compressed `emb` column cannot be serialized into the Vortex + // file or deserialized on read. + // + // SAFETY: single-threaded setup before any other thread exists. + unsafe { std::env::set_var(vortex_tensor::SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV, "1") }; + + let session = VortexSession::default().with_tokio(); + vortex_tensor::initialize(&session); + println!("session initialized with tensor plugins"); + + let dataset = VectorDataset::CohereSmall100k; // This is one of the smaller datasets. + + // Download the source parquet files. + let dataset_paths = vector_dataset::download(dataset, TrainLayout::Single).await?; + let (_id, query_vector) = get_query_vector(dataset_paths.test).await?; + println!( + "query vector selected (id = {_id}, dim = {})", + query_vector.len() + ); + + // Bring the parquet file into memory so that we can write it as a vortex file (after prep). + let single_train_file = dataset_paths + .train_files + .first() + .vortex_expect("we know that there must be a file here") + .clone(); + + println!("reading parquet into chunked array..."); + let chunked_table = parquet_to_vortex_chunks(single_train_file) + .await? + .into_array(); + let len = chunked_table.len(); + println!("parquet loaded: {len} rows"); + + let id = chunked_table.get_item("id")?; + let emb = chunked_table.get_item("emb")?; + + println!("converting emb column to Vector extension type..."); + let vector_array = list_to_vector_ext(emb)?; + + let fields = [("id", id), ("emb", vector_array)]; + let struct_array = StructArray::from_fields(&fields)?.into_array(); + + println!("compressing with TurboQuant and writing to in-memory Vortex file..."); + let bytes = write_turboquant(&session, struct_array.clone().into_array()).await?; + println!("vortex file written: {} bytes", bytes.len()); + + println!("verifying roundtrip fidelity..."); + verify_roundtrip(&session, &bytes, struct_array.clone()).await?; + + println!("verifying filter pushdown with cosine similarity..."); + verify_filter_pushdown(&session, &bytes, &query_vector, struct_array).await?; + + println!("all checks passed!"); + Ok(()) +} + +async fn get_query_vector(query_vectors_path: PathBuf) -> Result<(usize, Vec)> { + let test_vectors = parquet_to_vortex_chunks(query_vectors_path).await?; + + // Get a random query vector. + let idx = rand::random_range(0..test_vectors.len()); + let struct_scalar = test_vectors.scalar_at(idx)?; + let id_scalar = struct_scalar + .as_struct() + .field("id") + .vortex_expect("test parquet file missing `id` field"); + + ensure!( + id_scalar + .as_primitive() + .as_::() + .vortex_expect("id was not a i64") + == idx as i64 + ); + + let emb_scalar = struct_scalar + .as_struct() + .field("emb") + .vortex_expect("test parquet file missing `emb` field"); + + // Pack into a `Vec`. + let query_vector: Vec = emb_scalar + .as_list() + .elements() + .vortex_expect("somehow had a null test vector") + .iter() + .map(|element| { + element + .as_primitive() + .as_::() + .vortex_expect("value was not a f32") + }) + .collect(); + + Ok((idx, query_vector)) +} + +async fn write_turboquant(session: &VortexSession, array: ArrayRef) -> Result { + let compressor = BtrBlocksCompressorBuilder::default() + .with_turboquant() + .build(); + + // TurboQuant produces `L2Denorm(SorfTransform(FSL(Dict(...))), norms)`. The default write + // allow-list only covers canonical/compressed array encodings, so the tensor scalar-fn + // encodings it emits get rejected during normalization. Extend the set with the two encoding + // IDs this scheme actually uses. + let mut allowed = ALLOWED_ENCODINGS.clone(); + allowed.insert(L2Denorm.id()); + allowed.insert(SorfTransform.id()); + + let strategy = WriteStrategyBuilder::default() + .with_compressor(compressor) + .with_allow_encodings(allowed) + .build(); + + let mut buf = ByteBufferMut::empty(); + session + .write_options() + .with_strategy(strategy) + .write(&mut buf, array.to_array_stream()) + .await?; + Ok(buf.freeze()) +} + +async fn verify_roundtrip( + session: &VortexSession, + bytes: &ByteBuffer, + original: ArrayRef, +) -> Result<()> { + let chunks: Vec = session + .open_options() + .open_buffer(bytes.clone())? + .scan()? + .into_array_stream()? + .try_collect() + .await?; + + let mut ctx = session.create_execution_ctx(); + + let read: StructArray = ChunkedArray::try_new(chunks, original.dtype().clone())? + .into_array() + .execute(&mut ctx)?; + let original: StructArray = original.execute(&mut ctx)?; + ensure!(read.len() == original.len()); + + let read_emb = read.unmasked_field_by_name("emb")?.clone(); + let original_emb = original.unmasked_field_by_name("emb")?.clone(); + + let decoded = flatten_vector_column(read_emb, &mut ctx)?; + let original_decoded = flatten_vector_column(original_emb, &mut ctx)?; + + let (max_abs, mean_abs) = diff_stats(&original_decoded, &decoded); + println!( + "roundtrip fidelity: max_abs_diff = {max_abs:.6}, mean_abs_diff = {mean_abs:.6} \ + (tol = {MAX_ABS_DIFF_TOL})" + ); + if max_abs > MAX_ABS_DIFF_TOL { + bail!("TurboQuant max_abs_diff {max_abs} exceeds tolerance {MAX_ABS_DIFF_TOL}"); + } + + Ok(()) +} + +async fn verify_filter_pushdown( + session: &VortexSession, + bytes: &ByteBuffer, + query: &[f32], + original: ArrayRef, +) -> Result<()> { + // Build the filter as `cosine_similarity(emb, ) > threshold`. The RHS of + // `CosineSimilarity` is a `lit(...)` wrapping a `Vector` scalar; during scan + // evaluation the Literal expands to a ConstantArray whose row count matches the current batch, + // satisfying `CosineSimilarity`'s same-length requirement. The entire expression is pushed + // through `with_filter`, so row selection happens inside the scan rather than after the whole + // column is materialized. + println!("query: {}", preview_vector(query)); + + let query_scalar = build_query_vector_scalar(query)?; + let cosine_expr = CosineSimilarity.new_expr(EmptyOptions, [col("emb"), lit(query_scalar)]); + let filter = gt(cosine_expr, lit(COSINE_THRESHOLD)); + + let scan_start = Instant::now(); + let chunks: Vec = session + .open_options() + .open_buffer(bytes.clone())? + .scan()? + .with_filter(filter) + .into_array_stream()? + .try_collect() + .await?; + let scan_ms = scan_start.elapsed().as_secs_f64() * 1e3; + + let hits: usize = chunks.iter().map(|c| c.len()).sum(); + println!( + "pushed down `cosine_similarity(emb, query) > {COSINE_THRESHOLD}`: {hits} rows survived \ + in {scan_ms:.2} ms" + ); + if hits == 0 { + println!(" no rows survived the filter for this random query"); + return Ok(()); + } + + // Materialize the matching rows and dump each `emb` vector so the reader can see what the + // pushed-down filter actually selected. Vectors are truncated to the first few elements since + // DIM is typically large. + let mut ctx = session.create_execution_ctx(); + let filtered: StructArray = ChunkedArray::try_new(chunks, original.dtype().clone())? + .into_array() + .execute(&mut ctx)?; + + let emb = filtered.unmasked_field_by_name("emb")?.clone(); + let flat = flatten_vector_column(emb, &mut ctx)?; + + let dim = query.len(); + for (i, row) in flat.chunks_exact(dim).enumerate() { + let cos = cosine_similarity(query, row); + ensure!( + cos >= COSINE_THRESHOLD - COSINE_THRESHOLD_TOL, + "filtered row {i} had decoded cosine {cos:+.6}, below threshold {COSINE_THRESHOLD} \ + by more than tolerance {COSINE_THRESHOLD_TOL}" + ); + println!(" match {i}: cos = {cos:+.6} {}", preview_vector(row)); + } + + Ok(()) +} + +/// Plain `dot(a, b) / (||a|| * ||b||)` over two equal-length f32 slices. Used purely for reporting +/// — the actual row selection is done inside the scan by the pushed-down `CosineSimilarity` +/// expression. This lets the reader cross-check that the surviving rows really do clear the +/// threshold once decoded. +fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + let mut dot = 0.0f32; + let mut a_sq = 0.0f32; + let mut b_sq = 0.0f32; + for (&x, &y) in a.iter().zip(b) { + dot += x * y; + a_sq += x * x; + b_sq += y * y; + } + dot / (a_sq.sqrt() * b_sq.sqrt()) +} + +/// Render a vector as `[v0, v1, ..., vN-1, vN]` with the first 4 and last 1 elements at 4-decimal +/// precision. Keeps the output compact for high-dim embeddings while still giving the reader +/// something concrete to eyeball. +fn preview_vector(row: &[f32]) -> String { + let dim = row.len(); + if dim <= 5 { + return format!("[{}] (dim = {dim})", fmt_slice(row)); + } + format!( + "[{}, ..., {}] (dim = {dim})", + fmt_slice(&row[..4]), + fmt_slice(&row[dim - 1..]) + ) +} + +fn fmt_slice(s: &[f32]) -> String { + s.iter() + .map(|v| format!("{v:+.4}")) + .collect::>() + .join(", ") +} + +/// Wrap a query vector in a `Vector` extension scalar suitable for use as the RHS of a +/// `CosineSimilarity` filter expression via `lit(...)`. +fn build_query_vector_scalar(query: &[f32]) -> Result { + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let children: Vec = query + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let fsl_scalar = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + Ok(Scalar::extension::(EmptyMetadata, fsl_scalar)) +} + +/// Decode a `Vector` extension array's storage down to its flat f32 buffer. +fn flatten_vector_column(emb: ArrayRef, ctx: &mut ExecutionCtx) -> Result> { + let ext: ExtensionArray = emb.execute(ctx)?; + ensure!(ext.ext_dtype().is::()); + + let fsl: FixedSizeListArray = ext.storage_array().clone().execute(ctx)?; + let elements: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + Ok(elements.as_slice::().to_vec()) +} + +fn diff_stats(original: &[f32], decoded: &[f32]) -> (f32, f32) { + assert_eq!(original.len(), decoded.len()); + let (sum_abs, max_abs) = + original + .iter() + .zip(decoded) + .fold((0.0f32, 0.0f32), |(sum, peak), (&orig, &dec)| { + let diff = (orig - dec).abs(); + (sum + diff, peak.max(diff)) + }); + let mean_abs = sum_abs / original.len() as f32; + (max_abs, mean_abs) +} From d7f07debdf60c4ef2267428947c012f2b77c9f64 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Thu, 16 Apr 2026 15:43:30 +0200 Subject: [PATCH 067/250] vortex-array: Attempt casting inner array to target for extensions (#7469) ## Summary Getting ``` No CastKernel to cast canonical array vortex.ext from vortex.timestamp[ns, tz=UTC](i64) to i64 ``` When running a datafusion query like: ``` SELECT cast(timestamp as bigint) as timestamp from table; ``` Where the timestamp array is a utc timestamp type column. Since the inner array of a timestamp array is int64 already, all we really need to do is try to attempt to cast the inner array of the extension type to the target type. That's exactly what this PR does. ## Testing Added a unit test. Signed-off-by: Frederic Branczyk --- .../src/arrays/extension/compute/cast.rs | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index f2705f5c7b4..eda55a73f50 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -17,7 +17,9 @@ impl CastReduce for Extension { dtype: &DType, ) -> vortex_error::VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { - return Ok(None); + // Target is not the same extension type. + // Delegate to the storage array's cast. + return Ok(Some(array.storage_array().cast(dtype.clone())?)); } let DType::Extension(ext_dtype) = dtype else { @@ -51,9 +53,12 @@ mod tests { use super::*; use crate::IntoArray; use crate::arrays::PrimitiveArray; + use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::compute::conformance::cast::test_cast_conformance; + use crate::dtype::DType; use crate::dtype::Nullability; + use crate::dtype::PType; use crate::extension::datetime::TimeUnit; use crate::extension::datetime::Timestamp; @@ -108,6 +113,26 @@ mod tests { ); } + #[test] + fn cast_timestamp_to_i64() -> vortex_error::VortexResult<()> { + let ext_dtype = Timestamp::new_with_tz( + TimeUnit::Nanoseconds, + Some("UTC".into()), + Nullability::NonNullable, + ) + .erased(); + let storage = buffer![1i64, 2, 3].into_array(); + let arr = ExtensionArray::new(ext_dtype, storage).into_array(); + + let result = arr.cast(DType::Primitive(PType::I64, Nullability::NonNullable))?; + assert_eq!( + result.dtype(), + &DType::Primitive(PType::I64, Nullability::NonNullable) + ); + assert_arrays_eq!(result, buffer![1i64, 2, 3].into_array()); + Ok(()) + } + #[rstest] #[case(create_timestamp_array(TimeUnit::Milliseconds, false))] #[case(create_timestamp_array(TimeUnit::Microseconds, true))] From 3cd74b985faa00626172b2351bf6e40d5b16efd6 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 10:08:43 -0400 Subject: [PATCH 068/250] Fix scalar_at in example (#7471) ## Summary Semantic merge. ## Testing N/A Signed-off-by: Connor Tsui --- vortex/examples/turboquant_vector_search.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/vortex/examples/turboquant_vector_search.rs b/vortex/examples/turboquant_vector_search.rs index c4302e0c5ad..47e1e56ba3a 100644 --- a/vortex/examples/turboquant_vector_search.rs +++ b/vortex/examples/turboquant_vector_search.rs @@ -77,7 +77,7 @@ use vortex_tensor::vector::Vector; /// Cosine threshold for the demo filter. The query comes from the test split, so it may or may not /// have nearby rows in the train split. -const COSINE_THRESHOLD: f32 = 0.90; +const COSINE_THRESHOLD: f32 = 0.85; /// Slack for checking decoded rows against a predicate that was evaluated on TurboQuant's lossy /// readthrough representation. @@ -99,13 +99,14 @@ async fn main() -> Result<()> { let session = VortexSession::default().with_tokio(); vortex_tensor::initialize(&session); + let mut ctx = session.create_execution_ctx(); println!("session initialized with tensor plugins"); let dataset = VectorDataset::CohereSmall100k; // This is one of the smaller datasets. // Download the source parquet files. let dataset_paths = vector_dataset::download(dataset, TrainLayout::Single).await?; - let (_id, query_vector) = get_query_vector(dataset_paths.test).await?; + let (_id, query_vector) = get_query_vector(dataset_paths.test, &mut ctx).await?; println!( "query vector selected (id = {_id}, dim = {})", query_vector.len() @@ -142,18 +143,21 @@ async fn main() -> Result<()> { verify_roundtrip(&session, &bytes, struct_array.clone()).await?; println!("verifying filter pushdown with cosine similarity..."); - verify_filter_pushdown(&session, &bytes, &query_vector, struct_array).await?; + verify_filter_pushdown(&session, &bytes, &query_vector, struct_array, &mut ctx).await?; println!("all checks passed!"); Ok(()) } -async fn get_query_vector(query_vectors_path: PathBuf) -> Result<(usize, Vec)> { +async fn get_query_vector( + query_vectors_path: PathBuf, + ctx: &mut ExecutionCtx, +) -> Result<(usize, Vec)> { let test_vectors = parquet_to_vortex_chunks(query_vectors_path).await?; // Get a random query vector. let idx = rand::random_range(0..test_vectors.len()); - let struct_scalar = test_vectors.scalar_at(idx)?; + let struct_scalar = test_vectors.execute_scalar(idx, ctx)?; let id_scalar = struct_scalar .as_struct() .field("id") @@ -260,6 +264,7 @@ async fn verify_filter_pushdown( bytes: &ByteBuffer, query: &[f32], original: ArrayRef, + ctx: &mut ExecutionCtx, ) -> Result<()> { // Build the filter as `cosine_similarity(emb, ) > threshold`. The RHS of // `CosineSimilarity` is a `lit(...)` wrapping a `Vector` scalar; during scan @@ -297,13 +302,12 @@ async fn verify_filter_pushdown( // Materialize the matching rows and dump each `emb` vector so the reader can see what the // pushed-down filter actually selected. Vectors are truncated to the first few elements since // DIM is typically large. - let mut ctx = session.create_execution_ctx(); let filtered: StructArray = ChunkedArray::try_new(chunks, original.dtype().clone())? .into_array() - .execute(&mut ctx)?; + .execute(ctx)?; let emb = filtered.unmasked_field_by_name("emb")?.clone(); - let flat = flatten_vector_column(emb, &mut ctx)?; + let flat = flatten_vector_column(emb, ctx)?; let dim = query.len(); for (i, row) in flat.chunks_exact(dim).enumerate() { From 8b09d0c6f14a579ae60f38abb45defebbb1bc5c4 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Apr 2026 15:30:35 +0100 Subject: [PATCH 069/250] Split up the monster `ci.yml` workflow (#7472) ## Summary `ci.yml` is reaching 1000 lines of YAML, which no one wants. This PR splits into 5 files: 1. ci.yml - now only contains linters and test runs 2. codspeed.yml - just codspeed benchmarks 3. cuda.yaml - all the cuda incantations 4. publish-dry-runs.yaml - all the dark magic that we have to try and make sure we can actually publish 5. rust-instrumented.yaml - rust-specific jobs that require nightly and some extra instrumentations (coverage, miri, etc) The real magic here is that there are no changes to the job names, so all rules still apply! This also makes github retries much easier, as github only allows retries after the whole workflow finishes. --------- Signed-off-by: Adam Gutglick --- .github/workflows/ci.yml | 478 +----------------------- .github/workflows/codspeed.yml | 67 ++++ .github/workflows/cuda.yaml | 154 ++++++++ .github/workflows/publish-dry-runs.yml | 123 ++++++ .github/workflows/rust-instrumented.yml | 221 +++++++++++ 5 files changed, 567 insertions(+), 476 deletions(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 .github/workflows/cuda.yaml create mode 100644 .github/workflows/publish-dry-runs.yml create mode 100644 .github/workflows/rust-instrumented.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17c557c8480..1067a62e1ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,9 +1,8 @@ -name: CI +name: Linters and Tests # Concurrency control: # - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. -# - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks -# run on every commit for our benchmarks website. +# - Push to develop: runs queue sequentially, never cancelled. # - `workflow_dispatch`: groups by branch and queues if run on develop. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -114,47 +113,6 @@ jobs: uv run --all-packages make html working-directory: docs/ - python-wheel-build: - name: "Python (wheel build)" - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6 - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref_name == 'develop' }} - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - uses: mlugg/setup-zig@v2.2.1 - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 - with: - sync: false - prune-cache: false - - name: Ensure wheel and sdist can be built on Linux - PyVortex - shell: bash - run: | - echo "Clearing wheel target directory" - rm -rf ../target/wheels/ - - uv venv - uv tool run maturin@1.10 build --interpreter python3.11 --zig - uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist - - file_count=$(ls -1 ../target/wheels/ | wc -l) - - if [[ $file_count -ne 2 ]]; then - echo "Unexpected number of files detected ${file_count}:" - ls ../target/wheels/ - exit 1 - else - echo "Generated two files" - fi - working-directory: vortex-python/ - rust-docs: name: "Rust (docs)" timeout-minutes: 30 @@ -333,313 +291,6 @@ jobs: exit 1 fi - rust-coverage: - name: "Rust tests (coverage) (${{ matrix.suite }})" - timeout-minutes: 30 - permissions: - id-token: write - strategy: - matrix: - include: - - suite: tests - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) - || 'ubuntu-latest' }} - env: - RUSTFLAGS: "-Cinstrument-coverage -A warnings" - CARGO_INCREMENTAL: 0 # Disable incremental compilation to get accurate coverage - LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" - GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - name: Rust Tests - if: ${{ matrix.suite == 'tests' }} - run: | - cargo nextest run --locked --workspace --all-features --no-fail-fast - - name: Generate coverage report - run: | - grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ - --threads $(nproc) \ - --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \ - --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \ - --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \ - --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \ - --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \ - --ignore benchmarks/* --ignore 'vortex-test/*' \ - -o ${{ env.GRCOV_OUTPUT_FILE }} - - name: Codecov - uses: codecov/codecov-action@v5 - with: - name: run-${{ matrix.suite }} - files: ${{ env.GRCOV_OUTPUT_FILE }} - disable_search: true - flags: ${{ matrix.suite }} - use_oidc: true - - rust-test-sanitizer: - strategy: - fail-fast: false - matrix: - include: - # We don't run memory sanitizer as it provides many false positives - # for std - - sanitizer: asan - sanitizer_flags: "-Zsanitizer=address,leak" - - sanitizer: tsan - sanitizer_flags: "-Zsanitizer=thread" - name: "Rust tests (${{ matrix.sanitizer }})" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 30 - env: - ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" - LSAN_OPTIONS: "report_objects=1" - ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - MSAN_OPTIONS: "symbolize=1" - MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt" - TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - VORTEX_SKIP_SLOW_TESTS: "1" - # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins - # unset -Zsanitizer flag and we should allow that. - RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - name: Install Rust nightly toolchain - run: | - rustup toolchain install $NIGHTLY_TOOLCHAIN - rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview - - name: Build tests with sanitizer - run: | - RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \ - --target x86_64-unknown-linux-gnu -Zbuild-std \ - -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array - - - name: Run tests with sanitizer - run: | - RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \ - --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ - -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array - - # vortex-ffi requires --no-default-features as otherwise we pull in - # Mimalloc which interferes with sanitizers - # cargo nextest reports less sanitizer issues than cargo test - # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in - - name: Run vortex-ffi tests with sanitizer - run: | - RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \ - --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ - -p vortex-ffi -- --no-capture - - rust-ffi-test-sanitizer: - strategy: - fail-fast: false - matrix: - include: - # We don't run memory sanitizer as it's clang-only and provides many - # false positives for Catch2 - - sanitizer: asan - sanitizer_flags: "-Zsanitizer=address,leak" - - sanitizer: tsan - sanitizer_flags: "-Zsanitizer=thread" - name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})" - timeout-minutes: 30 - env: - ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" - LSAN_OPTIONS: "report_objects=1" - ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - MSAN_OPTIONS: "symbolize=1" - MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - TSAN_OPTIONS: "symbolize=1" - TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - VORTEX_SKIP_SLOW_TESTS: "1" - # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins - # unset -Zsanitizer flag and we should allow that. - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - name: Install rustfilt - run: | - cargo install rustfilt - - name: Install Rust nightly toolchain - run: | - rustup toolchain install $NIGHTLY_TOOLCHAIN - rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview - - name: Build FFI library - run: | - # TODO(myrrc): remove --no-default-features - RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ - -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ - ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ - --target x86_64-unknown-linux-gnu -Zbuild-std \ - -p vortex-ffi - - name: Build FFI library tests - run: | - cd vortex-ffi - cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" - cmake --build build -j - - name: Run tests - run: | - set -o pipefail - ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i- - - cuda-build-lint: - if: github.repository == 'vortex-data/vortex' - name: "CUDA build & lint" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/check-rebuild - with: - command: >- - cargo build --profile ci --locked --all-features --all-targets - -p vortex-cuda -p vortex-cub -p vortex-nvcomp - -p gpu-scan-cli -p vortex-test-e2e-cuda - - name: Clippy CUDA crates - run: | - cargo clippy --profile ci --locked --all-features --all-targets \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p gpu-scan-cli \ - -p vortex-test-e2e-cuda \ - -- -D warnings - - cuda-test: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install nextest - uses: taiki-e/install-action@v2 - with: - tool: nextest - - name: Rust Tests - env: - FLAT_LAYOUT_INLINE_ARRAY_NODE: true - run: | - cargo nextest run \ - --cargo-profile ci \ - --locked \ - -p vortex-file \ - -p vortex-cuda \ - -p vortex-cub \ - -p vortex-nvcomp \ - -p vortex-test-e2e-cuda \ - --all-features \ - --no-fail-fast \ - --target x86_64-unknown-linux-gnu \ - --verbose - - cuda-test-sanitizer: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (${{ matrix.sanitizer }})" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer - strategy: - fail-fast: false - matrix: - include: - - sanitizer: memcheck - runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" - # TODO(joe): try to re-enable racecheck, it is hanging in CI. - # - sanitizer: racecheck - # runner_flags: "--tool racecheck --error-exitcode 1" - - sanitizer: synccheck - runner_flags: "--tool synccheck --error-exitcode 1" - - sanitizer: initcheck - runner_flags: "--tool initcheck --error-exitcode 1" - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build tests - run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run - - name: "CUDA - ${{ matrix.sanitizer }}" - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" - run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu - - cuda-test-cudf: - if: github.repository == 'vortex-data/vortex' - name: "CUDA tests (cudf)" - timeout-minutes: 30 - runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf - steps: - - uses: runs-on/action@v2 - with: - sccache: s3 - - name: Display NVIDIA SMI details - run: | - nvidia-smi - nvidia-smi -L - nvidia-smi -q -d Memory - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build cudf test library - run: cargo build --profile ci --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu - - name: Download and run cudf-test-harness - run: | - curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz - cd cudf-test-harness-x86_64 - compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so - rust-test-other: name: "Rust tests (${{ matrix.os }})" timeout-minutes: 30 @@ -711,49 +362,6 @@ jobs: - run: ./gradlew test --parallel working-directory: ./java - bench-codspeed: - strategy: - matrix: - include: - - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" } - - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } - - { shard: 3, name: "Main library", packages: "vortex" } - - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } - - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } - - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } - - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } - - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" } - name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" - timeout-minutes: 30 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-prebuild - - uses: ./.github/actions/system-info - - name: Install Codspeed - uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 - with: - tool: cargo-codspeed - - name: Build benchmarks - env: - RUSTFLAGS: "-C target-feature=+avx2" - run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench - - name: Run benchmarks - uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2 - with: - run: bash scripts/bench-taskset.sh cargo codspeed run - token: ${{ secrets.CODSPEED_TOKEN }} - mode: "simulation" - license-check-and-audit-check: name: License Check and Audit Check runs-on: ubuntu-latest @@ -842,29 +450,6 @@ jobs: - run: wasmer run ./target/wasm32-wasip1/ci/wasm-test.wasm working-directory: ./wasm-test - miri: - name: "Rust tests (miri)" - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) - || 'ubuntu-latest' }} - timeout-minutes: 30 - env: - MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE - RUSTFLAGS: "-A warnings" - RUST_BACKTRACE: full - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - name: Install nightly with miri - run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri - - name: Run Miri - run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi - generated-files: name: "Check generated source files are up to date" runs-on: >- @@ -928,62 +513,3 @@ jobs: cmake -Bbuild -DRUST_BUILD_PROFILE=ci cmake --build build -j $(nproc) ctest --test-dir build -j $(nproc) - - check-java-publish-build: - runs-on: ${{ matrix.target.runs-on }} - container: - image: "ubuntu:20.04" - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - target: - - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } - - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - uses: ./.github/actions/prepare-java-linux - - uses: actions/setup-java@v5 - with: - distribution: "corretto" - java-version: "17" - - uses: ./.github/actions/setup-rust - with: - targets: ${{ matrix.target.target }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - run: cargo build --profile ci --package vortex-jni - - compat-check: - name: "Compat check" - uses: ./.github/workflows/compat-validation.yml - with: - mode: last - - rust-publish-dry-run: - name: "Rust publish dry-run" - timeout-minutes: 30 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) - || 'ubuntu-latest' }} - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-prebuild - - name: Install cargo-edit - uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 - with: - tool: cargo-edit - - name: Set Version - run: | - # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io - cargo set-version --workspace 0.100000.0 - - name: Test Release - run: | - cargo publish --dry-run --no-verify --allow-dirty --workspace diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000000..48ad834f3d1 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,67 @@ +name: Codspeed Benchmarking + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. This allows us to have benchmarks +# run on every commit for our benchmarks website. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + NIGHTLY_TOOLCHAIN: nightly-2026-02-05 + +jobs: + bench-codspeed: + strategy: + matrix: + include: + - { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" } + - { shard: 2, name: "Arrays", packages: "vortex-array", features: "--features _test-harness" } + - { shard: 3, name: "Main library", packages: "vortex" } + - { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" } + - { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" } + - { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" } + - { shard: 7, name: "Encodings 4", packages: "vortex-sparse vortex-zigzag vortex-zstd" } + - { shard: 8, name: "Storage formats", packages: "vortex-flatbuffers vortex-proto vortex-btrblocks" } + name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=bench-codspeed-{1}', github.run_id, matrix.shard) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + - uses: ./.github/actions/setup-prebuild + - uses: ./.github/actions/system-info + - name: Install Codspeed + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 + with: + tool: cargo-codspeed + - name: Build benchmarks + env: + RUSTFLAGS: "-C target-feature=+avx2" + run: cargo codspeed build ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) --profile bench + - name: Run benchmarks + uses: CodSpeedHQ/action@d872884a306dd4853acf0f584f4b706cf0cc72a2 + with: + run: bash scripts/bench-taskset.sh cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} + mode: "simulation" diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml new file mode 100644 index 00000000000..c1b33b37746 --- /dev/null +++ b/.github/workflows/cuda.yaml @@ -0,0 +1,154 @@ +name: CUDA + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + cuda-build-lint: + if: github.repository == 'vortex-data/vortex' + name: "CUDA build & lint" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/check-rebuild + with: + command: >- + cargo build --profile ci --locked --all-features --all-targets + -p vortex-cuda -p vortex-cub -p vortex-nvcomp + -p gpu-scan-cli -p vortex-test-e2e-cuda + - name: Clippy CUDA crates + run: | + cargo clippy --profile ci --locked --all-features --all-targets \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p gpu-scan-cli \ + -p vortex-test-e2e-cuda \ + -- -D warnings + + cuda-test: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install nextest + uses: taiki-e/install-action@v2 + with: + tool: nextest + - name: Rust Tests + env: + FLAT_LAYOUT_INLINE_ARRAY_NODE: true + run: | + cargo nextest run \ + --cargo-profile ci \ + --locked \ + -p vortex-file \ + -p vortex-cuda \ + -p vortex-cub \ + -p vortex-nvcomp \ + -p vortex-test-e2e-cuda \ + --all-features \ + --no-fail-fast \ + --target x86_64-unknown-linux-gnu \ + --verbose + + cuda-test-sanitizer: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (${{ matrix.sanitizer }})" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer + strategy: + fail-fast: false + matrix: + include: + - sanitizer: memcheck + runner_flags: "--tool memcheck --leak-check=full --error-exitcode 1" + # TODO(joe): try to re-enable racecheck, it is hanging in CI. + # - sanitizer: racecheck + # runner_flags: "--tool racecheck --error-exitcode 1" + - sanitizer: synccheck + runner_flags: "--tool synccheck --error-exitcode 1" + - sanitizer: initcheck + runner_flags: "--tool initcheck --error-exitcode 1" + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build tests + run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu --no-run + - name: "CUDA - ${{ matrix.sanitizer }}" + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "compute-sanitizer ${{ matrix.runner_flags }}" + run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu + + cuda-test-cudf: + if: github.repository == 'vortex-data/vortex' + name: "CUDA tests (cudf)" + timeout-minutes: 30 + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf + steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build cudf test library + run: cargo build --profile ci --locked -p vortex-test-e2e-cuda --target x86_64-unknown-linux-gnu + - name: Download and run cudf-test-harness + run: | + curl -fsSL https://github.com/vortex-data/cudf-test-harness/releases/latest/download/cudf-test-harness-x86_64.tar.gz | tar -xz + cd cudf-test-harness-x86_64 + compute-sanitizer --tool memcheck --error-exitcode 1 ./cudf-test-harness check $GITHUB_WORKSPACE/target/x86_64-unknown-linux-gnu/ci/libvortex_test_e2e_cuda.so diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml new file mode 100644 index 00000000000..bea9f4e9b02 --- /dev/null +++ b/.github/workflows/publish-dry-runs.yml @@ -0,0 +1,123 @@ +name: Publish Dry Runs + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + NIGHTLY_TOOLCHAIN: nightly-2026-02-05 + +jobs: + python-wheel-build: + name: "Python (wheel build)" + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref_name == 'develop' }} + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - uses: mlugg/setup-zig@v2.2.1 + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + with: + sync: false + prune-cache: false + - name: Ensure wheel and sdist can be built on Linux - PyVortex + shell: bash + run: | + echo "Clearing wheel target directory" + rm -rf ../target/wheels/ + + uv venv + uv tool run maturin@1.10 build --interpreter python3.11 --zig + uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist + + file_count=$(ls -1 ../target/wheels/ | wc -l) + + if [[ $file_count -ne 2 ]]; then + echo "Unexpected number of files detected ${file_count}:" + ls ../target/wheels/ + exit 1 + else + echo "Generated two files" + fi + working-directory: vortex-python/ + + check-java-publish-build: + runs-on: ${{ matrix.target.runs-on }} + container: + image: "ubuntu:20.04" + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + target: + - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } + - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: ./.github/actions/prepare-java-linux + - uses: actions/setup-java@v5 + with: + distribution: "corretto" + java-version: "17" + - uses: ./.github/actions/setup-rust + with: + targets: ${{ matrix.target.target }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: "false" + - run: cargo build --profile ci --package vortex-jni + + compat-check: + name: "Compat check" + uses: ./.github/workflows/compat-validation.yml + with: + mode: last + + rust-publish-dry-run: + name: "Rust publish dry-run" + timeout-minutes: 30 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/tag=rust-publish-dry-run', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - name: Install cargo-edit + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 + with: + tool: cargo-edit + - name: Set Version + run: | + # This is mostly a dummy version, we don't actually publish anything but it cannot exist in crates.io + cargo set-version --workspace 0.100000.0 + - name: Test Release + run: | + cargo publish --dry-run --no-verify --allow-dirty --workspace diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml new file mode 100644 index 00000000000..20bd674093a --- /dev/null +++ b/.github/workflows/rust-instrumented.yml @@ -0,0 +1,221 @@ +name: Rust Instrumented + +# Concurrency control: +# - PRs: new commits on a feature branch will cancel in-progress (outdated) runs. +# - Push to develop: runs queue sequentially, never cancelled. +# - `workflow_dispatch`: groups by branch and queues if run on develop. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + NIGHTLY_TOOLCHAIN: nightly-2026-02-05 + +jobs: + rust-coverage: + name: "Rust tests (coverage) (${{ matrix.suite }})" + timeout-minutes: 30 + permissions: + id-token: write + strategy: + matrix: + include: + - suite: tests + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/tag=rust-coverage-suite-{1}', github.run_id, matrix.suite) + || 'ubuntu-latest' }} + env: + RUSTFLAGS: "-Cinstrument-coverage -A warnings" + CARGO_INCREMENTAL: 0 # Disable incremental compilation to get accurate coverage + LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" + GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - name: Rust Tests + if: ${{ matrix.suite == 'tests' }} + run: | + cargo nextest run --locked --workspace --all-features --no-fail-fast + - name: Generate coverage report + run: | + grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \ + --threads $(nproc) \ + --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'vortex-bench/*' \ + --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \ + --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \ + --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \ + --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \ + --ignore benchmarks/* --ignore 'vortex-test/*' \ + -o ${{ env.GRCOV_OUTPUT_FILE }} + - name: Codecov + uses: codecov/codecov-action@v5 + with: + name: run-${{ matrix.suite }} + files: ${{ env.GRCOV_OUTPUT_FILE }} + disable_search: true + flags: ${{ matrix.suite }} + use_oidc: true + + rust-test-sanitizer: + strategy: + fail-fast: false + matrix: + include: + # We don't run memory sanitizer as it provides many false positives + # for std + - sanitizer: asan + sanitizer_flags: "-Zsanitizer=address,leak" + - sanitizer: tsan + sanitizer_flags: "-Zsanitizer=thread" + name: "Rust tests (${{ matrix.sanitizer }})" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/pool=amd64-medium-pre-v2/tag=rust-test-sanitizer', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 30 + env: + ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" + LSAN_OPTIONS: "report_objects=1" + ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + MSAN_OPTIONS: "symbolize=1" + MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt" + TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + VORTEX_SKIP_SLOW_TESTS: "1" + # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins + # unset -Zsanitizer flag and we should allow that. + RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - name: Install Rust nightly toolchain + run: | + rustup toolchain install $NIGHTLY_TOOLCHAIN + rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview + - name: Build tests with sanitizer + run: | + RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ + cargo +$NIGHTLY_TOOLCHAIN build --locked --all-features \ + --target x86_64-unknown-linux-gnu -Zbuild-std \ + -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array + + - name: Run tests with sanitizer + run: | + RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ + cargo +$NIGHTLY_TOOLCHAIN nextest run --locked --all-features \ + --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ + -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array + + # vortex-ffi requires --no-default-features as otherwise we pull in + # Mimalloc which interferes with sanitizers + # cargo nextest reports less sanitizer issues than cargo test + # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in + - name: Run vortex-ffi tests with sanitizer + run: | + RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ + cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \ + --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ + -p vortex-ffi -- --no-capture + + rust-ffi-test-sanitizer: + strategy: + fail-fast: false + matrix: + include: + # We don't run memory sanitizer as it's clang-only and provides many + # false positives for Catch2 + - sanitizer: asan + sanitizer_flags: "-Zsanitizer=address,leak" + - sanitizer: tsan + sanitizer_flags: "-Zsanitizer=thread" + name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})" + timeout-minutes: 30 + env: + ASAN_OPTIONS: "symbolize=1:check_initialization_order=1:detect_leaks=1:leak_check_at_exit=1" + LSAN_OPTIONS: "report_objects=1" + ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + MSAN_OPTIONS: "symbolize=1" + MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + TSAN_OPTIONS: "symbolize=1" + TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" + VORTEX_SKIP_SLOW_TESTS: "1" + # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins + # unset -Zsanitizer flag and we should allow that. + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - name: Install rustfilt + run: | + cargo install rustfilt + - name: Install Rust nightly toolchain + run: | + rustup toolchain install $NIGHTLY_TOOLCHAIN + rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview + - name: Build FFI library + run: | + # TODO(myrrc): remove --no-default-features + RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ + -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ + ${{ matrix.sanitizer_flags }}" \ + cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ + --target x86_64-unknown-linux-gnu -Zbuild-std \ + -p vortex-ffi + - name: Build FFI library tests + run: | + cd vortex-ffi + cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" + cmake --build build -j + - name: Run tests + run: | + set -o pipefail + ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i- + + miri: + name: "Rust tests (miri)" + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-miri', github.run_id) + || 'ubuntu-latest' }} + timeout-minutes: 30 + env: + MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE + RUSTFLAGS: "-A warnings" + RUST_BACKTRACE: full + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-prebuild + - name: Install nightly with miri + run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri + - name: Run Miri + run: cargo +$NIGHTLY_TOOLCHAIN miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi From cd6caff77a60a369a5c78616e26689d0553d118b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:49:32 +0000 Subject: [PATCH 070/250] Update actions/download-artifact action to v8 (#7474) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/download-artifact](https://redirect.github.com/actions/download-artifact) | action | major | `v4` → `v8` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes

actions/download-artifact (actions/download-artifact) ### [`v8.0.1`](https://redirect.github.com/actions/download-artifact/releases/tag/v8.0.1) [Compare Source](https://redirect.github.com/actions/download-artifact/compare/v8.0.0...v8.0.1) ##### What's Changed - Support for CJK characters in the artifact name by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​471](https://redirect.github.com/actions/download-artifact/pull/471) - Add a regression test for artifact name + content-type mismatches by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​472](https://redirect.github.com/actions/download-artifact/pull/472) **Full Changelog**: ### [`v8.0.0`](https://redirect.github.com/actions/download-artifact/releases/tag/v8.0.0) [Compare Source](https://redirect.github.com/actions/download-artifact/compare/v7.0.0...v8.0.0) ##### v8 - What's new ##### Direct downloads To support direct uploads in `actions/upload-artifact`, the action will no longer attempt to unzip all downloaded files. Instead, the action checks the `Content-Type` header ahead of unzipping and skips non-zipped files. Callers wishing to download a zipped file as-is can also set the new `skip-decompress` parameter to `false`. ##### Enforced checks (breaking) A previous release introduced digest checks on the download. If a download hash didn't match the expected hash from the server, the action would log a warning. Callers can now configure the behavior on mismatch with the `digest-mismatch` parameter. To be secure by default, we are now defaulting the behavior to `error` which will fail the workflow run. ##### ESM To support new versions of the @​actions/\* packages, we've upgraded the package to ESM. ##### What's Changed - Don't attempt to un-zip non-zipped downloads by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​460](https://redirect.github.com/actions/download-artifact/pull/460) - Add a setting to specify what to do on hash mismatch and default it to `error` by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​461](https://redirect.github.com/actions/download-artifact/pull/461) **Full Changelog**: ### [`v7.0.0`](https://redirect.github.com/actions/download-artifact/releases/tag/v7.0.0) [Compare Source](https://redirect.github.com/actions/download-artifact/compare/v6.0.0...v7.0.0) ##### v7 - What's new > \[!IMPORTANT] > actions/download-artifact\@​v7 now runs on Node.js 24 (`runs.using: node24`) and requires a minimum Actions Runner version of 2.327.1. If you are using self-hosted runners, ensure they are updated before upgrading. ##### Node.js 24 This release updates the runtime to Node.js 24. v6 had preliminary support for Node 24, however this action was by default still running on Node.js 20. Now this action by default will run on Node.js 24. ##### What's Changed - Update GHES guidance to include reference to Node 20 version by [@​patrikpolyak](https://redirect.github.com/patrikpolyak) in [#​440](https://redirect.github.com/actions/download-artifact/pull/440) - Download Artifact Node24 support by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​415](https://redirect.github.com/actions/download-artifact/pull/415) - fix: update [@​actions/artifact](https://redirect.github.com/actions/artifact) to fix Node.js 24 punycode deprecation by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​451](https://redirect.github.com/actions/download-artifact/pull/451) - prepare release v7.0.0 for Node.js 24 support by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​452](https://redirect.github.com/actions/download-artifact/pull/452) ##### New Contributors - [@​patrikpolyak](https://redirect.github.com/patrikpolyak) made their first contribution in [#​440](https://redirect.github.com/actions/download-artifact/pull/440) - [@​salmanmkc](https://redirect.github.com/salmanmkc) made their first contribution in [#​415](https://redirect.github.com/actions/download-artifact/pull/415) **Full Changelog**: ### [`v6.0.0`](https://redirect.github.com/actions/download-artifact/releases/tag/v6.0.0) [Compare Source](https://redirect.github.com/actions/download-artifact/compare/v5.0.0...v6.0.0) #### What's Changed **BREAKING CHANGE:** this update supports Node `v24.x`. This is not a breaking change per-se but we're treating it as such. - Update README for download-artifact v5 changes by [@​yacaovsnc](https://redirect.github.com/yacaovsnc) in [#​417](https://redirect.github.com/actions/download-artifact/pull/417) - Update README with artifact extraction details by [@​yacaovsnc](https://redirect.github.com/yacaovsnc) in [#​424](https://redirect.github.com/actions/download-artifact/pull/424) - Readme: spell out the first use of GHES by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​431](https://redirect.github.com/actions/download-artifact/pull/431) - Bump `@actions/artifact` to `v4.0.0` - Prepare `v6.0.0` by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​438](https://redirect.github.com/actions/download-artifact/pull/438) #### New Contributors - [@​danwkennedy](https://redirect.github.com/danwkennedy) made their first contribution in [#​431](https://redirect.github.com/actions/download-artifact/pull/431) **Full Changelog**: ### [`v5.0.0`](https://redirect.github.com/actions/download-artifact/releases/tag/v5.0.0) [Compare Source](https://redirect.github.com/actions/download-artifact/compare/v4.3.0...v5.0.0) #### What's Changed - Update README.md by [@​nebuk89](https://redirect.github.com/nebuk89) in [#​407](https://redirect.github.com/actions/download-artifact/pull/407) - BREAKING fix: inconsistent path behavior for single artifact downloads by ID by [@​GrantBirki](https://redirect.github.com/GrantBirki) in [#​416](https://redirect.github.com/actions/download-artifact/pull/416) #### v5.0.0 ##### 🚨 Breaking Change This release fixes an inconsistency in path behavior for single artifact downloads by ID. **If you're downloading single artifacts by ID, the output path may change.** ##### What Changed Previously, **single artifact downloads** behaved differently depending on how you specified the artifact: - **By name**: `name: my-artifact` → extracted to `path/` (direct) - **By ID**: `artifact-ids: 12345` → extracted to `path/my-artifact/` (nested) Now both methods are consistent: - **By name**: `name: my-artifact` → extracted to `path/` (unchanged) - **By ID**: `artifact-ids: 12345` → extracted to `path/` (fixed - now direct) ##### Migration Guide ##### ✅ No Action Needed If: - You download artifacts by **name** - You download **multiple** artifacts by ID - You already use `merge-multiple: true` as a workaround ##### ⚠️ Action Required If: You download **single artifacts by ID** and your workflows expect the nested directory structure. **Before v5 (nested structure):** ```yaml - uses: actions/download-artifact@v4 with: artifact-ids: 12345 path: dist ### Files were in: dist/my-artifact/ ``` > Where `my-artifact` is the name of the artifact you previously uploaded **To maintain old behavior (if needed):** ```yaml - uses: actions/download-artifact@v5 with: artifact-ids: 12345 path: dist/my-artifact # Explicitly specify the nested path ``` #### New Contributors - [@​nebuk89](https://redirect.github.com/nebuk89) made their first contribution in [#​407](https://redirect.github.com/actions/download-artifact/pull/407) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 534e2e3a100..2e2ff49539b 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -107,7 +107,7 @@ jobs: name: github-pages url: ${{ steps.deploy.outputs.deployment-url }} steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: vortex-explorer path: dist From 68e5ce267cc3281231747567bc5a8cefa82f1844 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:58:47 +0000 Subject: [PATCH 071/250] Update gradle/actions action to v6 (#7478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [gradle/actions](https://redirect.github.com/gradle/actions) | action | major | `v5` → `v6` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
gradle/actions (gradle/actions) ### [`v6.1.0`](https://redirect.github.com/gradle/actions/releases/tag/v6.1.0) [Compare Source](https://redirect.github.com/gradle/actions/compare/v6.0.1...v6.1.0) #### New: Basic Cache Provider A new MIT-licensed **Basic Caching** provider is now available as an alternative to the proprietary **Enhanced Caching** provided by `gradle-actions-caching`. Choose Basic Caching by setting `cache-provider: basic` on `setup-gradle` or `dependency-submission` actions. - Built on `@actions/cache` -- fully open source - Caches `~/.gradle/caches` and `~/.gradle/wrapper` directories - Cache key derived from build files (`*.gradle*`, `gradle-wrapper.properties`, etc.) - Clean cache on build file changes (no restore keys, preventing stale entry accumulation) **Limitations vs Enhanced Caching:** No cache cleanup, no deduplication of cached content, cached content is fixed unless build files change. #### Revamped Licensing & Distribution Documentation - New **DISTRIBUTION.md** documents the licensing of each component (particularly Basic Caching vs Enhanced Caching) - Simplified licensing notices in README, docs, and runtime log output - Clear usage tiers: Enhanced Caching is free for public repos and in Free Preview for private repos #### What's Changed - Use a unique cache entry for wrapper-validation test by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​921](https://redirect.github.com/gradle/actions/pull/921) - Update Dependencies by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​922](https://redirect.github.com/gradle/actions/pull/922) - Update dependencies and resolve npm vulnerabilities by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​933](https://redirect.github.com/gradle/actions/pull/933) - Add open-source 'basic' cache provider and revamp licensing documentation by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​930](https://redirect.github.com/gradle/actions/pull/930) - Restructure caching documentation for basic and enhanced providers by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​934](https://redirect.github.com/gradle/actions/pull/934) **Full Changelog**: ### [`v6.0.1`](https://redirect.github.com/gradle/actions/releases/tag/v6.0.1) [Compare Source](https://redirect.github.com/gradle/actions/compare/v6.0.0...v6.0.1) > \[!IMPORTANT] > The release of `gradle/actions@v6` contains important changes to the license terms. More details in [this blog post](https://blog.gradle.org/github-actions-for-gradle-v6). > **TL;DR**: By upgrading to v6, you accept the [Terms of Use](https://gradle.com/legal/terms-of-use/) for the `gradle-actions-caching` component. ##### Summary The [license changes in v6](https://blog.gradle.org/github-actions-for-gradle-v6) introduced a `gradle-actions-caching` license notice that is printed in logs and in each job summary. With this release, the license notice will be muted if build-scan terms have been accepted, or if a Develocity access key is provided. ##### What's Changed - Bump actions used in docs by [@​Goooler](https://redirect.github.com/Goooler) in [#​792](https://redirect.github.com/gradle/actions/pull/792) - Add typing information for use by typesafegithub by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​910](https://redirect.github.com/gradle/actions/pull/910) - Mute license warning when terms are accepted by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​911](https://redirect.github.com/gradle/actions/pull/911) - Mention explicit license acceptance in notice by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​912](https://redirect.github.com/gradle/actions/pull/912) - Bump com.fasterxml.jackson.dataformat:jackson-dataformat-smile from 2.21.1 to 2.21.2 in /sources/test/init-scripts in the gradle group across 1 directory by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​907](https://redirect.github.com/gradle/actions/pull/907) **Full Changelog**: ### [`v6.0.0`](https://redirect.github.com/gradle/actions/releases/tag/v6.0.0) [Compare Source](https://redirect.github.com/gradle/actions/compare/v5.0.2...v6.0.0) > \[!IMPORTANT] > The release of `gradle/actions@v6` contains important changes to the license terms. More details in [this blog post](https://blog.gradle.org/github-actions-for-gradle-v6). > **TL;DR**: By upgrading to v6, you accept the [Terms of Use](https://gradle.com/legal/terms-of-use/) for the `gradle-actions-caching` component. ##### Summary - Caching functionality of 'gradle-actions' has been extracted into a separate `gradle-actions-caching` library, and is no longer open-source. See [this blog post](https://blog.gradle.org/github-actions-for-gradle-v6) for more context. - Existing, rudimentary, configuration-cache support has been removed, pending a fully functional implementation in `gradle-actions-caching`. - Dependencies updated to address security vulnerabilities > \[!IMPORTANT] > > #### Licensing notice > > The caching functionality in \`gradle-actions\` has been extracted into \`gradle-actions-caching\`, a proprietary commercial component that is not covered by the MIT License. > The bundled \`gradle-actions-caching\` component is licensed and governed by a separate license, available at . > > The \`gradle-actions-caching\` component is used only when caching is enabled and is not loaded or used when caching is disabled. > > Use of the \`gradle-actions-caching\` component is subject to a separate license, available at . > If you do not agree to these license terms, do not use the \`gradle-actions-caching\` component. ##### What's Changed - Bump the npm-dependencies group in /sources with 2 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​866](https://redirect.github.com/gradle/actions/pull/866) - Update known wrapper checksums by [@​github-actions](https://redirect.github.com/github-actions)\[bot] in [#​868](https://redirect.github.com/gradle/actions/pull/868) - Dependency updates by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​876](https://redirect.github.com/gradle/actions/pull/876) - Update known wrapper checksums by [@​github-actions](https://redirect.github.com/github-actions)\[bot] in [#​878](https://redirect.github.com/gradle/actions/pull/878) - Bump [@​types/node](https://redirect.github.com/types/node) from 25.3.3 to 25.3.5 in /sources in the npm-dependencies group across 1 directory by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​877](https://redirect.github.com/gradle/actions/pull/877) - Bump the github-actions group across 3 directories with 3 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​867](https://redirect.github.com/gradle/actions/pull/867) - Update known wrapper checksums by [@​github-actions](https://redirect.github.com/github-actions)\[bot] in [#​881](https://redirect.github.com/gradle/actions/pull/881) - Bump the npm-dependencies group in /sources with 6 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​879](https://redirect.github.com/gradle/actions/pull/879) - Bump the github-actions group across 3 directories with 5 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​880](https://redirect.github.com/gradle/actions/pull/880) - Remove configuration-cache support by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​884](https://redirect.github.com/gradle/actions/pull/884) - Extract caching logic into a separate `gradle-actions-caching` component by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​885](https://redirect.github.com/gradle/actions/pull/885) - Update gradle-actions-caching library to v0.3.0 by [@​bot-githubaction](https://redirect.github.com/bot-githubaction) in [#​899](https://redirect.github.com/gradle/actions/pull/899) - Avoid windows shutdown bug by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​900](https://redirect.github.com/gradle/actions/pull/900) - Dependency updates by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​905](https://redirect.github.com/gradle/actions/pull/905) - Fix critical and high npm vulnerabilities by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​904](https://redirect.github.com/gradle/actions/pull/904) - Fix rendering of job-disabled message by [@​bigdaz](https://redirect.github.com/bigdaz) in [#​909](https://redirect.github.com/gradle/actions/pull/909) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2bb9a4135d0..494e3d509e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -89,7 +89,7 @@ jobs: with: distribution: "corretto" java-version: "17" - - uses: gradle/actions/setup-gradle@v5 + - uses: gradle/actions/setup-gradle@v6 - uses: actions/download-artifact@v8 with: pattern: libvortex_jni_*.zip From f9d4febbb4377912fbe6e8e1f8fd7ccd51a3e510 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:05:28 +0000 Subject: [PATCH 072/250] Update actions/setup-node action to v6 (#7475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/setup-node](https://redirect.github.com/actions/setup-node) | action | major | `v4` → `v6` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
actions/setup-node (actions/setup-node) ### [`v6.3.0`](https://redirect.github.com/actions/setup-node/releases/tag/v6.3.0) [Compare Source](https://redirect.github.com/actions/setup-node/compare/v6.2.0...v6.3.0) #### What's Changed ##### Enhancements: - Support parsing `devEngines` field by [@​susnux](https://redirect.github.com/susnux) in [#​1283](https://redirect.github.com/actions/setup-node/pull/1283) > When using node-version-file: package.json, setup-node now prefers devEngines.runtime over engines.node. ##### Dependency updates: - Fix npm audit issues by [@​gowridurgad](https://redirect.github.com/gowridurgad) in [#​1491](https://redirect.github.com/actions/setup-node/pull/1491) - Replace uuid with crypto.randomUUID() by [@​trivikr](https://redirect.github.com/trivikr) in [#​1378](https://redirect.github.com/actions/setup-node/pull/1378) - Upgrade minimatch from 3.1.2 to 3.1.5 by [@​dependabot](https://redirect.github.com/dependabot) in [#​1498](https://redirect.github.com/actions/setup-node/pull/1498) ##### Bug fixes: - Remove hardcoded bearer for mirror-url [@​marco-ippolito](https://redirect.github.com/marco-ippolito) in [#​1467](https://redirect.github.com/actions/setup-node/pull/1467) - Scope test lockfiles by package manager and update cache tests by [@​gowridurgad](https://redirect.github.com/gowridurgad) in [#​1495](https://redirect.github.com/actions/setup-node/pull/1495) #### New Contributors - [@​susnux](https://redirect.github.com/susnux) made their first contribution in [#​1283](https://redirect.github.com/actions/setup-node/pull/1283) **Full Changelog**: ### [`v6.2.0`](https://redirect.github.com/actions/setup-node/compare/v6.1.0...v6.2.0) [Compare Source](https://redirect.github.com/actions/setup-node/compare/v6.1.0...v6.2.0) ### [`v6.1.0`](https://redirect.github.com/actions/setup-node/releases/tag/v6.1.0) [Compare Source](https://redirect.github.com/actions/setup-node/compare/v6.0.0...v6.1.0) #### What's Changed ##### Enhancement: - Remove always-auth configuration handling by [@​priyagupta108](https://redirect.github.com/priyagupta108) in [#​1436](https://redirect.github.com/actions/setup-node/pull/1436) ##### Dependency updates: - Upgrade [@​actions/cache](https://redirect.github.com/actions/cache) from 4.0.3 to 4.1.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1384](https://redirect.github.com/actions/setup-node/pull/1384) - Upgrade actions/checkout from 5 to 6 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1439](https://redirect.github.com/actions/setup-node/pull/1439) - Upgrade js-yaml from 3.14.1 to 3.14.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1435](https://redirect.github.com/actions/setup-node/pull/1435) ##### Documentation update: - Add example for restore-only cache in documentation by [@​aparnajyothi-y](https://redirect.github.com/aparnajyothi-y) in [#​1419](https://redirect.github.com/actions/setup-node/pull/1419) **Full Changelog**: ### [`v6.0.0`](https://redirect.github.com/actions/setup-node/releases/tag/v6.0.0) [Compare Source](https://redirect.github.com/actions/setup-node/compare/v5.0.0...v6.0.0) ##### What's Changed **Breaking Changes** - Limit automatic caching to npm, update workflows and documentation by [@​priyagupta108](https://redirect.github.com/priyagupta108) in [#​1374](https://redirect.github.com/actions/setup-node/pull/1374) **Dependency Upgrades** - Upgrade ts-jest from 29.1.2 to 29.4.1 and document breaking changes in v5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1336](https://redirect.github.com/actions/setup-node/pull/1336) - Upgrade prettier from 2.8.8 to 3.6.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1334](https://redirect.github.com/actions/setup-node/pull/1334) - Upgrade actions/publish-action from 0.3.0 to 0.4.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1362](https://redirect.github.com/actions/setup-node/pull/1362) **Full Changelog**: ### [`v5.0.0`](https://redirect.github.com/actions/setup-node/releases/tag/v5.0.0) [Compare Source](https://redirect.github.com/actions/setup-node/compare/v4.4.0...v5.0.0) ##### What's Changed ##### Breaking Changes - Enhance caching in setup-node with automatic package manager detection by [@​priya-kinthali](https://redirect.github.com/priya-kinthali) in [#​1348](https://redirect.github.com/actions/setup-node/pull/1348) This update, introduces automatic caching when a valid `packageManager` field is present in your `package.json`. This aims to improve workflow performance and make dependency management more seamless. To disable this automatic caching, set `package-manager-cache: false` ```yaml steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: package-manager-cache: false ``` - Upgrade action to use node24 by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​1325](https://redirect.github.com/actions/setup-node/pull/1325) Make sure your runner is on version v2.327.1 or later to ensure compatibility with this release. [See Release Notes](https://redirect.github.com/actions/runner/releases/tag/v2.327.1) ##### Dependency Upgrades - Upgrade [@​octokit/request-error](https://redirect.github.com/octokit/request-error) and [@​actions/github](https://redirect.github.com/actions/github) by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1227](https://redirect.github.com/actions/setup-node/pull/1227) - Upgrade uuid from 9.0.1 to 11.1.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1273](https://redirect.github.com/actions/setup-node/pull/1273) - Upgrade undici from 5.28.5 to 5.29.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1295](https://redirect.github.com/actions/setup-node/pull/1295) - Upgrade form-data to bring in fix for critical vulnerability by [@​gowridurgad](https://redirect.github.com/gowridurgad) in [#​1332](https://redirect.github.com/actions/setup-node/pull/1332) - Upgrade actions/checkout from 4 to 5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1345](https://redirect.github.com/actions/setup-node/pull/1345) ##### New Contributors - [@​priya-kinthali](https://redirect.github.com/priya-kinthali) made their first contribution in [#​1348](https://redirect.github.com/actions/setup-node/pull/1348) - [@​salmanmkc](https://redirect.github.com/salmanmkc) made their first contribution in [#​1325](https://redirect.github.com/actions/setup-node/pull/1325) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 2e2ff49539b..ebe3e4abd8e 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -52,7 +52,7 @@ jobs: - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: . - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: "22" cache: "npm" @@ -81,7 +81,7 @@ jobs: - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: . - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: "22" cache: "npm" From 63e5aee6218a0bbc6ae5718aecfb9186c8c7caa9 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Thu, 16 Apr 2026 11:06:42 -0400 Subject: [PATCH 073/250] java: use triple-quote strings instead of concat (#7481) A little easier to read/edit Signed-off-by: Andrew Duffy --- .../spark/VortexDataSourceWriteTest.java | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java index 80764949bc0..fa2833af0da 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java @@ -332,14 +332,16 @@ public void testWriteAndReadTemporalAndStructColumns() throws IOException { .selectExpr( "cast(id as int) as id", "CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END AS event_date", - "CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) " - + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END AS event_ts", - "named_struct(" - + "'event_date', CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END, " - + "'event_ts', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) " - + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, " - + "'label', CASE WHEN id = 0 THEN 'alpha' ELSE 'beta' END" - + ") AS payload"); + """ + CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END AS event_ts""", + """ + named_struct( + 'event_date', CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END, + 'event_ts', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, + 'label', CASE WHEN id = 0 THEN 'alpha' ELSE 'beta' END + ) AS payload"""); Path outputPath = tempDir.resolve("temporal_struct_output"); originalDf @@ -371,15 +373,13 @@ public void testWriteAndReadTemporalAndStructColumns() throws IOException { @Test @DisplayName("Write TimestampNTZ columns and nested structs") public void testWriteTimestampNtzColumns() throws IOException { - Dataset timestampNtzDf = spark.range(0, 2) - .selectExpr( - "cast(id as int) as id", - "CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) " - + "ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz", - "named_struct(" - + "'event_ntz', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) " - + "ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END" - + ") AS payload"); + Dataset timestampNtzDf = spark.range(0, 2).selectExpr("cast(id as int) as id", """ + CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) + ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz""", """ + named_struct( + 'event_ntz', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END + ) AS payload"""); Path outputPath = tempDir.resolve("timestamp_ntz_output"); assertDoesNotThrow(() -> timestampNtzDf @@ -406,18 +406,15 @@ private Dataset createTestDataFrame(int numRows) { } private List projectTemporalAndStructRows(Dataset df) { - return df - .orderBy("id") - .selectExpr("to_json(named_struct(" - + "'id', id, " - + "'event_date', cast(event_date as string), " - + "'event_ts', date_format(event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), " - + "'payload_event_date', cast(payload.event_date as string), " - + "'payload_event_ts', date_format(payload.event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), " - + "'payload_label', payload.label" - + ")) as json") - .collectAsList() - .stream() + return df.orderBy("id").selectExpr(""" + to_json(named_struct( + 'id', id, + 'event_date', cast(event_date as string), + 'event_ts', date_format(event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), + 'payload_event_date', cast(payload.event_date as string), + 'payload_event_ts', date_format(payload.event_ts, 'yyyy-MM-dd HH:mm:ss.SSSSSS'), + 'payload_label', payload.label + )) as json""").collectAsList().stream() .map(row -> row.getString(0)) .collect(Collectors.toList()); } From 6813e17dc698aeea424f846e9807cefef2e27e0b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:10:01 +0000 Subject: [PATCH 074/250] Update octokit/request-action action to v3 (#7479) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [octokit/request-action](https://redirect.github.com/octokit/request-action) | action | major | `v2.4.0` → `v3.0.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
octokit/request-action (octokit/request-action) ### [`v3.0.0`](https://redirect.github.com/octokit/request-action/releases/tag/v3.0.0) [Compare Source](https://redirect.github.com/octokit/request-action/compare/v2.4.0...v3.0.0) ##### Bug Fixes - **deps:** update dependency [@​octokit/action](https://redirect.github.com/octokit/action) to v8, update runner to v24, switch build from NCC to ESBuild ([#​324](https://redirect.github.com/octokit/request-action/issues/324)) ([b91aaba](https://redirect.github.com/octokit/request-action/commit/b91aabaa861c777dcdb14e2387e30eddf04619ae)) ##### BREAKING CHANGES - **deps:** Update runner to Node 24 Co-authored-by: uzlopak Co-authored-by: wolfy1339 Co-authored-by: wolfy1339 <4595477+wolfy1339@​users.noreply.github.com> Co-authored-by: Audrey Romanet <7204715+aromanet42@​users.noreply.github.com> Co-authored-by: renovate\[bot] <29139614+renovate\[bot][@​users](https://redirect.github.com/users).noreply.github.com>
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index e6eb3f56e40..fb3fe1892bf 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Get PR Labels from API id: get_labels_api - uses: octokit/request-action@v2.4.0 # Use an action to make API requests + uses: octokit/request-action@v3.0.0 # Use an action to make API requests with: route: GET /repos/{owner}/{repo}/issues/{pull_number}/labels owner: ${{ github.repository_owner }} From e34208e68c9bfe2f39530d0568739297a08e6560 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:10:39 +0000 Subject: [PATCH 075/250] Update actions/upload-pages-artifact action to v5 (#7476) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/upload-pages-artifact](https://redirect.github.com/actions/upload-pages-artifact) | action | major | `v4` → `v5` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
actions/upload-pages-artifact (actions/upload-pages-artifact) ### [`v5.0.0`](https://redirect.github.com/actions/upload-pages-artifact/releases/tag/v5.0.0) [Compare Source](https://redirect.github.com/actions/upload-pages-artifact/compare/v4.0.0...v5.0.0) ### Changelog - Update upload-artifact action to version 7 [@​Tom-van-Woudenberg](https://redirect.github.com/Tom-van-Woudenberg) ([#​139](https://redirect.github.com/actions/upload-pages-artifact/issues/139)) - feat: add `include-hidden-files` input [@​jonchurch](https://redirect.github.com/jonchurch) ([#​137](https://redirect.github.com/actions/upload-pages-artifact/issues/137)) See details of [all code changes](https://redirect.github.com/actions/upload-pages-artifact/compare/v4.0.0...v4.0.1) since previous release.
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b1870c2e7f6..b718d731baf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -55,7 +55,7 @@ jobs: uv run --all-packages make -C docs html - name: Upload static files as artifact id: deployment - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v5 with: path: docs/_build/html From 29ad9847b04d8f1caac5294fde9351d49ecd4b74 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 16 Apr 2026 16:24:02 +0100 Subject: [PATCH 076/250] Renovate can create unlimited prs during morning monday hours, limit overall renovate prs to 20 (#7482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Essentially these 3 settings combined mean that we will only ever create 8 renovate prs per week 1. schedule:earlyMondays (line 14) — only runs "before 4am on Monday" (UTC). Renovate has a ~4 hour window, period. Anything not created in that window waits another week. 2. prHourlyLimit: 2 — inherited from config:recommended. Max 2 new PRs per hour, so the early-Monday window caps out at roughly 8 PRs per run. 3. prConcurrentLimit: 10 — also from config:recommended. Once 10 Renovate PRs are open, no new ones open until some are merged/closed. This pr uncappes the concurrent pr opening on Mondays, however, still keeps a limit of 20 open renovate prs Signed-off-by: Robert Kruszewski --- renovate.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/renovate.json b/renovate.json index 9f3ceea4caf..fbc453fae56 100644 --- a/renovate.json +++ b/renovate.json @@ -19,6 +19,8 @@ }, "autoApprove": true, "automergeStrategy": "squash", + "prHourlyLimit": 0, + "prConcurrentLimit": 20, "rebaseWhen": "conflicted", "cloneSubmodules": true, "platformAutomerge": true, From 01679cd7f61287a7616b04b13d4190ade4017a78 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:27:27 +0100 Subject: [PATCH 077/250] Update actions/upload-artifact action to v7 (#7477) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/upload-artifact](https://redirect.github.com/actions/upload-artifact) | action | major | `v4` → `v7` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
actions/upload-artifact (actions/upload-artifact) ### [`v7.0.1`](https://redirect.github.com/actions/upload-artifact/releases/tag/v7.0.1) [Compare Source](https://redirect.github.com/actions/upload-artifact/compare/v7.0.0...v7.0.1) ##### What's Changed - Update the readme with direct upload details by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​795](https://redirect.github.com/actions/upload-artifact/pull/795) - Readme: bump all the example versions to v7 by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​796](https://redirect.github.com/actions/upload-artifact/pull/796) - Include changes in typespec/ts-http-runtime 0.3.5 by [@​yacaovsnc](https://redirect.github.com/yacaovsnc) in [#​797](https://redirect.github.com/actions/upload-artifact/pull/797) **Full Changelog**: ### [`v7.0.0`](https://redirect.github.com/actions/upload-artifact/releases/tag/v7.0.0) [Compare Source](https://redirect.github.com/actions/upload-artifact/compare/v6.0.0...v7.0.0) #### v7 What's new ##### Direct Uploads Adds support for uploading single files directly (unzipped). Callers can set the new `archive` parameter to `false` to skip zipping the file during upload. Right now, we only support single files. The action will fail if the glob passed resolves to multiple files. The `name` parameter is also ignored with this setting. Instead, the name of the artifact will be the name of the uploaded file. ##### ESM To support new versions of the `@actions/*` packages, we've upgraded the package to ESM. #### What's Changed - Add proxy integration test by [@​Link-](https://redirect.github.com/Link-) in [#​754](https://redirect.github.com/actions/upload-artifact/pull/754) - Upgrade the module to ESM and bump dependencies by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​762](https://redirect.github.com/actions/upload-artifact/pull/762) - Support direct file uploads by [@​danwkennedy](https://redirect.github.com/danwkennedy) in [#​764](https://redirect.github.com/actions/upload-artifact/pull/764) #### New Contributors - [@​Link-](https://redirect.github.com/Link-) made their first contribution in [#​754](https://redirect.github.com/actions/upload-artifact/pull/754) **Full Changelog**: ### [`v6.0.0`](https://redirect.github.com/actions/upload-artifact/compare/v5.0.0...v6.0.0) [Compare Source](https://redirect.github.com/actions/upload-artifact/compare/v5.0.0...v6.0.0) ### [`v5.0.0`](https://redirect.github.com/actions/upload-artifact/compare/v4.6.2...v5.0.0) [Compare Source](https://redirect.github.com/actions/upload-artifact/compare/v4.6.2...v5.0.0)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index ebe3e4abd8e..3c0381a30cb 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -91,7 +91,7 @@ jobs: RUSTFLAGS: --cfg getrandom_backend="unsupported" run: npm run build - name: Upload build artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: vortex-explorer path: vortex-web/dist/ From 12f63a4ef17cfe5eb6c5725b16f6e617219348da Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Thu, 16 Apr 2026 11:51:09 -0400 Subject: [PATCH 078/250] Add AGENTS.md and some basic skills (#7484) The skills are cribbed from the Velox repository and updated for Vortex. Also permits renovate bot to self-approve PRs. --------- Signed-off-by: Nicholas Gates --- .agents/skills/ci-failure-analysis/SKILL.md | 100 ++++++++ .agents/skills/pr-review/SKILL.md | 93 ++++++++ .agents/skills/query/SKILL.md | 36 +++ .claude | 1 + .github/workflows/approvals.yml | 4 +- .gitignore | 3 +- AGENTS.md | 240 +++++++++++++------- REUSE.toml | 5 + 8 files changed, 402 insertions(+), 80 deletions(-) create mode 100644 .agents/skills/ci-failure-analysis/SKILL.md create mode 100644 .agents/skills/pr-review/SKILL.md create mode 100644 .agents/skills/query/SKILL.md create mode 120000 .claude diff --git a/.agents/skills/ci-failure-analysis/SKILL.md b/.agents/skills/ci-failure-analysis/SKILL.md new file mode 100644 index 00000000000..7a0bf81b0cc --- /dev/null +++ b/.agents/skills/ci-failure-analysis/SKILL.md @@ -0,0 +1,100 @@ +--- +name: ci-failure-analysis +description: Analyze Vortex GitHub Actions CI failures. Use when asked to investigate failed CI runs, failed jobs, or when the user mentions "/ci-failure-analysis". +--- + +# Vortex CI Failure Analysis Skill + +Analyze failed GitHub Actions runs for the Vortex repository and identify whether the failure is +caused by the PR, pre-existing flakiness, infrastructure, or an unrelated main-branch issue. + +## Inputs + +Use any PR number, repository, run ID, failed job metadata, or log snippets supplied by the user or +automation prompt. If a needed value is missing, discover it with the narrowest `gh` command that +can answer the question. + +## Workflow + +1. List failed jobs for the workflow run: + + ```bash + gh run view --repo --json jobs + ``` + +2. Fetch only failed job logs first: + + ```bash + gh run view --repo --job --log-failed + ``` + + If that fails, use the Actions API: + + ```bash + gh api repos//actions/jobs//logs + ``` + +3. If any `gh` command fails with `error connecting to api.github.com` in a sandbox, rerun it with + escalated network permissions immediately. + +4. Classify each failure: + + - Rust build errors: compiler diagnostics, spans, trait bound failures, feature-gate issues. + - Rust test failures: failing test name, panic/assertion output, expected vs actual values, + source path and line. + - Clippy failures: lint name, file path, line, and suggested fix if shown. + - Formatting or public API failures: changed files and commands needed to regenerate output. + - Python/docs failures: pytest, maturin, Sphinx, doctest, or packaging output. + - Infrastructure failures: toolchain download, cache, runner, network, disk, timeout, or + service issues. + +5. Fetch the PR diff and metadata only after the failing log section is understood: + + ```bash + gh pr view --repo --json title,body,baseRefName,headRefName,files,commits + gh pr diff --repo + ``` + +6. Reproduce narrowly when practical: + + ```bash + cargo test -p + cargo clippy -p --all-targets --all-features + make -C docs doctest + uv run --all-packages pytest + ``` + +7. Check whether the same failure appears on recent main-branch runs or open issues before calling + it PR-caused. + +## Report Format + +Post or return one concise Markdown report: + +````markdown +## CI Failure Analysis + +### Status + + +### Failed Jobs +- ``: + +### Relevant Log Output +```text + +``` + +### Correlation With PR Changes + + +### Recommended Next Step + +```` + +## Rules + +- Show relevant failure excerpts, not full logs. +- If many tests fail, detail the first few distinct failures and summarize the rest. +- Do not guess. If causation is unclear, say what was checked and what would resolve it. +- Prefer one PR comment or one final report over multiple fragmented updates. diff --git a/.agents/skills/pr-review/SKILL.md b/.agents/skills/pr-review/SKILL.md new file mode 100644 index 00000000000..4c073dd85fe --- /dev/null +++ b/.agents/skills/pr-review/SKILL.md @@ -0,0 +1,93 @@ +--- +name: pr-review +description: Review Vortex pull requests for correctness, Rust soundness, performance, API compatibility, and test coverage. Use when reviewing PRs, reviewing code changes, or when the user mentions "/pr-review". +--- + +# Vortex PR Review Skill + +Review Vortex changes for issues that CI may miss: semantic correctness, Rust soundness, +zero-copy and alignment invariants, performance, API compatibility, and missing regression +coverage. + +## Usage Modes + +### GitHub Actions Mode + +When invoked by a PR automation, the prompt may already include PR metadata, issue body, +comments, and changed files. In this mode, use git commands for the diff and history: + +```bash +git diff origin/...HEAD +git diff --stat origin/...HEAD +git log origin/..HEAD --oneline +``` + +If the base ref is missing, fetch only that ref: + +```bash +git fetch origin --depth=1 +``` + +Do not refetch data already provided in the prompt. Use the supplied comments and metadata as +review context. + +### Local CLI Mode + +When the user provides a PR number or URL, use `gh` to fetch the PR metadata, comments, and diff: + +```bash +gh pr view --json title,body,author,baseRefName,headRefName,files,additions,deletions,commits +gh pr view --json comments,reviews +gh pr diff +``` + +If `gh` cannot connect to `api.github.com` because of sandbox networking, rerun with escalated +network permissions. + +## Review Workflow + +1. Read `AGENTS.md` and any nested `AGENTS.md` for project conventions. +2. Identify the change intent from the PR title, body, commits, and tests. +3. Group changed files by area: arrays, encodings, buffers, file/layout, integrations, bindings, + docs, or CI. +4. Trace changed behavior through callers, trait implementations, dtype/nullability handling, + validity masks, and tests. +5. Focus findings on actionable defects. Avoid commenting on formatting or issues already covered + by clippy, rustfmt, or generated API checks. +6. Scope verification to the change. Rust/API changes need Rust checks; docs-only, agent-only, + symlink-only, and metadata-only changes should use targeted validation such as Markdown review, + `ls`, `find`, `git status`, or relevant config linters. + +## Review Areas + +| Area | Focus | +| --- | --- | +| Correctness | Length and dtype invariants, nullability, validity masks, offset math, canonicalization, boundary conditions, empty arrays, scalar vs array behavior | +| Rust soundness | `unsafe` blocks, aliasing, lifetimes, alignment, FFI boundaries, panic safety, ownership of buffers and arrays | +| Compression and IO | Encoding metadata, statistics, layout evolution, file compatibility, scan projection/filter behavior, async IO edge cases | +| Performance | Unnecessary copies, lost zero-copy behavior, avoidable allocations, poor cache locality, quadratic loops, excessive dynamic dispatch in hot paths | +| Error handling | Correct `vortex_err!` and `vortex_bail!` usage, useful messages, no accidental panics on user data | +| API compatibility | Public API docs, public-api lock updates, feature flags, crate boundaries, Python/Java binding impacts | +| Tests | Regression coverage, edge cases, parameterized cases with `rstest`, use of `assert_arrays_eq!`, docs doctests when docs change | +| Verification scope | Avoid requesting or running expensive workspace checks when the PR only changes docs, agent files, symlinks, or metadata | + +## Output + +Lead with findings, ordered by severity. For each finding include: + +- File and line reference. +- Why the issue is a real bug or material risk. +- A concrete fix or verification path when possible. + +Use inline review comments when the environment supports them and a precise changed line is the +best place for the feedback. Keep broad design feedback in the summary. + +If no issues are found, say so explicitly and mention any residual risk or tests not run. + +## Principles + +- Review the code that changed, but inspect enough surrounding code to validate invariants. +- Do not infer causation from commit messages alone. Verify with code, tests, or logs. +- Do not ask for broad rewrites when a narrow fix would address the risk. +- Do not downgrade a correctness or soundness issue to a nit because it is inconvenient. +- Be specific and proportionate. diff --git a/.agents/skills/query/SKILL.md b/.agents/skills/query/SKILL.md new file mode 100644 index 00000000000..2a07f3ef312 --- /dev/null +++ b/.agents/skills/query/SKILL.md @@ -0,0 +1,36 @@ +--- +name: query +description: Answer questions about the Vortex codebase or pull requests. Use when asked a question via "/query" or when the user wants to understand code, architecture, behavior, or implementation details. +--- + +# Vortex Query Skill + +Answer questions about the Vortex project, its pull requests, and its implementation. + +## Key Context + +- Vortex is a Rust workspace for columnar arrays, compression encodings, file IO, and scan + integrations. +- `vortex-array` defines the core array traits, dtype system, canonical arrays, and base + encodings. +- `vortex-buffer` owns aligned zero-copy buffers. +- `vortex-file` and `vortex-layout` implement file and layout reading. +- `encodings/*` contains specialized compressed encodings. +- Python, Java, DuckDB, and DataFusion integrations live in their own workspace areas. + +## Workflow + +1. Read `AGENTS.md` and any closer scoped `AGENTS.md` before relying on conventions. +2. Use `rg` and targeted file reads to identify the relevant crate, module, and tests. +3. If the question is about a PR, inspect the diff and comments before answering. +4. If the question is about behavior, trace the implementation through public entry points, + encoding-specific implementations, and tests. +5. Answer with concrete file paths and line numbers when they help. + +## Answering Guidelines + +- Separate confirmed facts from inference. +- Prefer precise code references over broad descriptions. +- Mention important uncertainty and describe what would verify it. +- Do not invent architecture. If the repository does not answer the question, say what you + checked and what is still missing. diff --git a/.claude b/.claude new file mode 120000 index 00000000000..c0ca4685663 --- /dev/null +++ b/.claude @@ -0,0 +1 @@ +.agents \ No newline at end of file diff --git a/.github/workflows/approvals.yml b/.github/workflows/approvals.yml index 57439b97b32..74ee32d9b42 100644 --- a/.github/workflows/approvals.yml +++ b/.github/workflows/approvals.yml @@ -33,10 +33,12 @@ jobs: const authorType = pr.user.type; // 'Bot' vs 'User' const authorLogin = pr.user.login; // e.g. 'github-actions[bot]' const isBot = authorType === 'Bot' || authorLogin.endsWith('[bot]'); + const oneApprovalBotAuthors = new Set(['renovate[bot]']); - const required = isBot ? 2 : 1; + const required = isBot && !oneApprovalBotAuthors.has(authorLogin) ? 2 : 1; console.log(`PR author: ${authorLogin} (${authorType}), isBot: ${isBot}`); + console.log(`One-approval bot allowlist: ${oneApprovalBotAuthors.has(authorLogin)}`); console.log(`Approvals: ${approvalCount} / ${required} required`); if (approvalCount < required) { diff --git a/.gitignore b/.gitignore index 3eaf3c2579f..7fa79fb2162 100644 --- a/.gitignore +++ b/.gitignore @@ -223,7 +223,8 @@ compile_commands.json /justfile # AI Agents -.claude +.agents/settings.local.json +.claude/settings.local.json .opencode # cargo sweep output diff --git a/AGENTS.md b/AGENTS.md index 3d8fcc84eaa..1317ca34895 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,82 +1,166 @@ -# Vortex - -## Development Guidelines - -- Project is a monorepo Rust workspace, java bindings in `/java`, python bindings in - `/vortex-python` -- Run `cargo build -p` to build a specific crate -- Use `cargo clippy --all-targets --all-features` to make sure a project is free of lint issues. - Please do this every time you reach a stopping point or think you've finished work. -- Run `cargo +nightly fmt --all` to format Rust source files. Please do this every time you reach a - stopping point or think you've finished work. -- Run `./scripts/public-api.sh` to re-generate the public API lock files. Please do this every time - you reach a stopping point or think you've finished work. -- You can try running +# AGENTS.md + +Guidance for Claude, Codex and other coding agents working in the Vortex repository. + +## Task Routing + +- When asked to review a PR, especially via `/pr-review`, use the + `.agents/skills/pr-review` skill. +- When asked a question about the PR or codebase, especially via `/query`, use the + `.agents/skills/query` skill. +- When asked to investigate a CI failure, especially via `/ci-failure-analysis`, use the + `.agents/skills/ci-failure-analysis` skill. + +## Overview + +Vortex is a Rust monorepo for columnar array processing, compression encodings, and file IO. +The workspace also contains Java bindings in `java/`, Python bindings in `vortex-python/`, +documentation in `docs/`, and benchmark tooling in `vortex-bench/` and `benchmarks/`. + +## Repository Layout + +- `vortex-buffer` defines zero-copy aligned `Buffer` and `BufferMut`, guaranteed to + be aligned to `T` or to a requested runtime alignment. +- `vortex-array/src/dtype` contains the `DType` logical type system used throughout Vortex. +- `vortex-array` contains the core `Array` trait and the base encodings, including most + Apache Arrow-style encodings. +- `encodings/*` contains more specialized compressed encodings. +- `vortex-file` implements file IO. It uses `LayoutReader` from `vortex-layout`. +- `vortex-scan`, `vortex-session`, `vortex-datafusion`, and `vortex-duckdb` contain scan + and execution integrations. +- `vortex-python` contains Python bindings. RST-flavored project docs live in `docs/`. + +## Build + +Prefer narrow crate builds while iterating: + +```bash +cargo build -p +``` + +Use workspace-wide builds only when the change spans crate boundaries or before handing off a +broad refactor: + +```bash +cargo build --workspace +``` + +## Testing + +Run tests for the crate or binding you touched before broader checks: + +```bash +cargo test -p +``` + +Examples: + +```bash +cargo test -p vortex-array +make -C docs doctest +uv run --all-packages pytest vortex-python/test +``` + +Run docs doctests from the docs directory with `make -C docs doctest` so the correct Sphinx +Makefile target is used. + +## Linting, Formatting, and Generated Files + +Run verification that matches the files changed. Do not run expensive Rust checks for changes that +only touch Markdown, agent configuration, comments outside Rust code, symlinks, or other metadata +with no Rust/API behavior impact. For docs/config-only changes, validate formatting by inspection +or with a targeted doc/config command, and verify symlink or path changes with `ls`, `find`, and +`git status`. + +For Rust code, public API, feature flag, or generated-file changes, run these before stopping: + +```bash +cargo +nightly fmt --all +./scripts/public-api.sh +cargo clippy --all-targets --all-features +``` + +Notes: + +- `./scripts/public-api.sh` regenerates all `public-api.lock` files through the `xtask` + wrapper. Run it when public Rust APIs may have changed, not for docs-only or agent-only edits. +- For `.github/` changes, follow `.github/AGENTS.md` and run + `yamllint --strict -c .yamllint.yaml` on changed workflow files. +- You can try `cargo fix --lib --allow-dirty --allow-staged && cargo clippy --fix --lib --allow-dirty --allow-staged` - to automatically many fix minor errors. -- When iterating on CI failures, fetch only failed job logs first - (`gh run view --job --log-failed`) and run narrow local repro commands for the - affected crate/tests before running workspace-wide checks. -- If `gh` commands fail with `error connecting to api.github.com` in sandbox, immediately rerun with - escalated network permissions instead of retrying in sandbox. -- If cargo fails with `sccache: error: Operation not permitted`, rerun the command with - `RUSTC_WRAPPER=` so rustc runs directly. You should ONLY do this if you get this exact error, as - this is only a concern when you are running on our CI. -- Run docs doctests from the docs directory (`make -C docs doctest`) so the correct Sphinx Makefile - target is used. - -## Architecture - -- `vortex-buffer` defines zero-copy aligned `Buffer` and `BufferMut` that are guaranteed to be - aligned to `T` (or whatever requested runtime alignment). -- `vortex-array/src/dtype` contains the basic `DType` logical type enum that is the basis of the - Vortex type system -- `vortex-array` contains the basic `Array` trait, as well as several encodings which impl that - trait for each encoding. It includes all of most of the Apache Arrow encodings. -- More exotic compressed encodings live in the crates inside of `/encodings/*` -- File IO is defined in `vortex-file`. It uses the concept of a `LayoutReader` defined in - `vortex-layout` crate. -- `/vortex-python` contains the python bindings. rst flavored docs for the project are in `/docs` - -## Code Style - -- Prefer `impl AsRef` to `&T` for public interfaces where possible, e.g. `impl AsRef` -- Avoid usage of unsafe where not necessary, use zero-cost safe abstractions wherever possible, or - cheap non-zero-cost abstractions. -- Every new public API definition must have a doc comment. Examples are nice to have but not - strictly required. -- Use `vortex_err!` to create a `VortexError` with a format string and `vortex_bail!` to do the same - but immediately return it as a `VortexResult` to the surrounding context. -- When writing tests, strongly consider using `rstest` cases to parameterize repetitive test logic. -- If you want to create a large number of tests to an existing file module called `foo.rs`, and if - you think doing so would be too many to inline in a `tests` submodule within `foo.rs`, then first - promote `foo` to a directory module. You can do this by running - `mkdir foo && mv foo.rs foo/mod.rs`. Then, you can create a test file `foo/tests.rs` that you - include in `foo/mod.rs` with the appropriate test config flag. -- If you encounter clippy errors in tests that should only pertain to production code (e.g., - prohibiting panic/unwrap, possible numerical truncation, etc.), then consider allowing those lints - at the test module level. -- Prefer naming test modules `tests`, not `test`. -- Prefer having test return VortexResult<()> and use ? over unwrap. -- All imports must be at the top of the module, never inside functions. The only exception is - `#[cfg(test)]` blocks, where imports should be at the top of the test module. Function-scoped - imports are only acceptable when (a) required, or (b) it would be exceptionally verbose otherwise, - such as a match statement where left and right sides have similar names. -- Imports should be preferred over qualified identifiers. -- Only write comments that explain non-obvious logic or important context. Avoid commenting simple - or self-explanatory code. -- Use `assert_arrays_eq!` macro for comparing arrays in tests instead of element-by-element - comparison. -- Keep tests concise and to the point - avoid unnecessary setup or verbose assertions. -- Run tests for a specific crate with `cargo test -p ` (e.g., - `cargo test -p vortex-array`). - -## Other - -- When summarizing your work, please produce summaries in valid Markdown that can be easily - copied/pasted to Github. + to fix minor Rust diagnostics automatically when working on Rust code. +- If cargo fails with exactly `sccache: error: Operation not permitted`, rerun that command + with `RUSTC_WRAPPER=` so rustc runs directly. Only do this for that exact error. + +## CI Investigation + +- When iterating on CI failures, fetch only failed job logs first: + `gh run view --job --log-failed`. +- Run narrow local repro commands for the affected crate, test, docs target, or binding before + running workspace-wide checks. +- If a `gh` command fails with `error connecting to api.github.com` in the sandbox, immediately + rerun it with escalated network permissions instead of retrying in the sandbox. +- Verify causation from logs, diffs, and local repros before attributing a failure to a PR. + +## Rust Code Style + +- Prefer `impl AsRef` to `&T` for public interfaces where practical, for example + `impl AsRef`. +- Avoid `unsafe` unless it is necessary. Prefer zero-cost safe abstractions, or cheap + non-zero-cost safe abstractions, over hand-written unsafe code. +- Every new public API definition must have a doc comment. Examples are useful but not required. +- Use `vortex_err!` to create a `VortexError` with a format string. +- Use `vortex_bail!` to create and immediately return a `VortexError` as a `VortexResult`. +- Keep imports at the top of the module. The only exception is a `#[cfg(test)]` test module, + where imports should be at the top of that module. +- Prefer imports over qualified identifiers when the name is used repeatedly. +- Avoid function-scoped imports unless required or unless fully qualifying both sides would be + exceptionally verbose. +- Only write comments that explain non-obvious logic or important context. Do not comment + self-explanatory code. +- Keep public APIs small and consistent with neighboring crates. + +## Tests + +- Strongly consider `rstest` cases when parameterizing repetitive test logic. +- Prefer test functions that return `VortexResult<()>` and use `?` instead of `unwrap`. +- Prefer test module names `tests`, not `test`. +- Use `assert_arrays_eq!` for array comparisons instead of element-by-element assertions. +- Keep tests concise and focused on behavior, edge cases, and regressions. +- If a bug fix is requested, add or identify a failing test first when practical. A test that + passes before and after the fix does not prove the fix. +- If clippy lints in tests prohibit patterns that are acceptable only in test code, consider + allowing the lint at the test module level. +- If an existing `foo.rs` module needs many tests, promote it to a directory module: + `foo/mod.rs` plus `foo/tests.rs`, included from `foo/mod.rs` behind the appropriate test + configuration. + +## Common Mistakes + +Check new and modified lines against this list before finishing: + +- Public API changes without doc comments or refreshed `public-api.lock` files. +- Running `cargo fmt`, `./scripts/public-api.sh`, or workspace clippy for docs-only, agent-only, + symlink-only, or other metadata-only changes. +- Running broad CI-style commands before trying a narrow local repro. +- Using `unwrap`, `expect`, or panic-oriented assertions in tests where `VortexResult<()>` and + `?` would be clearer. +- Comparing arrays element by element instead of using `assert_arrays_eq!`. +- Adding imports inside functions when module-level imports would work. +- Introducing `unsafe` without proving that safe Rust cannot express the same operation. +- Updating expected test output to match buggy behavior without independently verifying the + intended semantics. +- Silently reducing the scope of an approved plan when implementation is harder than expected. + +## Summaries + +When summarizing work, write valid Markdown that can be copied into GitHub. Include the checks +you ran and call out any checks you could not run. ## Commits -- All commits must be signed of by the committers in the form - `Signed-off-by: "COMMITTER" `. +All commits must be signed off by the committers in this form: + +```text +Signed-off-by: "COMMITTER" +``` diff --git a/REUSE.toml b/REUSE.toml index 37cc868b2c2..161f6e3086a 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -30,6 +30,11 @@ path = ["**/README.md", "AGENTS.md", "CLAUDE.md", "CONTRIBUTING.md", "STYLE.md", SPDX-FileCopyrightText = "Copyright the Vortex contributors" SPDX-License-Identifier = "CC-BY-4.0" +[[annotations]] +path = ".agents/skills/**" +SPDX-FileCopyrightText = "Copyright the Vortex contributors" +SPDX-License-Identifier = "CC-BY-4.0" + [[annotations]] path = ["**/.gitignore", ".gitmodules", ".python-version", "**/*.lock", "**/*.lockfile", "**/*.toml", "**/*.json", ".idea/**", ".github/**", "codecov.yml", "java/gradle/wrapper/gradle-wrapper.properties"] precedence = "override" From b4dca29b674964253c6a68bb107f44dadcb3fc0d Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:11:19 -0400 Subject: [PATCH 079/250] Add new `vector-search-bench` benchmarking crate (#7458) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Adds a new `vector-search-bench` crate. Right now this is just utilities is preparation for adding proper vector search benchmarks (that are pulled from disk, not just in-memory). This just includes data downloading and file preparation, conversion (from parquet lists to vortex vector arrays), and some different compression and scan utilities. ## Testing Some basic unit tests but the real stress test will come later when we actually benchmark stuff. Signed-off-by: Connor Tsui --- Cargo.lock | 24 ++ Cargo.toml | 1 + benchmarks/vector-search-bench/Cargo.toml | 40 +++ .../vector-search-bench/src/compression.rs | 103 ++++++++ .../vector-search-bench/src/expression.rs | 97 ++++++++ benchmarks/vector-search-bench/src/ingest.rs | 218 ++++++++++++++++ benchmarks/vector-search-bench/src/lib.rs | 25 ++ benchmarks/vector-search-bench/src/prepare.rs | 233 ++++++++++++++++++ 8 files changed, 741 insertions(+) create mode 100644 benchmarks/vector-search-bench/Cargo.toml create mode 100644 benchmarks/vector-search-bench/src/compression.rs create mode 100644 benchmarks/vector-search-bench/src/expression.rs create mode 100644 benchmarks/vector-search-bench/src/ingest.rs create mode 100644 benchmarks/vector-search-bench/src/lib.rs create mode 100644 benchmarks/vector-search-bench/src/prepare.rs diff --git a/Cargo.lock b/Cargo.lock index b746598d3b1..deacfaf7977 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10042,6 +10042,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vector-search-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "arrow-array 58.0.0", + "arrow-buffer 58.0.0", + "arrow-schema 58.0.0", + "clap", + "futures", + "indicatif", + "parquet 58.0.0", + "rand 0.10.1", + "serde", + "tabled", + "tempfile", + "tokio", + "tracing", + "vortex", + "vortex-bench", + "vortex-btrblocks", + "vortex-tensor", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index a6429bf04e6..1ed524d1c6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ members = [ "benchmarks/datafusion-bench", "benchmarks/duckdb-bench", "benchmarks/random-access-bench", + "benchmarks/vector-search-bench", ] exclude = ["java/testfiles", "wasm-test"] resolver = "2" diff --git a/benchmarks/vector-search-bench/Cargo.toml b/benchmarks/vector-search-bench/Cargo.toml new file mode 100644 index 00000000000..126b62d5e2d --- /dev/null +++ b/benchmarks/vector-search-bench/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "vector-search-bench" +description = "Vector similarity search benchmarks for Vortex on public embedding datasets" +authors.workspace = true +categories.workspace = true +edition.workspace = true +homepage.workspace = true +include.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true +publish = false + +[dependencies] +anyhow = { workspace = true } +arrow-array = { workspace = true } +arrow-buffer = { workspace = true } +arrow-schema = { workspace = true } +clap = { workspace = true, features = ["derive"] } +futures = { workspace = true } +indicatif = { workspace = true } +parquet = { workspace = true, features = ["async"] } +rand = { workspace = true } +serde = { workspace = true, features = ["derive"] } +tabled = { workspace = true, features = ["std"] } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +vortex = { workspace = true, features = ["files", "tokio", "unstable_encodings"] } +vortex-bench = { workspace = true, features = ["unstable_encodings"] } +vortex-btrblocks = { workspace = true, features = ["unstable_encodings"] } +vortex-tensor = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/benchmarks/vector-search-bench/src/compression.rs b/benchmarks/vector-search-bench/src/compression.rs new file mode 100644 index 00000000000..6e2733cc1cd --- /dev/null +++ b/benchmarks/vector-search-bench/src/compression.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Vector compression flavors exercised by the benchmark. +//! +//! Each [`VectorFlavor`] variant maps to a [`vortex::file::WriteStrategyBuilder`] configuration +//! applied to the same input data. +//! +//! The benchmark writes one `.vortex` file per flavor per data file, then scans them all with the +//! same query so the comparison is apples-to-apples with the Parquet files. +//! +//! Note that the handrolled `&[f32]` parquet baseline is **not** a flavor here. + +use clap::ValueEnum; +use vortex::array::ArrayId; +use vortex::array::scalar_fn::ScalarFnVTable; +use vortex::file::ALLOWED_ENCODINGS; +use vortex::file::VortexWriteOptions; +use vortex::file::WriteOptionsSessionExt; +use vortex::file::WriteStrategyBuilder; +use vortex::session::VortexSession; +use vortex::utils::aliases::hash_set::HashSet; +use vortex_bench::Format; +use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_tensor::scalar_fns::l2_denorm::L2Denorm; +use vortex_tensor::scalar_fns::sorf_transform::SorfTransform; + +/// Every [`VectorFlavor`] variant in CLI-help order. +pub const ALL_VECTOR_FLAVORS: &[VectorFlavor] = + &[VectorFlavor::Uncompressed, VectorFlavor::TurboQuant]; + +/// One write-side compression configuration we measure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ValueEnum)] +pub enum VectorFlavor { + /// `BtrBlocksCompressorBuilder::empty()` + #[clap(name = "vortex-uncompressed")] + Uncompressed, + /// `BtrBlocksCompressorBuilder::default().with_turboquant()`. + #[clap(name = "vortex-turboquant")] + TurboQuant, + // TODO(connor): We will want to add `Default` here which is just the default compressor. +} + +impl VectorFlavor { + /// Stable kebab-cased label used in CLI args and metric names. + pub fn label(&self) -> &'static str { + match self { + VectorFlavor::Uncompressed => "vortex-uncompressed", + VectorFlavor::TurboQuant => "vortex-turboquant", + } + } + + /// The `target.format` value emitted on measurements for this flavor. Both flavors produce + /// `.vortex` files, so the compression label carries the flavor split. + pub fn as_format(&self) -> Format { + match self { + VectorFlavor::Uncompressed => Format::OnDiskVortex, + VectorFlavor::TurboQuant => Format::OnDiskVortex, + } + } + + /// Subdirectory name under the per-dataset cache root used to store this flavor's `.vortex` + /// files. + pub fn dir_name(&self) -> &'static str { + match self { + VectorFlavor::Uncompressed => "vortex-uncompressed", + VectorFlavor::TurboQuant => "vortex-turboquant", + } + } + + /// Build the [`vortex::file::WriteStrategyBuilder`]-backed write options for this flavor. + /// + /// TurboQuant produces `L2Denorm(SorfTransform(...))` which the default file + /// `ALLOWED_ENCODINGS` set rejects on normalization — we extend the allow-list with the two + /// scalar-fn array IDs the scheme actually emits. + pub fn create_write_options(&self, session: &VortexSession) -> VortexWriteOptions { + let strategy = match self { + VectorFlavor::Uncompressed => { + let compressor = BtrBlocksCompressorBuilder::empty().build(); + + WriteStrategyBuilder::default() + .with_compressor(compressor) + .build() + } + VectorFlavor::TurboQuant => { + let compressor = BtrBlocksCompressorBuilder::default() + .with_turboquant() + .build(); + + let mut allowed: HashSet = ALLOWED_ENCODINGS.clone(); + allowed.insert(L2Denorm.id()); + allowed.insert(SorfTransform.id()); + + WriteStrategyBuilder::default() + .with_compressor(compressor) + .with_allow_encodings(allowed) + .build() + } + }; + + session.write_options().with_strategy(strategy) + } +} diff --git a/benchmarks/vector-search-bench/src/expression.rs b/benchmarks/vector-search-bench/src/expression.rs new file mode 100644 index 00000000000..b1a5b1ef1c3 --- /dev/null +++ b/benchmarks/vector-search-bench/src/expression.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Cosine-similarity filter [`Expression`]s used by the file-scan path. +//! +//! We can easily build a cosine similarity filter by hand: +//! +//! ```text +//! gt( +//! cosine_similarity(col("emb"), lit(query_scalar)), +//! lit(threshold), +//! ) +//! ``` +//! +//! The query is wrapped as `Scalar::extension::(Scalar::fixed_size_list(F32, ...))` so +//! [`CosineSimilarity`] can treat it as a single-row `Vector` value during evaluation. +//! +//! At scan time the literal expands into a `ConstantArray` whose row count matches the chunk batch +//! size. + +use anyhow::Result; +use vortex::array::expr::Expression; +use vortex::array::expr::col; +use vortex::array::expr::gt; +use vortex::array::expr::lit; +use vortex::array::extension::EmptyMetadata; +use vortex::array::scalar::Scalar; +use vortex::array::scalar_fn::EmptyOptions; +use vortex::array::scalar_fn::ScalarFnVTableExt; +use vortex::dtype::DType; +use vortex::dtype::Nullability; +use vortex::dtype::PType; +use vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity; +use vortex_tensor::vector::Vector; + +/// Build the filter `cosine_similarity(emb, query) > threshold`. +pub fn similarity_filter(query: &[f32], threshold: f32) -> Result { + // Empty queries short-circuit to a literal `false`, so scans return no rows instead of trying + // to evaluate cosine similarity on a zero-dimensional vector. + if query.is_empty() { + return Ok(lit(false)); + } + + let query_lit = lit(query_scalar(query)?); + let cosine = CosineSimilarity.new_expr(EmptyOptions, [col("emb"), query_lit]); + Ok(gt(cosine, lit(threshold))) +} + +/// Wrap a query vector as `Scalar::extension::(Scalar::fixed_size_list(F32, ...))`. +pub fn query_scalar(query: &[f32]) -> Result { + let children: Vec = query + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let fsl = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + + Ok(Scalar::extension::(EmptyMetadata, fsl)) +} + +/// Project just the `emb` column. Used by the throughput-only scan path. +pub fn emb_projection() -> Expression { + col("emb") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn query_scalar_accepts_empty_query() { + let scalar = query_scalar(&[]).unwrap(); + match scalar.dtype() { + DType::Extension(_) => {} + other => panic!("expected Extension, got {other}"), + } + } + + #[test] + fn query_scalar_builds_extension_dtype() { + let scalar = query_scalar(&[1.0, 0.0, 0.0]).unwrap(); + match scalar.dtype() { + DType::Extension(_) => {} + other => panic!("expected Extension, got {other}"), + } + } + + #[test] + fn similarity_filter_uses_gt_operator() { + let expr = similarity_filter(&[1.0, 0.0, 0.0], 0.5).unwrap(); + // Quick sanity check: the printed form contains the operator and the threshold so + // future refactors that change the structure get caught here. + let printed = format!("{expr:?}"); + assert!(printed.contains("Gt") || printed.contains(">"), "{printed}"); + } +} diff --git a/benchmarks/vector-search-bench/src/ingest.rs b/benchmarks/vector-search-bench/src/ingest.rs new file mode 100644 index 00000000000..48904530170 --- /dev/null +++ b/benchmarks/vector-search-bench/src/ingest.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Per-chunk ingest transform. +//! +//! Bridges the parquet record-batch stream and the Vortex file writer: +//! +//! 1. Project the `emb` column out of each struct chunk. +//! 2. Rewrap the `emb` column as `Extension>` via +//! [`vortex_bench::vector_dataset::list_to_vector_ext`]. +//! 3. Cast the FSL element buffer from `f64` -> `f32` if the source is `f64`. After this point all +//! downstream code (compression, scan, recall) is f32-only. +//! 4. Optionally project the `scalar_labels` column through unchanged so future filtered-search +//! benchmarks have it without re-ingest. +//! 5. Repackage as `Struct { id: i64, emb: Vector, scalar_labels: ??? }`. + +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use anyhow::ensure; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; +use vortex::array::IntoArray; +use vortex::array::arrays::ExtensionArray; +use vortex::array::arrays::FixedSizeListArray; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::Struct; +use vortex::array::arrays::StructArray; +use vortex::array::arrays::extension::ExtensionArrayExt; +use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::array::extension::EmptyMetadata; +use vortex::array::validity::Validity; +use vortex::buffer::Buffer; +use vortex::dtype::DType; +use vortex::dtype::PType; +use vortex::dtype::extension::ExtDType; +use vortex_bench::vector_dataset::list_to_vector_ext; +use vortex_tensor::vector::AnyVector; +use vortex_tensor::vector::Vector; + +/// Configuration passed alongside each chunk so the transform can stay stateless. +#[derive(Debug, Clone, Copy)] +pub struct ChunkTransform { + /// Source element ptype as declared by the dataset catalog. Used purely to decide whether the + /// f64 -> f32 cast is needed. + pub src_ptype: PType, + // /// Whether to project the `scalar_labels` column through the output struct. + // pub include_scalar_labels: bool, +} + +impl ChunkTransform { + /// Apply the transform to a single struct chunk and return the rebuilt chunk. + /// + /// `chunk` must be a non-chunked `Struct { id: i64, emb: List }`, where all of the list + /// elements are + /// + /// The returned array is always a `Struct { id: i64, emb: Vector }`. + pub fn apply(&self, chunk: ArrayRef, ctx: &mut ExecutionCtx) -> Result { + let struct_view = chunk.as_opt::().with_context(|| { + format!("ingest: expected struct chunk, got dtype {}", chunk.dtype()) + })?; + + let id = struct_view + .unmasked_field_by_name("id") + .context("ingest: chunk missing `id` column")? + .clone(); + let emb = struct_view + .unmasked_field_by_name("emb") + .context("ingest: chunk missing `emb` column")? + .clone(); + + let emb_ext: ExtensionArray = list_to_vector_ext(emb)?.execute(ctx)?; + + let f32_vector_array = if self.src_ptype == PType::F64 { + convert_f64_to_f32_vectors(&emb_ext, ctx)? + } else { + emb_ext.into_array() + }; + + let fields = [("id", id), ("emb", f32_vector_array)]; + Ok(StructArray::from_fields(&fields)?.into_array()) + } +} + +/// Convert a `Vector` extension array down to `Vector`. +/// +/// This conversion is lossy, but we are generally ok with this because most vector search +/// operations do not demand a high amount of precision. +fn convert_f64_to_f32_vectors(ext: &ExtensionArray, ctx: &mut ExecutionCtx) -> Result { + ensure!(ext.ext_dtype().is::()); + + let fsl: FixedSizeListArray = ext.storage_array().clone().execute(ctx)?; + let validity = fsl.validity()?; + let elements: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + ensure!(elements.ptype() == PType::F64); + + let dim = match fsl.dtype() { + DType::FixedSizeList(_, dim, _) => *dim, + other => bail!("cast_vector_ext_to_f32: expected FSL dtype, got {other}"), + }; + + let f64_slice = elements.as_slice::(); + + #[expect( + clippy::cast_possible_truncation, + reason = "this is intentionally lossy" + )] + let f32_buf: Buffer = f64_slice + .iter() + .copied() + .map(|double| double as f32) + .collect(); + + let f32_elements = PrimitiveArray::new::(f32_buf, Validity::NonNullable).into_array(); + let new_fsl = FixedSizeListArray::try_new(f32_elements, dim, validity, fsl.len())?; + let ext_dtype = ExtDType::::try_new(EmptyMetadata, new_fsl.dtype().clone())?.erased(); + + Ok(ExtensionArray::new(ext_dtype, new_fsl.into_array()).into_array()) +} + +#[cfg(test)] +mod tests { + use vortex::VortexSessionDefault; + use vortex::array::VortexSessionExecute; + use vortex::array::arrays::List; + use vortex::buffer::BufferMut; + use vortex::dtype::Nullability; + use vortex::session::VortexSession; + + use super::*; + + fn list_chunk_f64(rows: &[&[f64]]) -> ArrayRef { + let mut elements = BufferMut::::with_capacity(rows.iter().map(|r| r.len()).sum()); + let mut offsets = BufferMut::::with_capacity(rows.len() + 1); + offsets.push(0); + for row in rows { + for &v in row.iter() { + elements.push(v); + } + offsets.push(i32::try_from(elements.len()).unwrap()); + } + let elements_array = + PrimitiveArray::new::(elements.freeze(), Validity::NonNullable).into_array(); + let offsets_array = + PrimitiveArray::new::(offsets.freeze(), Validity::NonNullable).into_array(); + vortex::array::Array::::new(elements_array, offsets_array, Validity::NonNullable) + .into_array() + } + + fn id_array(ids: &[i64]) -> ArrayRef { + PrimitiveArray::new::( + BufferMut::from_iter(ids.iter().copied()).freeze(), + Validity::NonNullable, + ) + .into_array() + } + + #[test] + fn f64_chunk_is_cast_to_f32() -> Result<()> { + let session = VortexSession::default(); + let mut ctx = session.create_execution_ctx(); + + let emb = list_chunk_f64(&[&[1.0, 2.0, 3.0], &[4.0, 5.0, 6.0]]); + let chunk = + StructArray::from_fields(&[("id", id_array(&[0, 1])), ("emb", emb)])?.into_array(); + let transform = ChunkTransform { + src_ptype: PType::F64, + }; + let out = transform.apply(chunk, &mut ctx)?; + let out_struct = out.as_opt::().expect("returns Struct"); + let out_emb = out_struct.unmasked_field_by_name("emb").unwrap().clone(); + let DType::Extension(ext) = out_emb.dtype() else { + panic!("expected extension dtype, got {}", out_emb.dtype()); + }; + match ext.storage_dtype() { + DType::FixedSizeList(elem, 3, Nullability::NonNullable) => { + assert_eq!( + **elem, + DType::Primitive(PType::F32, Nullability::NonNullable) + ); + } + other => panic!("unexpected storage dtype {other}"), + } + Ok(()) + } + + #[test] + fn f32_chunk_passes_through() -> Result<()> { + let session = VortexSession::default(); + let mut ctx = session.create_execution_ctx(); + + let mut elements = BufferMut::::with_capacity(6); + for v in [1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0] { + elements.push(v); + } + let mut offsets = BufferMut::::with_capacity(3); + offsets.push(0); + offsets.push(3); + offsets.push(6); + let emb = vortex::array::Array::::new( + PrimitiveArray::new::(elements.freeze(), Validity::NonNullable).into_array(), + PrimitiveArray::new::(offsets.freeze(), Validity::NonNullable).into_array(), + Validity::NonNullable, + ) + .into_array(); + let chunk = + StructArray::from_fields(&[("id", id_array(&[0, 1])), ("emb", emb)])?.into_array(); + + let transform = ChunkTransform { + src_ptype: PType::F32, + }; + let out = transform.apply(chunk, &mut ctx)?; + let out_struct = out.as_opt::().expect("returns Struct"); + assert_eq!(out_struct.len(), 2); + Ok(()) + } +} diff --git a/benchmarks/vector-search-bench/src/lib.rs b/benchmarks/vector-search-bench/src/lib.rs new file mode 100644 index 00000000000..ea41e773f47 --- /dev/null +++ b/benchmarks/vector-search-bench/src/lib.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! `vector-search-bench` vector similarity-search benchmark over several datasets. + +pub mod compression; +pub mod expression; +pub mod ingest; +pub mod prepare; + +use std::sync::LazyLock; + +use vortex::VortexSessionDefault; +use vortex::io::session::RuntimeSessionExt; +use vortex::session::VortexSession; + +pub static SESSION: LazyLock = LazyLock::new(|| { + // SAFETY: called from inside the LazyLock initializer, before any other access to + // `SESSION`. The first thread to dereference SESSION runs this once. + unsafe { std::env::set_var(vortex_tensor::SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV, "1") }; + + let session = VortexSession::default().with_tokio(); + vortex_tensor::initialize(&session); + session +}); diff --git a/benchmarks/vector-search-bench/src/prepare.rs b/benchmarks/vector-search-bench/src/prepare.rs new file mode 100644 index 00000000000..3674b353eac --- /dev/null +++ b/benchmarks/vector-search-bench/src/prepare.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Per-flavor on-disk ingest. +//! +//! For each `(dataset, layout, flavor)` triple, [`prepare_flavor`] streams every parquet shard +//! through the [`crate::ingest::ChunkTransform`] and writes one `.vortex` file per shard. The +//! pipeline is idempotent (existing `.vortex` files are skipped) and reports end-to-end wall-clock +//! time, summed input parquet bytes, and total output bytes. + +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::Result; +use futures::StreamExt; +use parquet::arrow::ParquetRecordBatchStreamBuilder; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; +use tracing::info; +use tracing::warn; +use vortex::array::VortexSessionExecute; +use vortex::array::stream::ArrayStreamAdapter; +use vortex::array::stream::ArrayStreamExt; +use vortex::error::VortexResult; +use vortex::error::vortex_err; +use vortex_bench::conversions::parquet_to_vortex_stream; +use vortex_bench::data_dir; +use vortex_bench::utils::file::idempotent_async; +use vortex_bench::vector_dataset::DatasetPaths; +use vortex_bench::vector_dataset::TrainLayout; +use vortex_bench::vector_dataset::VectorDataset; + +use crate::SESSION; +use crate::compression::VectorFlavor; +use crate::ingest::ChunkTransform; + +/// The paths of the vortex files that result from preparing one `(dataset, layout, flavor)` triple. +#[derive(Debug, Clone)] +pub struct CompressedVortexDataSet { + pub dataset: VectorDataset, + pub layout: TrainLayout, + pub flavor: VectorFlavor, + pub vortex_files: Vec, +} + +/// Drive [`prepare_flavor`] across a list of flavors, returning a [`CompressedVortexDataSet`] per flavor +/// in input order. +pub async fn prepare_all( + dataset: VectorDataset, + layout: TrainLayout, + paths_for_dataset: &DatasetPaths, + flavors: &[VectorFlavor], +) -> Result> { + let mut results = Vec::with_capacity(flavors.len()); + + for &flavor in flavors { + let r = prepare_flavor(dataset, layout, paths_for_dataset, flavor).await?; + results.push(r); + } + + Ok(results) +} + +// TODO(connor): This should probably download things in parallel? +/// Prepare one flavor of one dataset by writing one `.vortex` file per train shard. +/// +/// This function is sequential (for now). +pub async fn prepare_flavor( + dataset: VectorDataset, + layout: TrainLayout, + paths_for_dataset: &DatasetPaths, + flavor: VectorFlavor, +) -> Result { + let transform = ChunkTransform { + src_ptype: dataset.element_ptype(), + }; + + let mut vortex_files = Vec::with_capacity(paths_for_dataset.train_files.len()); + + for parquet_path in &paths_for_dataset.train_files { + let parquet_path = parquet_path.clone(); + let vortex_path = parquet_to_vortex_path(&parquet_path, dataset, layout, flavor)?; + + let already_cached = vortex_path.exists(); + if already_cached { + warn!( + "skipping cached vortex shard {} ({} flavor)", + vortex_path.display(), + flavor.label() + ); + } else { + info!( + "ingesting {} -> {} ({} flavor)", + parquet_path.display(), + vortex_path.display(), + flavor.label(), + ); + } + + let written_path = idempotent_async(vortex_path.as_path(), |tmp| async move { + write_shard_streaming(&parquet_path, &tmp, flavor, transform).await + }) + .await?; + + vortex_files.push(written_path); + } + + Ok(CompressedVortexDataSet { + dataset, + layout, + flavor, + vortex_files, + }) +} + +/// Stream one parquet shard through the chunk transform into a Vortex file. +/// +/// The output dtype is derived once from the first transformed chunk so the [`ArrayStreamAdapter`] +/// can declare it ahead of time. +async fn write_shard_streaming( + parquet_path: &Path, + vortex_path: &Path, + flavor: VectorFlavor, + transform: ChunkTransform, +) -> Result<()> { + let file = File::open(parquet_path).await?; + let builder = ParquetRecordBatchStreamBuilder::new(file).await?; + let mut array_stream = parquet_to_vortex_stream(builder.build()?); + + let mut ctx = SESSION.create_execution_ctx(); + + // We need to get the first chunk so that we know what the dtype of the file is. + let first = match array_stream.next().await { + Some(chunk) => transform_chunk(transform, chunk, &mut ctx, parquet_path, 1)?, + None => { + return Err(vortex_err!( + "ingest: parquet shard {} produced no chunks", + parquet_path.display(), + ) + .into()); + } + }; + let dtype = first.dtype().clone(); + let shard_path = parquet_path.to_path_buf(); + + let transformed = + futures::stream::iter(std::iter::once(Ok(first))).chain(array_stream.enumerate().map( + move |(chunk_offset, chunk_or_err)| { + let mut local_ctx = SESSION.create_execution_ctx(); + transform_chunk( + transform, + chunk_or_err, + &mut local_ctx, + &shard_path, + chunk_offset + 2, + ) + }, + )); + + let stream = ArrayStreamExt::boxed(ArrayStreamAdapter::new(dtype, transformed)); + + let mut output = tokio::fs::OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(vortex_path) + .await?; + + flavor + .create_write_options(&SESSION) + .write(&mut output, stream) + .await?; + output.flush().await?; + + Ok(()) +} + +fn transform_chunk( + transform: ChunkTransform, + chunk_or_err: VortexResult, + ctx: &mut vortex::array::ExecutionCtx, + parquet_path: &Path, + chunk_idx: usize, +) -> VortexResult { + let chunk = chunk_or_err.map_err(|err| { + vortex_err!( + "ingest: failed to read chunk {} from {}: {err:#}", + chunk_idx, + parquet_path.display(), + ) + })?; + + transform.apply(chunk, ctx).map_err(|err| { + vortex_err!( + "ingest: failed to transform chunk {} from {}: {err:#}", + chunk_idx, + parquet_path.display(), + ) + }) +} + +/// Translate a parquet shard path to its `.vortex` companion under the flavor directory. +/// +/// Just swaps the file extension and rebases the file name into the per-[`VectorFlavor`] flavor +/// directory. The shard stem is preserved so a directory listing pairs `00-of-10.parquet` with +/// `00-of-10.vortex`. +pub fn parquet_to_vortex_path( + parquet: &Path, + dataset: VectorDataset, + layout: TrainLayout, + flavor: VectorFlavor, +) -> Result { + let stem = parquet + .file_stem() + .with_context(|| format!("parquet path {} has no file stem", parquet.display()))? + .to_owned(); + + // TODO(connor): Is there a better way to do this? + let mut name = stem; + name.push(".vortex"); + + Ok(flavor_dir(dataset, layout, flavor).join(name)) +} + +/// `vortex-bench/data/vector-search////`. +fn flavor_dir(ds: VectorDataset, layout: TrainLayout, flavor: VectorFlavor) -> PathBuf { + data_dir() + .join("vector-search") + .join(ds.name()) + .join(layout.label()) + .join(flavor.dir_name()) +} From 61dffa93dfaa5242e364fd93ab4d1c19a4da3d4e Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Apr 2026 17:12:17 +0100 Subject: [PATCH 080/250] Use an unsafe trait to reinterpret packed struct references as bytes (#7336) ## Summary This is a cosmetic change to make it clearer what is happening when re-interpreting references as raw bytes to serialize plans, making it harder to mis-use by making it a more specific trait. Signed-off-by: Adam Gutglick --- vortex-cuda/src/dynamic_dispatch/mod.rs | 36 ++++++++++++++----------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index fd7190b034f..a0f93feb23f 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -86,22 +86,26 @@ pub fn tag_to_ptype(tag: PTypeTag) -> PType { } } -/// Serialize a `#[repr(C)]` struct to a byte vector for the packed plan. +/// Reinterpret a reference to a packed type as a raw bytes slice. /// -/// Copies field data into a pre-zeroed buffer so padding holes are -/// deterministically zero, avoiding UB from reading uninitialised bytes. -fn as_bytes(val: &T) -> Vec { - let n = size_of::(); - let mut buf = vec![0u8; n]; - // SAFETY: T is a bindgen-generated #[repr(C)] struct with only - // integer/float/enum fields. We overwrite the zeroed buffer with - // the struct's bytes; padding holes keep their zero value. - unsafe { - std::ptr::copy_nonoverlapping(std::ptr::addr_of!(*val).cast::(), buf.as_mut_ptr(), n); - } - buf +/// # Safety +/// +/// The caller must ensure `T` is a `#[repr(C)]` type whose layout is +/// compatible with the C ABI. All the types we serialise (`PlanHeader`, +/// `PackedStage`, `ScalarOp`) are bindgen-generated `#[repr(C)]` structs. +/// Padding bytes may be uninitialised on the Rust side, but the C reader +/// never inspects them, so the values are irrelevant. +unsafe trait AsPackedBytes: Sized { + /// Reinterpret a `&T` as a byte slice for serialization into the packed plan. + fn as_packed_bytes(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(std::ptr::addr_of!(*self).cast(), size_of::()) } + } } +unsafe impl AsPackedBytes for PlanHeader {} +unsafe impl AsPackedBytes for PackedStage {} +unsafe impl AsPackedBytes for ScalarOp {} + /// A stage used to build a [`CudaDispatchPlan`] on the host side. /// /// This is NOT a C ABI struct — it exists purely on the Rust side and is @@ -210,7 +214,7 @@ impl CudaDispatchPlan { output_ptype, plan_size_bytes: total_size as u16, }; - buffer.extend_from_slice(&as_bytes(&header)); + buffer.extend_from_slice(header.as_packed_bytes()); for stage in &stages { let packed_stage = PackedStage { @@ -221,9 +225,9 @@ impl CudaDispatchPlan { num_scalar_ops: stage.scalar_ops.len() as u8, source_ptype: stage.source_ptype, }; - buffer.extend_from_slice(&as_bytes(&packed_stage)); + buffer.extend_from_slice(packed_stage.as_packed_bytes()); for op in &stage.scalar_ops { - buffer.extend_from_slice(&as_bytes(op)); + buffer.extend_from_slice(op.as_packed_bytes()); } } From e03bd1dcc63a34f260a16c05395146e26d1c9df3 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 16 Apr 2026 17:21:59 +0100 Subject: [PATCH 081/250] Use `vx-bench` to run benchmarks in CI (#7466) ## Summary Use the same `vx-bench` tool we want to use locally to run the benchmarks in CI, removing bespoke bash argument parsing logic, generating data with it. --------- Signed-off-by: Adam Gutglick --- .github/scripts/run-sql-bench.sh | 141 --------- .github/workflows/nightly-bench.yml | 31 +- .github/workflows/sql-benchmarks.yml | 258 +++++++++++++---- bench-orchestrator/README.md | 45 ++- bench-orchestrator/bench_orchestrator/cli.py | 272 ++++++++++++++---- .../bench_orchestrator/config.py | 180 +++++++++++- .../bench_orchestrator/runner/builder.py | 21 +- .../bench_orchestrator/runner/executor.py | 138 ++++++--- .../bench_orchestrator/storage/schema.py | 3 + .../bench_orchestrator/storage/store.py | 1 + bench-orchestrator/tests/test_cli.py | 107 +++++++ bench-orchestrator/tests/test_config.py | 65 +++++ bench-orchestrator/tests/test_executor.py | 78 +++++ docs/developer-guide/benchmarking.md | 15 +- 14 files changed, 1026 insertions(+), 329 deletions(-) delete mode 100755 .github/scripts/run-sql-bench.sh create mode 100644 bench-orchestrator/tests/test_cli.py create mode 100644 bench-orchestrator/tests/test_config.py create mode 100644 bench-orchestrator/tests/test_executor.py diff --git a/.github/scripts/run-sql-bench.sh b/.github/scripts/run-sql-bench.sh deleted file mode 100755 index bdbcae3f7a0..00000000000 --- a/.github/scripts/run-sql-bench.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors -# -# Runs SQL benchmarks (datafusion-bench, duckdb-bench, lance-bench) for the given targets. -# This script is used by the sql-benchmarks.yml workflow. -# -# Usage: -# run-sql-bench.sh [options] -# -# Arguments: -# subcommand The benchmark subcommand (e.g., tpch, clickbench, tpcds) -# targets Comma-separated list of engine:format pairs -# (e.g., "datafusion:parquet,datafusion:vortex,duckdb:parquet") -# -# Options: -# --scale-factor Scale factor for the benchmark (e.g., 1.0, 10.0) -# --iterations Number of iterations to pass to each benchmark binary -# --remote-storage Remote storage URL (e.g., s3://bucket/path/) -# If provided, runs in remote mode (no lance support). -# --benchmark-id Benchmark ID for error messages (e.g., tpch-s3) - -set -Eeu -o pipefail - -export VORTEX_EXPERIMENTAL_PATCHED_ARRAY=1 -export FLAT_LAYOUT_INLINE_ARRAY_NODE=1 - -subcommand="$1" -targets="$2" -shift 2 - -scale_factor="" -iterations="" -remote_storage="" -benchmark_id="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --scale-factor) - scale_factor="$2" - shift 2 - ;; - --iterations) - iterations="$2" - shift 2 - ;; - --remote-storage) - remote_storage="$2" - shift 2 - ;; - --benchmark-id) - benchmark_id="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - exit 1 - ;; - esac -done - -is_remote=false -if [[ -n "$remote_storage" ]]; then - is_remote=true -fi - -# Lance on remote storage is not supported. The infrastructure to generate and upload lance files -# to S3 does not exist. If you need lance on S3, you must first implement: -# 1. Lance data generation in data-gen (or a separate step) -# 2. Lance data upload to S3 before this step runs -if $is_remote && echo "$targets" | grep -q 'lance'; then - echo "ERROR: Lance format is not supported for remote storage benchmarks." - echo "Remove 'datafusion:lance' from targets for benchmark '${benchmark_id:-unknown}'." - exit 1 -fi - -# Extract formats for each engine from the targets string. -# Example input: "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet" -# -# Pipeline: split by comma -> filter by engine prefix -> remove prefix -> rejoin with commas -# -# Lance is filtered out of df_formats because it uses a separate binary (lance-bench). -# -# The `|| true` is needed because some benchmarks don't use all engines (e.g., statpopgen only has -# duckdb targets). grep returns exit code 1 when no matches are found. Both greps must be in the -# subshell so that `|| true` covers the case where grep -v receives empty input. -df_formats=$(echo "$targets" | tr ',' '\n' | (grep '^datafusion:' | grep -v ':lance$' || true) | sed 's/datafusion://' | tr '\n' ',' | sed 's/,$//') -ddb_formats=$(echo "$targets" | tr ',' '\n' | (grep '^duckdb:' || true) | sed 's/duckdb://' | tr '\n' ',' | sed 's/,$//') -has_lance=$(echo "$targets" | grep -q 'datafusion:lance' && echo "true" || echo "false") - -# Build options string. -opts="" -if $is_remote; then - opts="--opt remote-data-dir=$remote_storage" -fi -if [[ -n "$scale_factor" ]]; then - if [[ -n "$opts" ]]; then - opts="--opt scale-factor=$scale_factor $opts" - else - opts="--opt scale-factor=$scale_factor" - fi -fi -if [[ -n "$iterations" ]]; then - opts="-i $iterations $opts" -fi - -touch results.json - -if [[ -n "$df_formats" ]]; then - # shellcheck disable=SC2086 - target/release_debug/datafusion-bench "$subcommand" \ - -d gh-json \ - --formats "$df_formats" \ - $opts \ - -o df-results.json - - cat df-results.json >> results.json -fi - -if [[ -n "$ddb_formats" ]]; then - # shellcheck disable=SC2086 - target/release_debug/duckdb-bench "$subcommand" \ - -d gh-json \ - --formats "$ddb_formats" \ - $opts \ - --delete-duckdb-database \ - -o ddb-results.json - - cat ddb-results.json >> results.json -fi - -# Lance-bench only runs for local benchmarks. -if ! $is_remote && [[ "$has_lance" == "true" ]] && [[ -f "target/release_debug/lance-bench" ]]; then - # shellcheck disable=SC2086 - target/release_debug/lance-bench "$subcommand" \ - -d gh-json \ - $opts \ - -o lance-results.json - - cat lance-results.json >> results.json -fi diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index fa8e46cb6b9..9fcb7e45a87 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -29,7 +29,20 @@ jobs: "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H on NVME", - "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], "scale_factor": "100" }, { @@ -38,9 +51,21 @@ jobs: "name": "TPC-H on S3", "local_dir": "vortex-bench/data/tpch/100.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", - "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], "scale_factor": "100.0" - }, + } ] strategy: # A single run not should kill the others diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 59c27a7ca70..5f57862937a 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -20,14 +20,51 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", - "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb", - "extra_data_formats": "vortex-compact" + "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "duckdb"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ] }, { "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", + "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], + "pr_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], "scale_factor": "1.0", "iterations": "10" }, @@ -37,7 +74,23 @@ on: "name": "TPC-H SF=1 on S3", "local_dir": "vortex-bench/data/tpch/1.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", + "data_formats": ["parquet", "vortex", "vortex-compact"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], "scale_factor": "1.0", "iterations": "10" }, @@ -45,7 +98,28 @@ on: "id": "tpch-nvme-10", "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", - "targets": "datafusion:arrow,datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", + "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], + "pr_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], "scale_factor": "10.0", "iterations": "10" }, @@ -55,7 +129,23 @@ on: "name": "TPC-H SF=10 on S3", "local_dir": "vortex-bench/data/tpch/10.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", + "data_formats": ["parquet", "vortex", "vortex-compact"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], "scale_factor": "10.0", "iterations": "10" }, @@ -63,7 +153,25 @@ on: "id": "tpcds-nvme", "subcommand": "tpcds", "name": "TPC-DS SF=1 on NVME", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb", + "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "duckdb"} + ], "scale_factor": "1.0" }, { @@ -71,14 +179,40 @@ on: "subcommand": "statpopgen", "name": "Statistical and Population Genetics", "local_dir": "vortex-bench/data/statpopgen", - "targets": "duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", + "data_formats": ["parquet", "vortex", "vortex-compact"], + "pr_targets": [ + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], + "develop_targets": [ + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], "scale_factor": "100" }, { "id": "fineweb", "subcommand": "fineweb", "name": "FineWeb NVMe", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", + "data_formats": ["parquet", "vortex", "vortex-compact"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], "scale_factor": "100" }, { @@ -87,21 +221,48 @@ on: "name": "FineWeb S3", "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact", + "data_formats": ["parquet", "vortex", "vortex-compact"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "vortex-compact"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"}, + {"engine": "duckdb", "format": "vortex-compact"} + ], "scale_factor": "100" }, { "id": "polarsignals", "subcommand": "polarsignals", "name": "PolarSignals Profiling", - "targets": "datafusion:vortex", + "data_formats": ["vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "vortex"} + ], "scale_factor": "1" - }, + } ] jobs: bench: timeout-minutes: 120 + env: + VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + # Makes python output nicer + COLUMNS: 120 strategy: fail-fast: false matrix: @@ -128,6 +289,10 @@ jobs: - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + with: + sync: false - name: Install DuckDB run: | @@ -137,24 +302,20 @@ jobs: - uses: ./.github/actions/system-info - - name: Resolve targets - id: resolve + - name: Resolve benchmark targets + id: targets shell: bash run: | - targets="${{ matrix.targets }}" - # Non-PR modes include additional targets - if [ "${{ inputs.mode }}" != "pr" ]; then - case "${{ matrix.subcommand }}" in - clickbench) targets="$targets,datafusion:vortex-compact,duckdb:vortex-compact" ;; - esac - # Lance comparisons for local clickbench/tpch (not S3) - if [ -z "${{ matrix.remote_storage }}" ]; then - case "${{ matrix.subcommand }}" in - clickbench|tpch) targets="$targets,datafusion:lance" ;; - esac - fi + if [ "${{ inputs.mode }}" = "pr" ]; then + targets='${{ toJSON(matrix.pr_targets) }}' + else + targets='${{ toJSON(matrix.develop_targets) }}' fi - echo "targets=$targets" >> $GITHUB_OUTPUT + { + echo 'targets_json<<__TARGETS_JSON__' + echo "$targets" + echo '__TARGETS_JSON__' + } >> "$GITHUB_OUTPUT" - name: Build binaries shell: bash @@ -172,24 +333,9 @@ jobs: env: RUST_BACKTRACE: full run: | - # Extract all unique formats from targets (e.g., "datafusion:parquet,duckdb:vortex" -> "parquet,vortex") - all_formats=$(echo "${{ steps.resolve.outputs.targets }}" | tr ',' '\n' | sed 's/^[^:]*://' | sort -u | tr '\n' ',' | sed 's/,$//') - - # Append extra data formats if specified (for file size tracking without benchmarking) - if [ -n "${{ matrix.extra_data_formats }}" ]; then - all_formats="$all_formats,${{ matrix.extra_data_formats }}" - fi - - # Build options string if scale_factor is set - opts="" - if [ -n "${{ matrix.scale_factor }}" ]; then - opts="--opt scale-factor=${{ matrix.scale_factor }}" - fi - - # Generate all data formats with a single command - target/release_debug/data-gen ${{ matrix.subcommand }} \ - --formats "$all_formats" \ - $opts + uv run --project bench-orchestrator vx-bench prepare-data "${{ matrix.subcommand }}" \ + --formats-json '${{ toJSON(matrix.data_formats) }}' \ + ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Setup AWS CLI if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false @@ -227,9 +373,12 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ steps.resolve.outputs.targets }}" \ + bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ + --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --output results.json \ + --no-build \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ - ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }} + ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Run ${{ matrix.name }} benchmark (remote) if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) @@ -242,16 +391,13 @@ jobs: OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | - bash scripts/bench-taskset.sh .github/scripts/run-sql-bench.sh "${{ matrix.subcommand }}" "${{ steps.resolve.outputs.targets }}" \ + bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ + --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --output results.json \ + --no-build \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ - --remote-storage "${{ matrix.remote_storage }}" \ - --benchmark-id "${{ matrix.id }}" \ - ${{ matrix.scale_factor && format('--scale-factor {0}', matrix.scale_factor) || '' }} - - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 - with: - sync: false + --opt remote-data-dir=${{ matrix.remote_storage }} \ + ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Compare results if: inputs.mode == 'pr' shell: bash diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 3d9d7ac99c6..23a927b96d3 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -41,18 +41,34 @@ vx-bench run [options] **Arguments:** -- `benchmark`: Benchmark suite to run (`tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `public-bi`, `statpopgen`) +- `benchmark`: Benchmark suite to run (`tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `polarsignals`, `public-bi`, `statpopgen`) **Options:** - `--engine, -e`: Engines to benchmark, comma-separated (default: `datafusion,duckdb`) - `--format, -f`: Formats to benchmark, comma-separated (default: `parquet,vortex`) +- `--targets-json`: Exact benchmark targets as JSON, e.g. `'[{"engine":"datafusion","format":"parquet"}]'` - `--queries, -q`: Specific queries to run (e.g., `1,2,5`) - `--exclude-queries`: Queries to skip - `--iterations, -i`: Iterations per query (default: 5) - `--label, -l`: Label for this run (useful for later reference) - `--track-memory`: Enable memory usage tracking - `--build/--no-build`: Build binaries before running (default: build) +- `--output`: Optional compatibility output path for raw JSON lines + +### `prepare-data` - Generate Benchmark Data + +Generate only the data formats needed for a benchmark run. + +```bash +vx-bench prepare-data [options] +``` + +**Options:** + +- `--format, -f`: Formats to generate, comma-separated +- `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'` +- `--opt`: Benchmark-specific options such as `scale-factor=10.0` ### `compare` - Compare Results @@ -204,10 +220,10 @@ Test performance at different data scales: ```bash # Run at SF1 -vx-bench run tpch -s 1 -l sf1 +vx-bench run tpch --opt scale-factor=1.0 -l sf1 # Run at SF10 -vx-bench run tpch -s 10 -l sf10 +vx-bench run tpch --opt scale-factor=10.0 -l sf10 # Compare scaling behavior vx-bench compare --runs sf1,sf10 @@ -253,10 +269,30 @@ vx-bench clean --older-than "30 days" --no-keep-labeled | Engine | Supported Formats | |------------|-------------------------------------------| -| datafusion | parquet, vortex, vortex-compact, lance | +| datafusion | arrow, parquet, vortex, vortex-compact, lance | | duckdb | parquet, vortex, vortex-compact, duckdb | | lance | lance | +`datafusion:lance` is executed via the `lance-bench` backend while preserving `datafusion:lance` +result labels. + +## CI Usage + +The SQL benchmark workflow uses explicit JSON target selection instead of parsing `engine:format` +strings in shell: + +```bash +uv run --project bench-orchestrator vx-bench prepare-data tpch \ + --formats-json '["parquet","vortex","vortex-compact"]' \ + --opt scale-factor=1.0 + +uv run --project bench-orchestrator vx-bench run tpch \ + --targets-json '[{"engine":"datafusion","format":"parquet"},{"engine":"duckdb","format":"vortex"}]' \ + --opt scale-factor=1.0 \ + --output results.json \ + --no-build +``` + ## Output Format Comparison results are displayed in a pivot table format: @@ -299,5 +335,6 @@ Benchmarks are built with: - Profile: `release_debug` - RUSTFLAGS: `-C target-cpu=native -C force-frame-pointers=yes` +- Features: `unstable_encodings` This enables native CPU optimizations while preserving debug symbols for profiling. diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index eba7521fe9f..8ee8a7ef055 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -3,7 +3,10 @@ """CLI for benchmark orchestration.""" +import subprocess +from contextlib import contextmanager from datetime import datetime, timedelta +from pathlib import Path from typing import Annotated import pandas as pd @@ -14,12 +17,16 @@ from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( - ENGINE_FORMATS, Benchmark, + BenchmarkTarget, BuildConfig, Engine, Format, RunConfig, + group_targets_by_backend, + parse_formats_json, + parse_targets_json, + resolve_axis_targets, ) from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor @@ -43,6 +50,21 @@ def parse_formats(value: str) -> list[Format]: return [Format(f.strip()) for f in value.split(",")] +def parse_options(value: list[str] | None) -> dict[str, str]: + """Parse repeated --opt key=value options into a dictionary.""" + parsed: dict[str, str] = {} + if not value: + return parsed + + for opt in value: + for segment in opt.split(","): + key, raw_value, *rest = segment.split("=") + if rest: + raise ValueError("Options must be key-value pairs separated by =") + parsed[key] = raw_value + return parsed + + def parse_queries(value: str | None) -> list[int] | None: """Parse comma-separated query numbers.""" if not value: @@ -62,6 +84,101 @@ def run_ref_auto_complete() -> list[str]: return list(map(lambda x: x.run_id, ResultStore().list_runs(limit=None))) +def targets_from_axes(engine: str, format: str) -> tuple[list[BenchmarkTarget], list[str]]: + """Resolve legacy engine/format axes into explicit benchmark targets.""" + return resolve_axis_targets(parse_engines(engine), parse_formats(format)) + + +def backends_for_engines(engines: list[Engine]) -> list[Engine]: + """Resolve legacy engine selections to unique execution engines.""" + seed_formats = { + Engine.DATAFUSION: Format.PARQUET, + Engine.DUCKDB: Format.PARQUET, + Engine.LANCE: Format.LANCE, + } + return list( + group_targets_by_backend(BenchmarkTarget(engine=engine, format=seed_formats[engine]) for engine in engines) + ) + + +@contextmanager +def open_results_output(path: Path | None): + """Open an optional compatibility JSONL output file.""" + if path is None: + yield None + return + + if path.parent != Path(): + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8") as handle: + yield handle + + +def write_result_line(line: str, store_writer, compatibility_file) -> None: + """Write a raw result line to the run store and optional compatibility output.""" + store_writer(line) + if compatibility_file is None: + return + + line = line.strip() + if line.startswith("{"): + compatibility_file.write(line + "\n") + compatibility_file.flush() + + +@app.command("prepare-data") +def prepare_data( + benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to prepare data for")], + format: Annotated[ + str | None, + typer.Option("--format", "-f", help="Formats to generate (comma-separated)"), + ] = None, + formats_json: Annotated[ + str | None, + typer.Option("--formats-json", help="Exact data formats to generate as a JSON array"), + ] = None, + verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Log underlying commands")] = False, + options: Annotated[list[str] | None, typer.Option("--opt", help="Benchmark-specific options")] = None, +) -> None: + """Generate benchmark data for explicitly requested formats.""" + if format and formats_json: + console.print("[red]Specify only one of --format or --formats-json[/red]") + raise typer.Exit(1) + if not format and not formats_json: + console.print("[red]Must specify one of --format or --formats-json[/red]") + raise typer.Exit(1) + + try: + formats = parse_formats_json(formats_json) if formats_json else parse_formats(format or "") + bench_opts = parse_options(options) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + builder = BenchmarkBuilder(verbose=verbose) + binary_path = builder.get_data_generator_path() + if not binary_path.exists(): + console.print(f"[red]Binary not found: {binary_path}[/red]") + console.print("Build benchmark binaries before running prepare-data") + raise typer.Exit(1) + + cmd = [str(binary_path), benchmark.value, "--formats", ",".join(fmt.value for fmt in formats)] + if verbose: + cmd.append("--verbose") + for key, value in bench_opts.items(): + cmd.extend(["--opt", f"{key}={value}"]) + + if verbose: + console.print(f"[dim]$ {' '.join(cmd)}[/dim]") + + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as exc: + console.print(f"[red]Data generation failed: {exc}[/red]") + raise typer.Exit(1) from exc + + @app.command() def run( benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")], @@ -81,27 +198,41 @@ def run( tracing: Annotated[bool, typer.Option("--tracing", help="Record a trace for use with perfetto")] = False, build: Annotated[bool, typer.Option("--build/--no-build", help="Build binaries before running")] = True, verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Log underlying commands")] = False, + targets_json: Annotated[ + str | None, + typer.Option("--targets-json", help="Exact benchmark targets as a JSON array"), + ] = None, + output: Annotated[ + Path | None, + typer.Option("--output", help="Optional path for compatibility JSONL output"), + ] = None, options: Annotated[list[str] | None, typer.Option("--opt", help="Engine or benchmark specific options")] = None, ) -> None: """Run benchmarks with specified configuration.""" - engines = parse_engines(engine) - formats = parse_formats(format) query_list = parse_queries(queries) exclude_list = parse_queries(exclude_queries) - bench_opts = {} - # Build options dict - if options: - for opt in options: - for s in opt.split(","): - k, v, *rest = s.split("=") - if rest: - raise RuntimeError("Options must be key-value pairs separated by =") - bench_opts[k] = v + strict_failures = targets_json is not None + + try: + bench_opts = parse_options(options) + if targets_json: + targets = parse_targets_json(targets_json) + warnings: list[str] = [] + else: + targets, warnings = targets_from_axes(engine, format) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + for warning in warnings: + console.print(f"[yellow]Warning: {warning}[/yellow]") + if not targets: + console.print("[red]No valid benchmark targets selected[/red]") + raise typer.Exit(1) config = RunConfig( benchmark=benchmark, - engines=engines, - formats=formats, + targets=targets, queries=query_list, exclude_queries=exclude_list, iterations=iterations, @@ -110,73 +241,83 @@ def run( track_memory=track_memory, ) - # Validate configuration - warnings = config.validate() - for warning in warnings: - console.print(f"[yellow]Warning: {warning}[/yellow]") + errors = config.validate() + if errors: + for error in errors: + console.print(f"[red]{error}[/red]") + raise typer.Exit(1) build_config = BuildConfig() builder = BenchmarkBuilder(config=build_config, verbose=verbose) store = ResultStore() - # Build binaries if needed if build: - binary_paths = builder.build(engines) + binary_paths = builder.build(config.backends) else: - binary_paths = {e: builder.get_binary_path(e) for e in engines} - # Check binaries exist - for eng, path in binary_paths.items(): + binary_paths = {backend: builder.get_binary_path(backend) for backend in config.backends} + for backend, path in binary_paths.items(): if not path.exists(): console.print(f"[red]Binary not found: {path}[/red]") console.print("Run with --build to build binaries first") raise typer.Exit(1) console.print(f"\n[bold]Running {benchmark.value} benchmark[/bold]") - console.print(f" Engines: {', '.join(e.value for e in engines)}") - console.print(f" Formats: {', '.join(f.value for f in formats)}") + console.print(f" Targets: {', '.join(map(str, config.targets))}") console.print(f" Iterations: {iterations}") if label: console.print(f" Label: {label}") console.print() - # Run benchmarks and store results - with store.create_run(config, build_config) as ctx: - for eng in engines: - # Filter formats to those supported by this engine - supported_formats = ENGINE_FORMATS.get(eng, []) - engine_formats = [f for f in formats if f in supported_formats] - - if not engine_formats: - console.print(f"[yellow]Skipping {eng.value}: no supported formats[/yellow]") - continue + backend_groups = group_targets_by_backend(config.targets) + soft_failures: list[str] = [] - executor = BenchmarkExecutor(binary_paths[eng], eng, verbose=verbose) - - try: - results = executor.run( - benchmark=benchmark, - formats=engine_formats, - queries=query_list, - exclude_queries=exclude_list, - iterations=iterations, - options=bench_opts, - track_memory=track_memory, - samply=samply, - sample_rate=sample_rate, - tracing=tracing, - on_result=ctx.write_raw_json, - ) - console.print(f"[green]{eng.value}: {len(results)} results[/green]") - except RuntimeError as e: - console.print(f"[red]{eng.value} failed: {e}[/red]") - - # Update metadata with binary paths - ctx.metadata.binaries = {e.value: str(p) for e, p in binary_paths.items()} - - console.print(f"\n[green]Results saved to run: {ctx.metadata.run_id}[/green]") + try: + with store.create_run(config, build_config) as ctx, open_results_output(output) as compatibility_file: + for backend, backend_targets in backend_groups.items(): + executor = BenchmarkExecutor(binary_paths[backend], backend, verbose=verbose) + backend_formats = [target.format for target in backend_targets] + + try: + results = executor.run( + benchmark=benchmark, + formats=backend_formats, + queries=query_list, + exclude_queries=exclude_list, + iterations=iterations, + options=bench_opts, + track_memory=track_memory, + samply=samply, + sample_rate=sample_rate, + tracing=tracing, + on_result=lambda line, store_writer=ctx.write_raw_json, compatibility=compatibility_file: ( + write_result_line( + line, + store_writer, + compatibility, + ) + ), + ) + console.print(f"[green]{backend.value}: {len(results)} results[/green]") + except RuntimeError as exc: + ctx.metadata.partial = True + if strict_failures: + raise + console.print(f"[red]{backend.value} failed: {exc}[/red]") + soft_failures.append(str(exc)) + + ctx.metadata.binaries = {backend.value: str(path) for backend, path in binary_paths.items()} + except RuntimeError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc + + if soft_failures: + console.print(f"[yellow]Completed with {len(soft_failures)} backend failure(s)[/yellow]") + + metadata = ctx.metadata + console.print(f"\n[green]Results saved to run: {metadata.run_id}[/green]") # Show comparison table if we have multiple engine:format combinations - df = store.load_results(ctx.metadata.run_id) + df = store.load_results(metadata.run_id) if not df.empty: try: pivot = analyzer.compare_within_run(df) @@ -400,6 +541,10 @@ def show( console.print(f" Benchmark: {metadata.benchmark}") console.print(f" Engines: {', '.join(metadata.engines)}") console.print(f" Formats: {', '.join(metadata.formats)}") + if metadata.targets: + console.print( + " Targets: " + ", ".join(f"{target['engine']}:{target['format']}" for target in metadata.targets) + ) console.print(f" Iterations: {metadata.iterations}") console.print(f" Git commit: {metadata.git_commit[:8]}") console.print(f" Git branch: {metadata.git_branch}") @@ -439,6 +584,7 @@ def build( engines = parse_engines(engine) else: engines = list(Engine) + backends = backends_for_engines(engines) console.print(f"[bold]Building: {', '.join(e.value for e in engines)}[/bold]") console.print(f" Profile: {builder.config.profile}") @@ -446,10 +592,10 @@ def build( console.print() try: - paths = builder.build(engines) + paths = builder.build(backends) console.print("\n[green]Build complete:[/green]") - for eng, path in paths.items(): - console.print(f" {eng.value}: {path}") + for backend, path in paths.items(): + console.print(f" {backend.value}: {path}") except RuntimeError as e: console.print(f"[red]Build failed: {e}[/red]") raise typer.Exit(1) diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 5a99f7262e8..bd81ce64fb2 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -3,13 +3,16 @@ """Configuration types and enums for benchmark orchestration.""" +import json +from collections.abc import Iterable from dataclasses import dataclass, field from enum import Enum from pathlib import Path +from typing import Any, TypeVar class Engine(Enum): - """Execution engines for benchmarks.""" + """Logical execution engines for benchmark results.""" DUCKDB = "duckdb" DATAFUSION = "datafusion" @@ -17,7 +20,7 @@ class Engine(Enum): @property def binary_name(self) -> str: - """Return the cargo binary name for this engine.""" + """Return the cargo binary name for this engine when used to execute benchmarks.""" return { Engine.DUCKDB: "duckdb-bench", Engine.DATAFUSION: "datafusion-bench", @@ -28,6 +31,7 @@ def binary_name(self) -> str: class Format(Enum): """Data formats for benchmarks.""" + ARROW = "arrow" PARQUET = "parquet" VORTEX = "vortex" VORTEX_COMPACT = "vortex-compact" @@ -43,13 +47,15 @@ class Benchmark(Enum): CLICKBENCH = "clickbench" FINEWEB = "fineweb" GHARCHIVE = "gh-archive" + POLARSIGNALS = "polarsignals" PUBLIC_BI = "public-bi" STATPOPGEN = "statpopgen" -# Engine to supported formats mapping +# Engine to supported formats mapping. ENGINE_FORMATS: dict[Engine, list[Format]] = { Engine.DATAFUSION: [ + Format.ARROW, Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, @@ -64,14 +70,155 @@ class Benchmark(Enum): Engine.LANCE: [Format.LANCE], } +T = TypeVar("T") + + +def _unique_preserve_order(values: Iterable[T]) -> list[T]: + seen: set[T] = set() + unique: list[T] = [] + for value in values: + if value in seen: + continue + seen.add(value) + unique.append(value) + return unique + + +@dataclass(frozen=True) +class BenchmarkTarget: + """An explicit engine/format benchmark target.""" + + engine: Engine + format: Format + + def normalized(self) -> "BenchmarkTarget": + """Normalize legacy lance targets onto the logical CI identity.""" + if self.engine == Engine.LANCE and self.format == Format.LANCE: + return BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE) + return self + + @property + def backend(self) -> Engine: + """Return the engine whose benchmark binary should execute this target.""" + target = self.normalized() + if target.format == Format.LANCE: + return Engine.LANCE + if target.engine == Engine.DATAFUSION: + return Engine.DATAFUSION + if target.engine == Engine.DUCKDB: + return Engine.DUCKDB + raise ValueError(f"Unsupported benchmark target: {target}") + + def is_supported(self) -> bool: + """Return whether the logical engine supports this format.""" + target = self.normalized() + return target.format in ENGINE_FORMATS.get(target.engine, []) + + def to_dict(self) -> dict[str, str]: + target = self.normalized() + return {"engine": target.engine.value, "format": target.format.value} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "BenchmarkTarget": + if not isinstance(data, dict): + raise ValueError("Targets must be JSON objects with engine and format fields") + try: + target = cls( + engine=Engine(data["engine"]), + format=Format(data["format"]), + ) + except KeyError as exc: + raise ValueError("Targets must include engine and format fields") from exc + except ValueError as exc: + raise ValueError(f"Invalid benchmark target: {data}") from exc + return target.normalized() + + def __str__(self) -> str: + target = self.normalized() + return f"{target.engine.value}:{target.format.value}" + + +def parse_targets_json(value: str) -> list[BenchmarkTarget]: + """Parse a JSON array of explicit benchmark targets.""" + try: + raw_targets = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("Targets JSON must be a valid JSON array") from exc + + if not isinstance(raw_targets, list): + raise ValueError("Targets JSON must be an array") + + return _unique_preserve_order(BenchmarkTarget.from_dict(item) for item in raw_targets) + + +def parse_formats_json(value: str) -> list[Format]: + """Parse a JSON array of format names.""" + try: + raw_formats = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError("Formats JSON must be a valid JSON array") from exc + + if not isinstance(raw_formats, list): + raise ValueError("Formats JSON must be an array") + + formats: list[Format] = [] + for item in raw_formats: + if not isinstance(item, str): + raise ValueError("Formats JSON entries must be strings") + try: + formats.append(Format(item)) + except ValueError as exc: + raise ValueError(f"Invalid format: {item}") from exc + return _unique_preserve_order(formats) + + +def resolve_axis_targets( + engines: Iterable[Engine], formats: Iterable[Format] +) -> tuple[list[BenchmarkTarget], list[str]]: + """Expand engine/format axes into supported explicit targets.""" + warnings: list[str] = [] + targets: list[BenchmarkTarget] = [] + + for engine in engines: + for fmt in formats: + target = BenchmarkTarget(engine=engine, format=fmt).normalized() + if not target.is_supported(): + warnings.append(f"Format {fmt.value} is not supported by engine {engine.value}") + continue + targets.append(target) + + return _unique_preserve_order(targets), warnings + + +def group_targets_by_backend(targets: Iterable[BenchmarkTarget]) -> dict[Engine, list[BenchmarkTarget]]: + """Group logical benchmark targets by the backend binary required to run them.""" + groups: dict[Engine, list[BenchmarkTarget]] = {} + for target in _unique_preserve_order(target.normalized() for target in targets): + groups.setdefault(target.backend, []).append(target) + return groups + + +def validate_targets(targets: Iterable[BenchmarkTarget], options: dict[str, str]) -> list[str]: + """Validate explicit targets against benchmark runner constraints.""" + errors: list[str] = [] + + normalized_targets = [target.normalized() for target in targets] + for target in normalized_targets: + if not target.is_supported(): + errors.append(f"Format {target.format.value} is not supported by engine {target.engine.value}") + + if options.get("remote-data-dir") and any(target.format == Format.LANCE for target in normalized_targets): + errors.append("Lance format is not supported for remote storage benchmarks.") + + return _unique_preserve_order(errors) + @dataclass class RunConfig: """Configuration for a benchmark run.""" benchmark: Benchmark - engines: list[Engine] - formats: list[Format] + targets: list[BenchmarkTarget] queries: list[int] | None = None exclude_queries: list[int] | None = None iterations: int = 5 @@ -79,15 +226,21 @@ class RunConfig: options: dict[str, str] = field(default_factory=dict) track_memory: bool = False + @property + def engines(self) -> list[Engine]: + return _unique_preserve_order(target.engine for target in self.targets) + + @property + def formats(self) -> list[Format]: + return _unique_preserve_order(target.format for target in self.targets) + + @property + def backends(self) -> list[Engine]: + return _unique_preserve_order(target.backend for target in self.targets) + def validate(self) -> list[str]: - """Validate the configuration and return any warnings.""" - warnings = [] - for engine in self.engines: - supported = ENGINE_FORMATS.get(engine, []) - for fmt in self.formats: - if fmt not in supported: - warnings.append(f"Format {fmt.value} is not supported by engine {engine.value}") - return warnings + """Validate the configuration and return any errors.""" + return validate_targets(self.targets, self.options) @dataclass @@ -96,6 +249,7 @@ class BuildConfig: profile: str = "release_debug" rustflags: str = "-C target-cpu=native -C force-frame-pointers=yes" + features: tuple[str, ...] = ("unstable_encodings",) def get_workspace_root() -> Path: diff --git a/bench-orchestrator/bench_orchestrator/runner/builder.py b/bench-orchestrator/bench_orchestrator/runner/builder.py index 55e817c9eb2..070100f1660 100644 --- a/bench-orchestrator/bench_orchestrator/runner/builder.py +++ b/bench-orchestrator/bench_orchestrator/runner/builder.py @@ -29,21 +29,24 @@ def __init__( self.config = config or BuildConfig() self.verbose = verbose - def get_binary_path(self, engine: Engine) -> Path: + def get_binary_path(self, backend: Engine) -> Path: """Get path to built binary.""" - binary_name = engine.binary_name + binary_name = backend.binary_name return self.workspace_root / "target" / self.config.profile / binary_name - def build(self, engines: list[Engine]) -> dict[Engine, Path]: + def get_data_generator_path(self) -> Path: + """Get path to the built benchmark data generator binary.""" + return self.workspace_root / "target" / self.config.profile / "data-gen" + + def build(self, backends: list[Engine]) -> dict[Engine, Path]: """Build binaries for specified engines, return paths.""" - results = {} + results: dict[Engine, Path] = {} - # Build each binary env = os.environ.copy() env["RUSTFLAGS"] = self.config.rustflags - for engine in engines: - binary_name = engine.binary_name + for backend in backends: + binary_name = backend.binary_name console.print(f"[blue]Building {binary_name}...[/blue]") cmd = [ @@ -54,6 +57,8 @@ def build(self, engines: list[Engine]) -> dict[Engine, Path]: "--profile", self.config.profile, ] + if self.config.features: + cmd.extend(["--features", ",".join(self.config.features)]) if self.verbose: console.print(f"[dim]$ RUSTFLAGS='{self.config.rustflags}' {' '.join(cmd)}[/dim]") @@ -65,7 +70,7 @@ def build(self, engines: list[Engine]) -> dict[Engine, Path]: env=env, check=True, ) - results[engine] = self.get_binary_path(engine) + results[backend] = self.get_binary_path(backend) console.print(f"[green]Built {binary_name}[/green]") except subprocess.CalledProcessError as e: console.print(f"[red]Failed to build {binary_name}: {e}[/red]") diff --git a/bench-orchestrator/bench_orchestrator/runner/executor.py b/bench-orchestrator/bench_orchestrator/runner/executor.py index a80080d8d5a..4fa87076d64 100644 --- a/bench-orchestrator/bench_orchestrator/runner/executor.py +++ b/bench-orchestrator/bench_orchestrator/runner/executor.py @@ -3,7 +3,9 @@ """Benchmark binary execution.""" +import selectors import subprocess +from collections import deque from collections.abc import Callable from pathlib import Path from typing import final @@ -20,12 +22,12 @@ class BenchmarkExecutor: """Executes benchmark binaries and captures output.""" - def __init__(self, binary_path: Path, engine: Engine, verbose: bool = False): + def __init__(self, binary_path: Path, backend: Engine, verbose: bool = False): self.binary_path = binary_path - self.engine = engine + self.backend = backend self.verbose = verbose - def run( + def build_command( self, benchmark: Benchmark, formats: list[Format], @@ -37,24 +39,8 @@ def run( samply: bool = False, sample_rate: int | None = None, tracing: bool = False, - on_result: Callable[[str], None] | None = None, ) -> list[str]: - """ - Run benchmark and return results as JSON lines. - - Args: - benchmark: The benchmark suite to run - formats: Data formats to benchmark - queries: Specific queries to run (None for all) - exclude_queries: Queries to skip - iterations: Number of runs per query - options: Additional options (e.g., scale_factor) - track_memory: Enable memory tracking - on_result: Callback for each result line (for streaming) - - Returns: - List of JSON lines from the benchmark output - """ + """Build the command used to execute a benchmark binary.""" cmd = [ str(self.binary_path), benchmark.value, @@ -62,11 +48,14 @@ def run( "gh-json", "--iterations", str(iterations), - "--formats", - ",".join(f.value for f in formats), "--hide-progress-bar", ] + if self.backend in {Engine.DATAFUSION, Engine.DUCKDB}: + cmd.extend(["--formats", ",".join(fmt.value for fmt in formats)]) + if self.backend == Engine.DUCKDB: + cmd.append("--delete-duckdb-database") + if queries: cmd.extend(["--queries", ",".join(map(str, queries))]) if exclude_queries: @@ -76,25 +65,71 @@ def run( if tracing: cmd.append("--tracing") if options: - for k, v in options.items(): - cmd.extend(["--opt", f"{k}={v}"]) + for key, value in options.items(): + cmd.extend(["--opt", f"{key}={value}"]) if samply: cmd = ["--"] + cmd cmd_prefix = ["samply", "record"] if sample_rate: - cmd = cmd_prefix + ["--rate", sample_rate] + cmd + cmd = cmd_prefix + ["--rate", str(sample_rate)] + cmd else: cmd = cmd_prefix + cmd - if samply and self.engine == Engine.DUCKDB: - # Re-use the same DuckDB instance across runs make samply output readable - cmd += ["--reuse"] + if samply and self.backend == Engine.DUCKDB: + # Re-use the same DuckDB instance across runs to keep samply output readable. + cmd.append("--reuse") + + return cmd + + def run( + self, + benchmark: Benchmark, + formats: list[Format], + queries: list[int] | None = None, + exclude_queries: list[int] | None = None, + iterations: int = 5, + options: dict[str, str] | None = None, + track_memory: bool = False, + samply: bool = False, + sample_rate: int | None = None, + tracing: bool = False, + on_result: Callable[[str], None] | None = None, + ) -> list[str]: + """ + Run benchmark and return results as JSON lines. + + Args: + benchmark: The benchmark suite to run + formats: Data formats to benchmark + queries: Specific queries to run (None for all) + exclude_queries: Queries to skip + iterations: Number of runs per query + options: Additional options (e.g., scale_factor) + track_memory: Enable memory tracking + on_result: Callback for each result line (for streaming) + + Returns: + List of JSON lines from the benchmark output + """ + cmd = self.build_command( + benchmark=benchmark, + formats=formats, + queries=queries, + exclude_queries=exclude_queries, + iterations=iterations, + options=options, + track_memory=track_memory, + samply=samply, + sample_rate=sample_rate, + tracing=tracing, + ) if self.verbose: console.print(f"[dim]$ {' '.join(cmd)}[/dim]") - results = [] + results: list[str] = [] + diagnostic_lines: deque[str] = deque(maxlen=200) with Progress( SpinnerColumn(), @@ -102,28 +137,51 @@ def run( console=console, transient=True, ) as progress: - _task = progress.add_task(f"Running {self.engine.value} {benchmark.value}...", total=None) + _task = progress.add_task(f"Running {self.backend.value} {benchmark.value}...", total=None) + # Merge stderr into stdout so verbose benchmark logs cannot fill a separate pipe and + # block the child process before it emits JSON results. process = subprocess.Popen( cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, + bufsize=1, ) - for line in iter(process.stdout.readline, ""): - line = line.strip() - if line: - results.append(line) - if on_result: - on_result(line) + assert process.stdout is not None + selector = selectors.DefaultSelector() + selector.register(process.stdout, selectors.EVENT_READ) + + try: + while selector.get_map(): + for key, _mask in selector.select(timeout=0.1): + line = key.fileobj.readline() + if line == "": + selector.unregister(key.fileobj) + continue + + line = line.rstrip() + if not line: + continue + + if line.startswith("{"): + results.append(line) + if on_result: + on_result(line) + else: + diagnostic_lines.append(line) + console.print(line, markup=False) + finally: + selector.close() ret_code = process.wait() if ret_code != 0: - stderr = process.stderr.read() if process.stderr else "" console.print(f"[red]Benchmark failed with code {process.returncode}[/red]") - if stderr: - console.print(f"[red]{stderr}[/red]") - raise RuntimeError(f"Benchmark {self.engine.value} {benchmark.value} failed: {stderr}") + diagnostics = "\n".join(diagnostic_lines) + if diagnostics: + console.print(f"[red]{diagnostics}[/red]") + raise RuntimeError(f"Benchmark {self.backend.value} {benchmark.value} failed: {diagnostics}") return results diff --git a/bench-orchestrator/bench_orchestrator/storage/schema.py b/bench-orchestrator/bench_orchestrator/storage/schema.py index f9bd9d19cff..a43c16382f4 100644 --- a/bench-orchestrator/bench_orchestrator/storage/schema.py +++ b/bench-orchestrator/bench_orchestrator/storage/schema.py @@ -93,6 +93,7 @@ class RunMetadata: benchmark: str engines: list[str] formats: list[str] + targets: list[dict[str, str]] iterations: int git_commit: str git_branch: str @@ -116,6 +117,7 @@ def to_dict(self) -> dict[str, Any]: "dataset_config": self.dataset_config, "engines": self.engines, "formats": self.formats, + "targets": self.targets, "queries": self.queries, "iterations": self.iterations, "git_commit": self.git_commit, @@ -139,6 +141,7 @@ def from_dict(cls, data: dict[str, Any]) -> "RunMetadata": dataset_config=data.get("dataset_config", {}), engines=data["engines"], formats=data["formats"], + targets=data.get("targets", []), queries=data.get("queries", []), iterations=data["iterations"], git_commit=data["git_commit"], diff --git a/bench-orchestrator/bench_orchestrator/storage/store.py b/bench-orchestrator/bench_orchestrator/storage/store.py index 1703af1ad6f..54c15593a41 100644 --- a/bench-orchestrator/bench_orchestrator/storage/store.py +++ b/bench-orchestrator/bench_orchestrator/storage/store.py @@ -142,6 +142,7 @@ def create_run(self, config: RunConfig, build_config: BuildConfig) -> Iterator[R dataset_config=config.options, engines=[e.value for e in config.engines], formats=[f.value for f in config.formats], + targets=[target.to_dict() for target in config.targets], queries=config.queries or [], iterations=config.iterations, git_commit=git_commit, diff --git a/bench-orchestrator/tests/test_cli.py b/bench-orchestrator/tests/test_cli.py new file mode 100644 index 00000000000..ffb2e2b3ad9 --- /dev/null +++ b/bench-orchestrator/tests/test_cli.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +import json + +from bench_orchestrator import cli as cli_module +from bench_orchestrator.runner.executor import BenchmarkExecutor +from bench_orchestrator.storage.store import ResultStore +from typer.testing import CliRunner + +runner = CliRunner() + + +def test_prepare_data_uses_formats_json(tmp_path, monkeypatch) -> None: + data_gen = tmp_path / "data-gen" + data_gen.write_text("", encoding="utf-8") + + captured: dict[str, list[str]] = {} + + def fake_run(cmd: list[str], check: bool) -> None: + assert check is True + captured["cmd"] = cmd + + monkeypatch.setattr(cli_module.BenchmarkBuilder, "get_data_generator_path", lambda self: data_gen) + monkeypatch.setattr(cli_module.subprocess, "run", fake_run) + + result = runner.invoke( + cli_module.app, + [ + "prepare-data", + "tpch", + "--formats-json", + '["parquet","vortex"]', + "--opt", + "scale-factor=1.0", + ], + ) + + assert result.exit_code == 0 + assert captured["cmd"] == [ + str(data_gen), + "tpch", + "--formats", + "parquet,vortex", + "--opt", + "scale-factor=1.0", + ] + + +def test_run_writes_compatibility_results_output(tmp_path, monkeypatch) -> None: + run_store = ResultStore(base_dir=tmp_path / "runs") + output_path = tmp_path / "artifacts" / "results.json" + binary_path = tmp_path / "datafusion-bench" + binary_path.write_text("", encoding="utf-8") + + sample_line = json.dumps( + { + "name": "tpch_q1/datafusion:parquet", + "storage": "nvme", + "dataset": {"scale_factor": "1.0"}, + "unit": "ns", + "value": 100, + "all_runtimes": [95, 100, 105], + "target": {"engine": "datafusion", "format": "parquet"}, + "commit_id": "deadbeef", + "env_triple": { + "architecture": "x86_64", + "operating_system": "linux", + "environment": "gnu", + }, + } + ) + + monkeypatch.setattr(cli_module, "ResultStore", lambda: run_store) + monkeypatch.setattr(cli_module.BenchmarkBuilder, "get_binary_path", lambda self, backend: binary_path) + + def fake_run(self, **kwargs): + kwargs["on_result"](sample_line) + return [sample_line] + + monkeypatch.setattr(BenchmarkExecutor, "run", fake_run) + + result = runner.invoke( + cli_module.app, + [ + "run", + "tpch", + "--targets-json", + '[{"engine":"datafusion","format":"parquet"}]', + "--no-build", + "--output", + str(output_path), + ], + ) + + assert result.exit_code == 0 + assert output_path.read_text(encoding="utf-8") == sample_line + "\n" + + run_dirs = [path for path in (tmp_path / "runs").iterdir() if path.is_dir() and path.name != "latest"] + assert len(run_dirs) == 1 + + results_path = run_dirs[0] / "results.jsonl" + assert results_path.read_text(encoding="utf-8") == sample_line + "\n" + + metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8")) + assert metadata["targets"] == [{"engine": "datafusion", "format": "parquet"}] + assert metadata["binaries"] == {"datafusion": str(binary_path)} diff --git a/bench-orchestrator/tests/test_config.py b/bench-orchestrator/tests/test_config.py new file mode 100644 index 00000000000..c7e2d6bb291 --- /dev/null +++ b/bench-orchestrator/tests/test_config.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +from bench_orchestrator.config import ( + BenchmarkTarget, + Engine, + Format, + group_targets_by_backend, + parse_formats_json, + parse_targets_json, + resolve_axis_targets, + validate_targets, +) + + +def test_parse_targets_json_normalizes_and_dedupes_lance_targets() -> None: + targets = parse_targets_json('[{"engine":"lance","format":"lance"},{"engine":"datafusion","format":"lance"}]') + + assert targets == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)] + + +def test_parse_formats_json_accepts_ci_format_arrays() -> None: + formats = parse_formats_json('["parquet","vortex","duckdb"]') + + assert formats == [Format.PARQUET, Format.VORTEX, Format.DUCKDB] + + +def test_resolve_axis_targets_filters_unsupported_combinations() -> None: + targets, warnings = resolve_axis_targets( + [Engine.DATAFUSION, Engine.DUCKDB], + [Format.ARROW, Format.PARQUET], + ) + + assert targets == [ + BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.ARROW), + BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.PARQUET), + BenchmarkTarget(engine=Engine.DUCKDB, format=Format.PARQUET), + ] + assert warnings == ["Format arrow is not supported by engine duckdb"] + + +def test_validate_targets_rejects_remote_lance() -> None: + errors = validate_targets( + [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)], + {"remote-data-dir": "s3://benchmarks/tpch/"}, + ) + + assert errors == ["Lance format is not supported for remote storage benchmarks."] + + +def test_group_targets_by_backend_routes_lance_to_lance_binary() -> None: + groups = group_targets_by_backend( + [ + BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.PARQUET), + BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE), + BenchmarkTarget(engine=Engine.DUCKDB, format=Format.VORTEX), + ] + ) + + assert list(groups) == [ + Engine.DATAFUSION, + Engine.LANCE, + Engine.DUCKDB, + ] + assert groups[Engine.LANCE] == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)] diff --git a/bench-orchestrator/tests/test_executor.py b/bench-orchestrator/tests/test_executor.py new file mode 100644 index 00000000000..ade3dde1a67 --- /dev/null +++ b/bench-orchestrator/tests/test_executor.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +import sys +import textwrap +from pathlib import Path + +from bench_orchestrator.config import Benchmark, Engine, Format +from bench_orchestrator.runner.executor import BenchmarkExecutor + + +def test_build_command_adds_duckdb_cleanup_flag() -> None: + executor = BenchmarkExecutor(Path("/tmp/duckdb-bench"), Engine.DUCKDB) + + cmd = executor.build_command( + benchmark=Benchmark.TPCH, + formats=[Format.PARQUET, Format.VORTEX], + iterations=7, + options={"scale-factor": "1.0"}, + ) + + assert cmd[:5] == [ + "/tmp/duckdb-bench", + "tpch", + "--display-format", + "gh-json", + "--iterations", + ] + assert "--formats" in cmd + assert "parquet,vortex" in cmd + assert "--delete-duckdb-database" in cmd + assert "--opt" in cmd + assert "scale-factor=1.0" in cmd + + +def test_build_command_omits_formats_for_lance_backend() -> None: + executor = BenchmarkExecutor(Path("/tmp/lance-bench"), Engine.LANCE) + + cmd = executor.build_command( + benchmark=Benchmark.TPCH, + formats=[Format.LANCE], + queries=[1, 3], + ) + + assert cmd[0] == "/tmp/lance-bench" + assert "--formats" not in cmd + assert "--queries" in cmd + assert "1,3" in cmd + + +def test_run_streams_logs_without_counting_them(tmp_path: Path) -> None: + script = tmp_path / "fake-bench.py" + script.write_text( + textwrap.dedent( + f"""\ + #!{sys.executable} + import sys + + print("preparing duckdb tables", file=sys.stderr, flush=True) + print('{{"engine":"duckdb","format":"parquet","query":0}}', flush=True) + print("finished query 0", file=sys.stderr, flush=True) + """ + ) + ) + script.chmod(0o755) + + executor = BenchmarkExecutor(script, Engine.DUCKDB) + streamed: list[str] = [] + + results = executor.run( + benchmark=Benchmark.CLICKBENCH, + formats=[Format.PARQUET], + iterations=1, + on_result=streamed.append, + ) + + assert results == ['{"engine":"duckdb","format":"parquet","query":0}'] + assert streamed == results diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index 2e09c727dd7..0034965a056 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -168,7 +168,7 @@ Lance, DuckDB native). Before running SQL benchmarks, test data must be generated: ```bash -cargo run --release --bin data-gen -- --formats parquet,vortex +uv run --project bench-orchestrator vx-bench prepare-data --format parquet,vortex ``` The data generator creates base Parquet data and converts it to each requested format. Scale @@ -191,6 +191,19 @@ benchmarks across multiple engines, stores results, and provides comparison tool See [`bench-orchestrator/README.md`](https://github.com/vortex-data/vortex/blob/develop/bench-orchestrator/README.md) for installation, commands, and example workflows. +For CI, the reusable SQL workflow now drives `vx-bench` directly: + +```bash +uv run --project bench-orchestrator vx-bench prepare-data tpch \ + --formats-json '["parquet","vortex","vortex-compact"]' \ + --opt scale-factor=1.0 + +uv run --project bench-orchestrator vx-bench run tpch \ + --targets-json '[{"engine":"datafusion","format":"parquet"},{"engine":"duckdb","format":"vortex"}]' \ + --output results.json \ + --no-build +``` + ## CI Benchmarks Benchmarks run automatically on all commits to `develop` and can be run on-demand for PRs: From 1169d84e010f74fb2c14880da7b4dfb730337e7b Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 16 Apr 2026 17:23:19 +0100 Subject: [PATCH 082/250] Use cargo-zigbuild to link against old libc (#7480) We can use zig and link directly without having to spin up old oses --------- Signed-off-by: Robert Kruszewski --- .github/actions/prepare-java-linux/action.yml | 18 ----------- .github/workflows/package.yml | 32 +++++++++++-------- .github/workflows/publish-dry-runs.yml | 28 +++++++++------- 3 files changed, 34 insertions(+), 44 deletions(-) delete mode 100644 .github/actions/prepare-java-linux/action.yml diff --git a/.github/actions/prepare-java-linux/action.yml b/.github/actions/prepare-java-linux/action.yml deleted file mode 100644 index 06c51d5245a..00000000000 --- a/.github/actions/prepare-java-linux/action.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: "Prepare vortex dependencies for java" -description: "Install required packages for vortex java build" -runs: - using: "composite" - steps: - - shell: bash - run: | - apt update - apt install -y wget curl build-essential unzip clang - - name: Print GLIBC version - shell: bash - run: ldd --version - - name: Verify GLIBC version - shell: bash - run: | - set -eux - - ldd --version | grep 'GLIBC 2.31' diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index e137726b26c..36e251d1fb6 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -106,35 +106,39 @@ jobs: if-no-files-found: error prepare-java-linux: - runs-on: ${{ matrix.target.runs-on }} - container: - image: "ubuntu:20.04" timeout-minutes: 30 strategy: fail-fast: false matrix: - target: - - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } - - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } + include: + - target: x86_64-unknown-linux-gnu + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=prepare-java-linux-amd64 + - target: aarch64-unknown-linux-gnu + runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=prepare-java-linux-arm64 + runs-on: ${{ matrix.runner }} steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: ./.github/actions/prepare-java-linux - uses: actions/setup-java@v5 with: distribution: "corretto" java-version: "17" - - uses: ./.github/actions/setup-rust + - uses: ./.github/actions/setup-prebuild + - uses: mlugg/setup-zig@v2.2.1 + - name: Install cargo-zigbuild + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: - targets: ${{ matrix.target.target }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - run: cargo build --release --package vortex-jni + tool: cargo-zigbuild + - run: cargo zigbuild --release --target ${{ matrix.target }}.2.31 --package vortex-jni - uses: actions/upload-artifact@v7 with: - name: "libvortex_jni_${{ matrix.target.target }}.zip" - path: "target/release/libvortex_jni.so" + name: "libvortex_jni_${{ matrix.target }}.zip" + path: "target/${{ matrix.target }}/release/libvortex_jni.so" retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index bea9f4e9b02..3e5b6d32868 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -64,31 +64,35 @@ jobs: working-directory: vortex-python/ check-java-publish-build: - runs-on: ${{ matrix.target.runs-on }} - container: - image: "ubuntu:20.04" timeout-minutes: 30 strategy: fail-fast: false matrix: - target: - - { os: ubuntu, runs-on: "ubuntu-24.04-arm", target: aarch64-unknown-linux-gnu } - - { os: ubuntu, runs-on: "ubuntu-24.04", target: x86_64-unknown-linux-gnu } + include: + - target: x86_64-unknown-linux-gnu + runner: runs-on=${{ github.run_id }}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=check-java-publish-build-amd64 + - target: aarch64-unknown-linux-gnu + runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/tag=check-java-publish-build-arm64 + runs-on: ${{ matrix.runner }} steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: ./.github/actions/prepare-java-linux - uses: actions/setup-java@v5 with: distribution: "corretto" java-version: "17" - - uses: ./.github/actions/setup-rust + - uses: ./.github/actions/setup-prebuild + - uses: mlugg/setup-zig@v2.2.1 + - name: Install cargo-zigbuild + uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: - targets: ${{ matrix.target.target }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: "false" - - run: cargo build --profile ci --package vortex-jni + tool: cargo-zigbuild + - run: cargo zigbuild --profile ci --target ${{ matrix.target }}.2.31 --package vortex-jni compat-check: name: "Compat check" From 91b4c75d942d39fa8263fbccef914d05b66cb7fa Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 16 Apr 2026 18:18:46 +0100 Subject: [PATCH 083/250] perf[gpu]: reduce register pressure in dyn dispatch (#7489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We decrease the number of values per tile in the output stage each GPU thread uses, as well as limit the register count to 32 in the launch bounds. This brings the dynamic dispatch kernel into a reasonably close range compared to the standalone kernel for now. Type | Dynamic dispatch | Standalone | Ratio | |---|---|---|---| | u8 bw6 | 172 µs | 79 µs | 2.17× | | u16 bw6 | 140 µs | 88 µs | 1.59× | | u32 bw6 | 184 µs | 148 µs | 1.24× | | u64 bw8 | 303 µs | 276 µs | 1.10×| Signed-off-by: Alexander Droste --- vortex-cuda/kernels/src/dynamic_dispatch.cu | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.cu b/vortex-cuda/kernels/src/dynamic_dispatch.cu index 29bf26efefb..d3950485435 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.cu +++ b/vortex-cuda/kernels/src/dynamic_dispatch.cu @@ -279,7 +279,8 @@ __device__ void execute_output_stage(T *__restrict output, char *__restrict smem, uint64_t block_start, uint32_t block_len) { - constexpr uint32_t VALUES_PER_TILE = 32 / sizeof(T); + // Cap at 4 values per thread per tile to minimise register pressure. + constexpr uint32_t VALUES_PER_TILE = (32 / sizeof(T)) < 4 ? (32 / sizeof(T)) : 4; const uint32_t tile_size = blockDim.x * VALUES_PER_TILE; const auto &src = stage.source; const void *raw_input = reinterpret_cast(stage.input_ptr); @@ -472,9 +473,10 @@ dynamic_dispatch(T *__restrict output, uint64_t array_len, const uint8_t *__rest // matters is load_element(), which dispatches on the per-op PTypeTag to // sign-extend or zero-extend when widening a narrow source to T. #define GENERATE_KERNEL(suffix, Type) \ - extern "C" __global__ void dynamic_dispatch_##suffix(Type *__restrict output, \ - uint64_t array_len, \ - const uint8_t *__restrict packed_plan) { \ + extern "C" __global__ void __launch_bounds__(BLOCK_SIZE, 32) \ + dynamic_dispatch_##suffix(Type *__restrict output, \ + uint64_t array_len, \ + const uint8_t *__restrict packed_plan) { \ dynamic_dispatch(output, array_len, packed_plan); \ } From 67c165595ec142ca8927ff0da7b72b66100f8d88 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Apr 2026 18:37:44 +0100 Subject: [PATCH 084/250] fix: add validity no_nulls and fix usage (#7487) Signed-off-by: Joe Isaacs --- vortex-array/public-api.lock | 2 ++ vortex-array/src/arrow/record_batch.rs | 3 +-- vortex-array/src/validity.rs | 7 +++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index f5b0adb4f2c..dced3a9f4a7 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -19038,6 +19038,8 @@ pub fn vortex_array::validity::Validity::mask_eq(&self, other: &vortex_array::va pub fn vortex_array::validity::Validity::maybe_len(&self) -> core::option::Option +pub fn vortex_array::validity::Validity::no_nulls(&self) -> bool + pub fn vortex_array::validity::Validity::not(&self) -> vortex_error::VortexResult pub fn vortex_array::validity::Validity::nullability(&self) -> vortex_array::dtype::Nullability diff --git a/vortex-array/src/arrow/record_batch.rs b/vortex-array/src/arrow/record_batch.rs index 02ec62f626a..3163e967ca3 100644 --- a/vortex-array/src/arrow/record_batch.rs +++ b/vortex-array/src/arrow/record_batch.rs @@ -17,7 +17,6 @@ use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::StructArray; use crate::arrow::ArrowArrayExecutor; -use crate::validity::Validity; // deprecated(note = "Use ArrowArrayExecutor::execute_record_batch instead") impl TryFrom<&ArrayRef> for RecordBatch { @@ -29,7 +28,7 @@ impl TryFrom<&ArrayRef> for RecordBatch { }; vortex_ensure!( - matches!(struct_array.validity()?, Validity::AllValid), + struct_array.validity()?.no_nulls(), "RecordBatch can only be constructed from StructArray with no nulls" ); diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index cb261a9b0b2..84586754dcf 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -112,6 +112,13 @@ impl Validity { } } + /// Returns `true` if this validity guarantees no null values, i.e. it is either + /// [`Validity::NonNullable`] or [`Validity::AllValid`]. + #[inline] + pub fn no_nulls(&self) -> bool { + matches!(self, Self::NonNullable | Self::AllValid) + } + /// The union nullability and validity. #[inline] pub fn union_nullability(self, nullability: Nullability) -> Self { From 1b2f6ee352859640eea2908a418f21410615e204 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Apr 2026 18:47:35 +0100 Subject: [PATCH 085/250] perf: correctly try execute parent in the iterative child execute loop (#7386) We run iterative execution for executing arrays (decompressing). This PR add a execute_parent call when executing a child in a iterative fashion --------- Signed-off-by: Alfonso Subiotto Marques Signed-off-by: Joe Isaacs Co-authored-by: Alfonso Subiotto Marques --- encodings/runend/src/compute/filter.rs | 41 +++++++++++++++++++ vortex-array/src/executor.rs | 55 ++++++++++++-------------- 2 files changed, 66 insertions(+), 30 deletions(-) diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index 11093d482ac..23c72e4b6d2 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -115,6 +115,8 @@ fn filter_run_end_primitive + AsPrimitiv #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_error::VortexResult; @@ -142,4 +144,43 @@ mod tests { ); Ok(()) } + + /// Regression: Filter(Slice(RunEnd)) must preserve RunEnd after execution. + /// Previously Filter.execute() forced its child to canonical, decoding + /// Slice(RunEnd) → Primitive and destroying run structure. The fix lets + /// Filter unwrap one layer at a time so RunEnd's FilterKernel can fire. + #[test] + fn filter_sliced_run_end_preserves_encoding() -> VortexResult<()> { + // 4 runs of 32 each = 128 rows. Large enough that FilterKernel takes + // the run-preserving path (true_count >= 25). + let values: Vec = [10, 20, 30, 40] + .iter() + .flat_map(|&v| std::iter::repeat_n(v, 32)) + .collect(); + let arr = RunEnd::encode(PrimitiveArray::from_iter(values).into_array())?; + + // Slice off the first 16 rows. Slice(RunEnd), 112 rows, 4 runs. + let sliced = arr.into_array().slice(16..128)?; + + // Keep every other row = 112/2 = 56 rows. + let mask = Mask::from_iter((0..sliced.len()).map(|i| i % 2 == 0)); + let filtered = sliced.filter(mask)?; + + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let executed = filtered.execute_until::(&mut ctx)?; + assert_eq!( + executed.encoding_id().as_ref(), + "vortex.runend", + "Filter(Slice(RunEnd)) should preserve RunEnd encoding" + ); + + let expected: Vec = std::iter::repeat_n(10, 8) + .chain(std::iter::repeat_n(20, 16)) + .chain(std::iter::repeat_n(30, 16)) + .chain(std::iter::repeat_n(40, 16)) + .collect(); + assert_arrays_eq!(executed, PrimitiveArray::from_iter(expected)); + + Ok(()) + } } diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 74a013bf5bb..8e0c86ff44f 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -86,10 +86,18 @@ impl ArrayRef { /// Iteratively execute this array until the [`Matcher`] matches, using an explicit work /// stack. /// - /// The scheduler repeatedly: - /// 1. Checks if the current array matches `M` — if so, pops the stack or returns. - /// 2. Runs `execute_parent` on each child for child-driven optimizations. - /// 3. Calls `execute` which returns an [`ExecutionStep`]. + /// Each iteration proceeds through three steps in order: + /// + /// 1. **Done / canonical check** — if `current` satisfies the active done predicate or is + /// canonical, splice it back into the stacked parent (if any) and continue, or return. + /// 2. **`execute_parent` on children** — try each child's `execute_parent` against `current` + /// as the parent (e.g. `Filter(RunEnd)` → `FilterExecuteAdaptor` fires from RunEnd). + /// If there is a stacked parent frame, the rewritten child is spliced back into it so + /// that optimize and further `execute_parent` can fire on the reconstructed parent + /// (e.g. `Slice(RunEnd)` → `RunEnd` spliced into stacked `Filter` → `Filter(RunEnd)` + /// whose `FilterExecuteAdaptor` fires on the next iteration). + /// 3. **`execute`** — call the encoding's own execute step, which either returns `Done` or + /// `ExecuteSlot(i)` to push a child onto the stack for focused execution. /// /// Note: the returned array may not match `M`. If execution converges to a canonical form /// that does not match `M`, the canonical array is returned since no further execution @@ -103,51 +111,41 @@ impl ArrayRef { let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); for _ in 0..max_iterations() { - // Check for termination: use the stack frame's done predicate, or the root matcher. + // Step 1: done / canonical — splice back into stacked parent or return. let is_done = stack .last() .map_or(M::matches as DonePredicate, |frame| frame.2); - if is_done(¤t) { + if is_done(¤t) || AnyCanonical::matches(¤t) { match stack.pop() { None => { ctx.log(format_args!("-> {}", current)); return Ok(current); } Some((parent, slot_idx, _)) => { - current = parent.with_slot(slot_idx, current)?; - current = current.optimize()?; + current = parent.with_slot(slot_idx, current)?.optimize()?; continue; } } } - // If we've reached canonical form, we can't execute any further regardless - // of whether the matcher matched. - if AnyCanonical::matches(¤t) { - match stack.pop() { - None => { - ctx.log(format_args!("-> canonical (unmatched) {}", current)); - return Ok(current); - } - Some((parent, slot_idx, _)) => { - current = parent.with_slot(slot_idx, current)?; - current = current.optimize()?; - continue; - } - } - } - - // Try execute_parent (child-driven optimized execution) + // Step 2: execute_parent on children (current is the parent). + // If there is a stacked parent frame, splice the rewritten child back into it + // so that optimize and execute_parent can fire naturally on the reconstructed parent + // (e.g. Slice(RunEnd) -RunEndSliceKernel-> RunEnd, spliced back into Filter gives + // Filter(RunEnd), whose FilterExecuteAdaptor fires on the next iteration). if let Some(rewritten) = try_execute_parent(¤t, ctx)? { ctx.log(format_args!( "execute_parent rewrote {} -> {}", current, rewritten )); current = rewritten.optimize()?; + if let Some((parent, slot_idx, _)) = stack.pop() { + current = parent.with_slot(slot_idx, current)?.optimize()?; + } continue; } - // Execute the array itself. + // Step 4: execute the encoding's own step. let result = execute_step(current, ctx)?; let (array, step) = result.into_parts(); match step { @@ -177,9 +175,6 @@ impl ArrayRef { } /// Execution context for batch CPU compute. -/// -/// Accumulates a trace of execution steps. Individual steps are logged at TRACE level for -/// real-time following, and the full trace is dumped at DEBUG level when the context is dropped. #[derive(Debug, Clone)] pub struct ExecutionCtx { id: usize, @@ -193,8 +188,8 @@ impl ExecutionCtx { static EXEC_CTX_ID: AtomicUsize = AtomicUsize::new(0); let id = EXEC_CTX_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); Self { - id, session, + id, ops: Vec::new(), } } From ba08515d93ff70f5746c02fcc84b0099ea6663b4 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Apr 2026 19:20:19 +0100 Subject: [PATCH 086/250] skip: add a changelog deprecation label (#7492) Adds a changelog/deprecation Signed-off-by: Joe Isaacs --- .github/release-drafter.yml | 3 +++ .github/workflows/labels.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index c28b97aba4c..1608fb15570 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -4,6 +4,9 @@ categories: - title: "⚠️ Breaks" labels: - "changelog/break" + - title: "🚧 Deprecation" + labels: + - "changelog/deprecation" - title: "✨ Features" labels: - "changelog/feature" diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index fb3fe1892bf..5c1489addb2 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -34,6 +34,7 @@ jobs: run: | REQUIRED_LABELS=( "changelog/break" + "changelog/deprecation" "changelog/feature" "changelog/performance" "changelog/fix" From aca6251afb7e5d0f71fa4292598fc3a2eaaa4110 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Apr 2026 19:23:08 +0100 Subject: [PATCH 087/250] break: remove `try_from` struct array to record batch (#7488) We cannot deprecate this TryFrom so it has to be removed Use: ArrowArrayExecutor::execute_record_batch --------- Signed-off-by: Joe Isaacs --- benchmarks/compress-bench/src/lib.rs | 26 ------------------ vortex-array/public-api.lock | 6 ----- vortex-array/src/arrow/record_batch.rs | 27 ------------------- .../src/datasets/struct_list_of_ints.rs | 8 ++++-- 4 files changed, 6 insertions(+), 61 deletions(-) diff --git a/benchmarks/compress-bench/src/lib.rs b/benchmarks/compress-bench/src/lib.rs index 3705343fe5a..b1692c40975 100644 --- a/benchmarks/compress-bench/src/lib.rs +++ b/benchmarks/compress-bench/src/lib.rs @@ -1,33 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; - -use ::vortex::array::arrays::ChunkedArray; -use ::vortex::array::arrays::chunked::ChunkedArrayExt; -use ::vortex::array::arrays::listview::recursive_list_from_list_view; -use arrow_array::RecordBatch; -use arrow_schema::Schema; #[cfg(feature = "lance")] pub use lance_bench::compress::LanceCompressor; pub mod parquet; pub mod vortex; - -pub fn chunked_to_vec_record_batch( - chunked: ChunkedArray, -) -> anyhow::Result<(Vec, Arc)> { - assert!(chunked.nchunks() > 0, "empty chunks"); - - let batches = chunked - .iter_chunks() - .map(|array| { - // TODO(connor)[ListView]: The rust Parquet implementation does not support writing - // `ListView` to Parquet files yet. - let converted_array = recursive_list_from_list_view(array.clone())?; - Ok(RecordBatch::try_from(&converted_array)?) - }) - .collect::>>()?; - - let schema = batches[0].schema(); - Ok((batches, schema)) -} diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index dced3a9f4a7..532afd22ec9 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -22120,12 +22120,6 @@ impl core::convert::From for vorte pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::datetime::TemporalData) -> Self -impl core::convert::TryFrom<&vortex_array::ArrayRef> for arrow_array::record_batch::RecordBatch - -pub type arrow_array::record_batch::RecordBatch::Error = vortex_error::VortexError - -pub fn arrow_array::record_batch::RecordBatch::try_from(value: &vortex_array::ArrayRef) -> vortex_error::VortexResult - impl core::convert::TryFrom for vortex_array::arrays::datetime::TemporalData pub type vortex_array::arrays::datetime::TemporalData::Error = vortex_error::VortexError diff --git a/vortex-array/src/arrow/record_batch.rs b/vortex-array/src/arrow/record_batch.rs index 3163e967ca3..b57c307aed0 100644 --- a/vortex-array/src/arrow/record_batch.rs +++ b/vortex-array/src/arrow/record_batch.rs @@ -5,41 +5,14 @@ use arrow_array::RecordBatch; use arrow_array::cast::AsArray; use arrow_schema::DataType; use arrow_schema::Schema; -use vortex_error::VortexError; use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; -use crate::ArrayRef; -use crate::Canonical; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::StructArray; use crate::arrow::ArrowArrayExecutor; -// deprecated(note = "Use ArrowArrayExecutor::execute_record_batch instead") -impl TryFrom<&ArrayRef> for RecordBatch { - type Error = VortexError; - - fn try_from(value: &ArrayRef) -> VortexResult { - let Canonical::Struct(struct_array) = value.to_canonical()? else { - vortex_bail!("RecordBatch can only be constructed from ") - }; - - vortex_ensure!( - struct_array.validity()?.no_nulls(), - "RecordBatch can only be constructed from StructArray with no nulls" - ); - - let data_type = struct_array.dtype().to_arrow_dtype()?; - let array_ref = struct_array - .into_array() - .execute_arrow(Some(&data_type), &mut LEGACY_SESSION.create_execution_ctx())?; - Ok(RecordBatch::from(array_ref.as_struct())) - } -} - impl StructArray { pub fn into_record_batch_with_schema( self, diff --git a/vortex-bench/src/datasets/struct_list_of_ints.rs b/vortex-bench/src/datasets/struct_list_of_ints.rs index 4999c05adac..920d5cf7a52 100644 --- a/vortex-bench/src/datasets/struct_list_of_ints.rs +++ b/vortex-bench/src/datasets/struct_list_of_ints.rs @@ -5,7 +5,6 @@ use std::fs::File; use std::path::PathBuf; use anyhow::Result; -use arrow_array::RecordBatch; use async_trait::async_trait; use parquet::arrow::ArrowWriter; use rand::RngExt; @@ -13,12 +12,15 @@ use rand::SeedableRng; use rand::rngs::StdRng; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::ChunkedArray; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::chunked::ChunkedArrayExt; use vortex::array::arrays::listview::recursive_list_from_list_view; +use vortex::array::arrow::ArrowArrayExecutor; use vortex::array::validity::Validity; use vortex::dtype::FieldNames; @@ -123,7 +125,9 @@ impl Dataset for StructListOfInts { for chunk in chunked.iter_chunks() { let converted = recursive_list_from_list_view(chunk.clone())?; - let batch = RecordBatch::try_from(&converted)?; + let schema = converted.dtype().to_arrow_schema()?; + let batch = converted + .execute_record_batch(&schema, &mut LEGACY_SESSION.create_execution_ctx())?; if writer.is_none() { writer = Some(ArrowWriter::try_new( From c69957f9347d9a63e85538f1d006c0780d579d33 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 16 Apr 2026 19:38:59 +0100 Subject: [PATCH 088/250] fix: avoid ListView take_reduce rebuild for dense selections (#7339) The `take_reduce` path rebuilds the ListView, which is expensive. For dense selections (density above the rebuild threshold) there's little garbage to reclaim, so skip the rebuild and let the regular take path handle it. The rebuild is still worthwhile for sparse takes where we'd otherwise drag around a lot of unused elements when exporting (e.g. to DuckDB). This PR does a very crude estimation of the number of elements that stay in the ListViewArray. Potentially we should do this estimation at export time in the `ListViewExporter`, but that's less trivial. --------- Signed-off-by: Dimitar Dimitrov Signed-off-by: Dimitar Dimitrov Co-authored-by: Joe Isaacs --- vortex-array/public-api.lock | 12 ++ vortex-array/src/arrays/dict/execute.rs | 14 +- .../src/arrays/filter/execute/listview.rs | 28 ++-- .../src/arrays/listview/compute/mod.rs | 12 ++ .../src/arrays/listview/compute/take.rs | 132 ++++++++++-------- 5 files changed, 113 insertions(+), 85 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 532afd22ec9..b388e7e3a21 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2232,6 +2232,10 @@ impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::List pub fn vortex_array::arrays::List::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, indices: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::Primitive pub fn vortex_array::arrays::Primitive::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2970,6 +2974,10 @@ impl vortex_array::ValidityVTable for vortex_arr pub fn vortex_array::arrays::ListView::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>) -> vortex_error::VortexResult +impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, indices: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::arrays::dict::TakeReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, indices: &vortex_array::ArrayRef) -> vortex_error::VortexResult> @@ -5884,6 +5892,10 @@ impl vortex_array::ValidityVTable for vortex_arr pub fn vortex_array::arrays::ListView::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>) -> vortex_error::VortexResult +impl vortex_array::arrays::dict::TakeExecute for vortex_array::arrays::ListView + +pub fn vortex_array::arrays::ListView::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, indices: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + impl vortex_array::arrays::dict::TakeReduce for vortex_array::arrays::ListView pub fn vortex_array::arrays::ListView::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, indices: &vortex_array::ArrayRef) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/dict/execute.rs b/vortex-array/src/arrays/dict/execute.rs index d95cd239a44..19eb6d56b3d 100644 --- a/vortex-array/src/arrays/dict/execute.rs +++ b/vortex-array/src/arrays/dict/execute.rs @@ -46,7 +46,7 @@ pub fn take_canonical( Canonical::Primitive(a) => Canonical::Primitive(take_primitive(&a, codes, ctx)), Canonical::Decimal(a) => Canonical::Decimal(take_decimal(&a, codes, ctx)), Canonical::VarBinView(a) => Canonical::VarBinView(take_varbinview(&a, codes, ctx)), - Canonical::List(a) => Canonical::List(take_listview(&a, codes)), + Canonical::List(a) => Canonical::List(take_listview(&a, codes, ctx)), Canonical::FixedSizeList(a) => { Canonical::FixedSizeList(take_fixed_size_list(&a, codes, ctx)) } @@ -123,12 +123,16 @@ fn take_varbinview( .into_owned() } -fn take_listview(array: &ListViewArray, codes: &PrimitiveArray) -> ListViewArray { +fn take_listview( + array: &ListViewArray, + codes: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> ListViewArray { let codes_ref = codes.clone().into_array(); let array = array.as_view(); - ::take(array, &codes_ref) - .vortex_expect("take listview array") - .vortex_expect("take listview should not return None") + ::take(array, &codes_ref, ctx) + .vortex_expect("take listview execute") + .vortex_expect("ListView TakeExecute should not return None") .as_::() .into_owned() } diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index d1bea9d5128..4f7dbbeab2a 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -9,21 +9,10 @@ use vortex_mask::MaskValues; use crate::arrays::ListViewArray; use crate::arrays::filter::execute::filter_validity; use crate::arrays::filter::execute::values_to_mask; +use crate::arrays::listview; use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; -// TODO(connor)[ListView]: Make use of this threshold after we start migrating operators. -/// The threshold for triggering a rebuild of the [`ListViewArray`]. -/// -/// By default, we will not touch the underlying `elements` array of the [`ListViewArray`] since it -/// can be potentially expensive to reorganize the array based on what views we have into it. -/// -/// However, we also do not want to carry around a large amount of garbage data. Below this -/// threshold of the density of the selection mask, we will rebuild the [`ListViewArray`], removing -/// any garbage data. -#[expect(unused)] -const REBUILD_DENSITY_THRESHOLD: f64 = 0.1; - /// [`ListViewArray`] filter implementation. /// /// This implementation is deliberately simple and read-optimized. We just filter the `offsets` and @@ -70,13 +59,14 @@ pub fn filter_listview(array: &ListViewArray, selection_mask: &Arc) ListViewArray::new_unchecked(elements.clone(), new_offsets, new_sizes, new_validity) }; - // TODO(connor)[ListView]: Ideally, we would only rebuild after all `take`s and `filter` - // compute functions have run, at the "top" of the operator tree. However, we cannot do this - // right now, so we will just rebuild every time (similar to `ListArray`). - - new_array - .rebuild(ListViewRebuildMode::MakeZeroCopyToList) - .vortex_expect("ListViewArray rebuild to zero-copy List should always succeed") + let kept_row_fraction = selection_mask.true_count() as f32 / array.sizes().len() as f32; + if kept_row_fraction < listview::compute::REBUILD_DENSITY_THRESHOLD { + new_array + .rebuild(ListViewRebuildMode::MakeZeroCopyToList) + .vortex_expect("ListViewArray rebuild to zero-copy List should always succeed") + } else { + new_array + } } #[cfg(test)] diff --git a/vortex-array/src/arrays/listview/compute/mod.rs b/vortex-array/src/arrays/listview/compute/mod.rs index 9a43503c4b5..3ea82cafb33 100644 --- a/vortex-array/src/arrays/listview/compute/mod.rs +++ b/vortex-array/src/arrays/listview/compute/mod.rs @@ -6,3 +6,15 @@ mod mask; pub(crate) mod rules; mod slice; mod take; + +/// The threshold below which we rebuild the elements of a listview. +/// +/// We don't touch `elements` on the metadata-only path since reorganizing it can be expensive. +/// However, we also don't want to drag around a large amount of garbage data when the selection +/// is sparse. Below this fraction of list rows retained, the rebuild is worth it. +/// Rebuilding is needed when exporting the ListView's elements. +/// +// TODO(connor)[ListView]: Ideally, we would only rebuild after all `take`s and `filter` +// compute functions have run, at the "top" of the operator tree. However, we cannot do this +// right now, so we will just rebuild every time (similar to [`ListArray`]). +pub(crate) const REBUILD_DENSITY_THRESHOLD: f32 = 0.1; diff --git a/vortex-array/src/arrays/listview/compute/take.rs b/vortex-array/src/arrays/listview/compute/take.rs index 69f40a400a4..04e404a846e 100644 --- a/vortex-array/src/arrays/listview/compute/take.rs +++ b/vortex-array/src/arrays/listview/compute/take.rs @@ -4,11 +4,14 @@ use num_traits::Zero; use vortex_error::VortexResult; +use super::REBUILD_DENSITY_THRESHOLD; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::ListView; use crate::arrays::ListViewArray; +use crate::arrays::dict::TakeExecute; use crate::arrays::dict::TakeReduce; use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; @@ -17,72 +20,79 @@ use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::scalar::Scalar; -// TODO(connor)[ListView]: Make use of this threshold after we start migrating operators. -/// The threshold for triggering a rebuild of the [`ListViewArray`]. -/// -/// By default, we will not touch the underlying `elements` array of the [`ListViewArray`] since it -/// can be potentially expensive to reorganize the array based on what views we have into it. -/// -/// However, we also do not want to carry around a large amount of garbage data. Below this -/// threshold of the density of the selection mask, we will rebuild the [`ListViewArray`], removing -/// any garbage data. -#[expect(unused)] -const REBUILD_DENSITY_THRESHOLD: f64 = 0.1; - -/// [`ListViewArray`] take implementation. -/// -/// This implementation is deliberately simple and read-optimized. We just take the `offsets` and -/// `sizes` at the requested indices and reuse the original `elements` array. This works because -/// `ListView` (unlike `List`) allows non-contiguous and out-of-order lists. -/// -/// We don't slice the `elements` array because it would require computing min/max offsets and -/// adjusting all offsets accordingly, which is not really worth the small potential memory we would -/// be able to get back. -/// -/// The trade-off is that we may keep unreferenced elements in memory, but this is acceptable since -/// we're optimizing for read performance and the data isn't being copied. +/// Metadata-only take for [`ListViewArray`]. impl TakeReduce for ListView { fn take(array: ArrayView<'_, ListView>, indices: &ArrayRef) -> VortexResult> { - let elements = array.elements(); - let offsets = array.offsets(); - let sizes = array.sizes(); + // Approximate element density by the fraction of list rows retained. Assumes roughly + // uniform list sizes; good enough to decide whether dragging along the full `elements` + // buffer is worth avoiding a rebuild. + let kept_row_fraction = indices.len() as f32 / array.sizes().len() as f32; + if kept_row_fraction < REBUILD_DENSITY_THRESHOLD { + return Ok(None); + } - // Compute the new validity by combining the array's validity with the indices' validity. - let new_validity = array.validity()?.take(indices)?; + Ok(Some(apply_take(array, indices)?.into_array())) + } +} - // Take the offsets and sizes arrays at the requested indices. - // Take can reorder offsets, create gaps, and may introduce overlaps if the `indices` - // contain duplicates. - let nullable_new_offsets = offsets.take(indices.clone())?; - let nullable_new_sizes = sizes.take(indices.clone())?; +/// Execution-path take for [`ListViewArray`]. +/// +/// This does the same metadata-only take as [`TakeReduce`], but also rebuilds the array if the +/// resulting array will be less dense than `REBUILD_DENSITY_THRESHOLD`. +impl TakeExecute for ListView { + fn take( + array: ArrayView<'_, ListView>, + indices: &ArrayRef, + _ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let kept_row_fraction = indices.len() as f32 / array.sizes().len() as f32; + let taken = apply_take(array, indices)?; - // Since `take` returns nullable arrays, we simply cast it back to non-nullable (filled with - // zeros to represent null lists). - let new_offsets = match_each_integer_ptype!(nullable_new_offsets.dtype().as_ptype(), |O| { - nullable_new_offsets - .fill_null(Scalar::primitive(O::zero(), Nullability::NonNullable))? - }); - let new_sizes = match_each_integer_ptype!(nullable_new_sizes.dtype().as_ptype(), |S| { - nullable_new_sizes.fill_null(Scalar::primitive(S::zero(), Nullability::NonNullable))? - }); - // SAFETY: Take operation maintains all `ListViewArray` invariants: - // - `new_offsets` and `new_sizes` are derived from existing valid child arrays. - // - `new_offsets` and `new_sizes` are non-nullable. - // - `new_offsets` and `new_sizes` have the same length (both taken with the same - // `indices`). - // - Validity correctly reflects the combination of array and indices validity. - let new_array = unsafe { - ListViewArray::new_unchecked(elements.clone(), new_offsets, new_sizes, new_validity) - }; + if kept_row_fraction < REBUILD_DENSITY_THRESHOLD { + // TODO(connor)[ListView]: Ideally, we would only rebuild after all `take`s and `filter` + // compute functions have run, at the "top" of the operator tree. However, we cannot do + // this right now, so we will just rebuild every time (similar to `ListArray`). + Ok(Some( + taken + .rebuild(ListViewRebuildMode::MakeZeroCopyToList)? + .into_array(), + )) + } else { + Ok(Some(taken.into_array())) + } + } +} - // TODO(connor)[ListView]: Ideally, we would only rebuild after all `take`s and `filter` - // compute functions have run, at the "top" of the operator tree. However, we cannot do this - // right now, so we will just rebuild every time (similar to `ListArray`). +/// Shared metadata-only take: take `offsets`, `sizes` and `validity` at `indices` while reusing +/// the original `elements` buffer as-is. +fn apply_take(array: ArrayView<'_, ListView>, indices: &ArrayRef) -> VortexResult { + let elements = array.elements(); + let offsets = array.offsets(); + let sizes = array.sizes(); - Ok(Some( - new_array - .rebuild(ListViewRebuildMode::MakeZeroCopyToList)? - .into_array(), - )) - } + // Combine the array's validity with the indices' validity. + let new_validity = array.validity()?.take(indices)?; + + // Take can reorder offsets, create gaps, and may introduce overlaps if `indices` contain + // duplicates. + let nullable_new_offsets = offsets.take(indices.clone())?; + let nullable_new_sizes = sizes.take(indices.clone())?; + + // `take` returns nullable arrays; cast back to non-nullable (filling with zeros to represent + // the null lists — the validity mask tracks nullness separately). + let new_offsets = match_each_integer_ptype!(nullable_new_offsets.dtype().as_ptype(), |O| { + nullable_new_offsets.fill_null(Scalar::primitive(O::zero(), Nullability::NonNullable))? + }); + let new_sizes = match_each_integer_ptype!(nullable_new_sizes.dtype().as_ptype(), |S| { + nullable_new_sizes.fill_null(Scalar::primitive(S::zero(), Nullability::NonNullable))? + }); + + // SAFETY: Take operation maintains all `ListViewArray` invariants: + // - `new_offsets` and `new_sizes` are derived from existing valid child arrays. + // - `new_offsets` and `new_sizes` are non-nullable. + // - `new_offsets` and `new_sizes` have the same length (both taken with the same `indices`). + // - Validity correctly reflects the combination of array and indices validity. + Ok(unsafe { + ListViewArray::new_unchecked(elements.clone(), new_offsets, new_sizes, new_validity) + }) } From d441299a2aa9206ce84f5ca3bd842f9cfc8522a9 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 16 Apr 2026 19:49:25 +0100 Subject: [PATCH 089/250] depreacte non compute methods without a ctx (e.g. `to_canonical`) (#7473) All access to buffer should and will require a execution context. This PR deprecates all public methods that need one but don't have one --------- Signed-off-by: Joe Isaacs --- df-q6-parquet.json | 1 + encodings/alp/src/alp/array.rs | 2 + encodings/alp/src/alp/compress.rs | 16 ++++ encodings/alp/src/alp/compute/cast.rs | 23 ++++-- encodings/alp/src/alp/compute/compare.rs | 29 +++---- encodings/alp/src/alp/compute/filter.rs | 5 +- encodings/alp/src/alp/compute/mask.rs | 5 +- encodings/alp/src/alp/compute/take.rs | 5 +- encodings/alp/src/alp_rd/array.rs | 6 +- encodings/alp/src/alp_rd/compute/cast.rs | 2 + encodings/alp/src/alp_rd/compute/take.rs | 3 + encodings/datetime-parts/src/compress.rs | 18 +++-- encodings/datetime-parts/src/compute/cast.rs | 2 + encodings/datetime-parts/src/compute/take.rs | 2 + .../src/decimal_byte_parts/compute/cast.rs | 3 + .../fastlanes/benches/compute_between.rs | 2 + .../src/bitpacking/array/bitpack_compress.rs | 8 ++ .../bitpacking/array/bitpack_decompress.rs | 9 +++ .../fastlanes/src/bitpacking/array/mod.rs | 9 ++- .../src/bitpacking/compute/filter.rs | 8 +- .../src/bitpacking/compute/is_constant.rs | 3 + .../fastlanes/src/bitpacking/compute/take.rs | 6 +- .../src/delta/array/delta_compress.rs | 5 +- .../fastlanes/src/delta/vtable/operations.rs | 2 + .../fastlanes/src/for/array/for_compress.rs | 2 + .../fastlanes/src/for/compute/is_sorted.rs | 2 + encodings/fastlanes/src/lib.rs | 2 + encodings/fastlanes/src/rle/array/mod.rs | 8 ++ .../fastlanes/src/rle/array/rle_compress.rs | 13 +++ encodings/fastlanes/src/rle/compute/cast.rs | 7 +- .../fastlanes/src/rle/vtable/operations.rs | 4 + encodings/fsst/src/canonical.rs | 2 + encodings/fsst/src/compute/compare.rs | 3 + encodings/fsst/src/compute/like.rs | 2 + encodings/fsst/src/tests.rs | 2 + encodings/pco/src/array.rs | 5 +- encodings/pco/src/tests.rs | 2 + encodings/runend/src/compress.rs | 11 ++- encodings/runend/src/compute/cast.rs | 2 + encodings/runend/src/compute/take.rs | 2 + encodings/runend/src/compute/take_from.rs | 16 +++- encodings/runend/src/decompress_bool.rs | 2 + encodings/sequence/src/compress.rs | 4 + encodings/sequence/src/compute/cast.rs | 4 + encodings/sparse/src/canonical.rs | 26 +++++- encodings/sparse/src/compute/cast.rs | 9 ++- encodings/sparse/src/lib.rs | 23 +++--- encodings/sparse/src/ops.rs | 2 + encodings/zigzag/src/array.rs | 2 + encodings/zigzag/src/compress.rs | 29 +++---- encodings/zigzag/src/compute/mod.rs | 5 +- encodings/zstd/src/array.rs | 9 ++- encodings/zstd/src/compute/cast.rs | 4 + encodings/zstd/src/test.rs | 6 ++ fuzz/fuzz_targets/file_io.rs | 8 +- fuzz/src/array/cast.rs | 79 ++++++++++--------- fuzz/src/array/compare.rs | 56 +++++++------ fuzz/src/array/fill_null.rs | 20 +++-- fuzz/src/array/filter.rs | 6 ++ fuzz/src/array/mask.rs | 2 + fuzz/src/array/mod.rs | 61 +++++++------- fuzz/src/array/scalar_at.rs | 42 +++++----- fuzz/src/array/search_sorted.rs | 5 ++ fuzz/src/array/slice.rs | 8 ++ fuzz/src/array/sort.rs | 5 ++ fuzz/src/array/take.rs | 6 ++ fuzz/src/compress.rs | 1 + results.json | 0 vortex-array/benches/take_patches.rs | 17 +++- vortex-array/benches/varbinview_compact.rs | 5 +- vortex-array/src/array/erased.rs | 6 +- vortex-array/src/array/typed.rs | 4 +- vortex-array/src/arrays/arbitrary.rs | 15 ++-- .../src/arrays/bool/compute/fill_null.rs | 4 +- vortex-array/src/arrays/bool/compute/take.rs | 4 +- .../src/arrays/bool/vtable/operations.rs | 5 +- vortex-array/src/arrays/chunked/array.rs | 23 +++--- .../src/arrays/chunked/compute/take.rs | 4 +- .../src/arrays/chunked/compute/zip.rs | 4 +- .../src/arrays/chunked/paired_chunks.rs | 5 +- vortex-array/src/arrays/chunked/tests.rs | 7 +- .../src/arrays/chunked/vtable/canonical.rs | 7 +- vortex-array/src/arrays/chunked/vtable/mod.rs | 4 +- .../src/arrays/constant/compute/take.rs | 5 +- .../src/arrays/constant/vtable/canonical.rs | 40 +++++++++- vortex-array/src/arrays/datetime/test.rs | 10 +-- .../src/arrays/decimal/compute/cast.rs | 16 +++- .../src/arrays/decimal/compute/fill_null.rs | 7 +- vortex-array/src/arrays/dict/array.rs | 8 +- vortex-array/src/arrays/dict/compute/cast.rs | 13 +-- .../src/arrays/dict/compute/fill_null.rs | 4 +- vortex-array/src/arrays/dict/compute/mod.rs | 7 +- vortex-array/src/arrays/dict/vtable/mod.rs | 2 +- .../src/arrays/extension/compute/cast.rs | 12 +-- .../src/arrays/extension/compute/rules.rs | 8 +- .../src/arrays/filter/execute/bool.rs | 4 +- .../src/arrays/filter/execute/listview.rs | 9 ++- .../src/arrays/filter/execute/primitive.rs | 4 +- .../arrays/fixed_size_list/compute/take.rs | 4 +- .../arrays/fixed_size_list/tests/nested.rs | 15 +++- vortex-array/src/arrays/list/array.rs | 4 +- vortex-array/src/arrays/list/compute/cast.rs | 18 +++-- vortex-array/src/arrays/list/compute/take.rs | 7 +- vortex-array/src/arrays/listview/array.rs | 28 ++++--- .../src/arrays/listview/conversion.rs | 5 +- vortex-array/src/arrays/listview/rebuild.rs | 13 ++- .../src/arrays/listview/tests/filter.rs | 9 ++- .../src/arrays/listview/tests/operations.rs | 11 ++- .../src/arrays/listview/tests/take.rs | 8 +- vortex-array/src/arrays/masked/tests.rs | 2 + vortex-array/src/arrays/null/compute/mod.rs | 5 +- vortex-array/src/arrays/null/compute/take.rs | 4 +- .../src/arrays/patched/compute/take.rs | 6 ++ .../src/arrays/primitive/array/accessor.rs | 4 +- .../src/arrays/primitive/array/mod.rs | 41 +++++++--- .../src/arrays/primitive/array/patch.rs | 7 +- .../src/arrays/primitive/compute/cast.rs | 17 +++- .../src/arrays/primitive/compute/fill_null.rs | 7 +- .../src/arrays/struct_/compute/cast.rs | 11 ++- .../src/arrays/struct_/compute/rules.rs | 6 +- vortex-array/src/arrays/varbin/accessor.rs | 5 +- vortex-array/src/arrays/varbin/array.rs | 4 +- .../src/arrays/varbin/compute/compare.rs | 5 +- .../src/arrays/varbin/vtable/canonical.rs | 6 +- .../src/arrays/varbinview/accessor.rs | 4 +- .../src/arrays/varbinview/compute/mod.rs | 15 ++-- .../src/arrays/varbinview/compute/take.rs | 27 ++++--- .../src/arrays/varbinview/compute/zip.rs | 4 +- vortex-array/src/arrays/varbinview/tests.rs | 4 +- vortex-array/src/builders/bool.rs | 9 ++- vortex-array/src/builders/decimal.rs | 4 +- vortex-array/src/builders/dict/bytes.rs | 43 +++++----- vortex-array/src/builders/dict/mod.rs | 7 +- vortex-array/src/builders/dict/primitive.rs | 7 +- vortex-array/src/builders/extension.rs | 4 +- vortex-array/src/builders/fixed_size_list.rs | 23 +++++- vortex-array/src/builders/list.rs | 17 +++- vortex-array/src/builders/listview.rs | 5 +- vortex-array/src/builders/primitive.rs | 4 +- vortex-array/src/builders/struct_.rs | 4 +- vortex-array/src/builders/tests.rs | 1 + vortex-array/src/builders/varbinview.rs | 4 +- vortex-array/src/canonical.rs | 65 ++++++++------- .../src/compute/conformance/binary_numeric.rs | 5 +- .../src/compute/conformance/consistency.rs | 19 ++--- vortex-array/src/compute/conformance/take.rs | 17 ++-- vortex-array/src/display/mod.rs | 4 +- vortex-array/src/normalize.rs | 4 +- vortex-array/src/patches.rs | 12 ++- vortex-array/src/scalar_fn/fns/between/mod.rs | 7 +- .../src/scalar_fn/fns/binary/boolean.rs | 5 +- .../src/scalar_fn/fns/binary/compare.rs | 9 ++- .../src/scalar_fn/fns/list_contains/mod.rs | 22 ++++-- vortex-array/src/scalar_fn/fns/merge.rs | 24 ++++-- vortex-array/src/scalar_fn/fns/not/mod.rs | 14 ++-- vortex-array/src/scalar_fn/fns/pack.rs | 19 ++++- vortex-array/src/scalar_fn/fns/select.rs | 5 +- vortex-bench/src/datasets/tpch_l_comment.rs | 15 ++-- vortex-btrblocks/benches/compress.rs | 2 + vortex-btrblocks/src/schemes/decimal.rs | 2 + vortex-btrblocks/src/schemes/float.rs | 2 + vortex-btrblocks/src/schemes/integer.rs | 72 +++++++++-------- vortex-btrblocks/src/schemes/patches.rs | 8 +- vortex-btrblocks/src/schemes/string.rs | 20 +++-- vortex-btrblocks/src/schemes/temporal.rs | 31 +++----- vortex-compressor/src/builtins/dict/float.rs | 2 + .../src/builtins/dict/integer.rs | 2 + vortex-compressor/src/compressor.rs | 21 ++--- vortex-compressor/src/stats/cache.rs | 17 +++- vortex-compressor/src/stats/float.rs | 2 + vortex-cuda/benches/dynamic_dispatch_cuda.rs | 2 + vortex-cuda/gpu-scan-cli/src/main.rs | 4 +- vortex-cuda/src/arrow/canonical.rs | 2 + vortex-cuda/src/dynamic_dispatch/mod.rs | 2 + vortex-cuda/src/kernel/patches/mod.rs | 7 +- vortex-duckdb/src/convert/vector.rs | 18 +++++ vortex-duckdb/src/exporter/dict.rs | 1 + vortex-ffi/src/array.rs | 6 +- vortex-ffi/src/expression.rs | 8 +- vortex-ffi/src/struct_array.rs | 2 + vortex-file/src/tests.rs | 30 +++++++ vortex-file/tests/test_write_table.rs | 3 + vortex-jni/src/array.rs | 23 +++--- vortex-layout/src/layouts/dict/reader.rs | 13 ++- vortex-layout/src/layouts/file_stats.rs | 2 + vortex-layout/src/layouts/flat/writer.rs | 35 ++++---- vortex-layout/src/layouts/repartition.rs | 15 +++- vortex-layout/src/layouts/row_idx/mod.rs | 4 +- vortex-layout/src/layouts/struct_/reader.rs | 11 ++- vortex-layout/src/layouts/table.rs | 2 + vortex-layout/src/layouts/zoned/reader.rs | 2 + vortex-layout/src/layouts/zoned/zone_map.rs | 39 +++------ vortex-layout/src/scan/scan_builder.rs | 5 +- vortex-python/src/arrays/compressed.rs | 7 +- vortex-python/src/arrays/mod.rs | 11 ++- vortex-python/src/dataset.rs | 5 +- vortex-python/src/file.rs | 8 +- vortex-tui/src/browse/ui/layouts.rs | 2 + .../common_encoding_tree_throughput.rs | 13 ++- vortex/benches/single_encoding_throughput.rs | 3 + 200 files changed, 1465 insertions(+), 631 deletions(-) create mode 100644 df-q6-parquet.json create mode 100644 results.json diff --git a/df-q6-parquet.json b/df-q6-parquet.json new file mode 100644 index 00000000000..0df39f7ed3a --- /dev/null +++ b/df-q6-parquet.json @@ -0,0 +1 @@ +{"all_runtimes":[140238709],"commit_id":"5b3726ee1725a1c21e3ab935d21ef48435e97197","dataset":{"tpch":{"scale_factor":"10.0"}},"env_triple":{"architecture":"aarch64","environment":"unknown","operating_system":"darwin"},"name":"tpch_q06/datafusion:parquet","storage":"nvme","target":{"engine":"datafusion","format":"parquet"},"unit":"ns","value":140238709} diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 8a86019dcf9..887111299ab 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -526,6 +526,7 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -808,6 +809,7 @@ mod tests { let slice_len = slice_end - slice_start; let sliced_encoded = encoded.slice(slice_start..slice_end).unwrap(); + #[expect(deprecated)] let result_primitive = sliced_encoded.to_primitive(); for idx in 0..slice_len { diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 059c4b8821c..7be9c0b6537 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -136,6 +136,7 @@ mod tests { use f64::consts::E; use f64::consts::PI; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; @@ -280,6 +281,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(); + #[expect(deprecated)] let decompressed = alp_arr.into_array().to_primitive(); assert_eq!( @@ -303,6 +305,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(); + #[expect(deprecated)] let decoded = encoded.as_array().to_primitive(); for idx in 0..original.len() { let decoded_val = decoded.as_slice::()[idx]; @@ -331,14 +334,17 @@ mod tests { .unwrap(); let patches = encoded.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 3]); assert_arrays_eq!(chunk_offsets, expected_offsets); + #[expect(deprecated)] let patch_indices = patches.indices().to_primitive(); let expected_indices = PrimitiveArray::from_iter(vec![1023u64, 1024, 1025]); assert_arrays_eq!(patch_indices, expected_indices); + #[expect(deprecated)] let patch_values = patches.values().to_primitive(); let expected_values = PrimitiveArray::from_iter(vec![PI, E, PI]); assert_arrays_eq!(patch_values, expected_values); @@ -359,14 +365,17 @@ mod tests { .unwrap(); let patches = encoded.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]); assert_arrays_eq!(chunk_offsets, expected_offsets); + #[expect(deprecated)] let patch_indices = patches.indices().to_primitive(); let expected_indices = PrimitiveArray::from_iter(vec![0u64, 2048]); assert_arrays_eq!(patch_indices, expected_indices); + #[expect(deprecated)] let patch_values = patches.values().to_primitive(); let expected_values = PrimitiveArray::from_iter(vec![PI, E]); assert_arrays_eq!(patch_values, expected_values); @@ -386,14 +395,17 @@ mod tests { .unwrap(); let patches = encoded.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]); assert_arrays_eq!(chunk_offsets, expected_offsets); + #[expect(deprecated)] let patch_indices = patches.indices().to_primitive(); let expected_indices = PrimitiveArray::from_iter(vec![0u64]); assert_arrays_eq!(patch_indices, expected_indices); + #[expect(deprecated)] let patch_values = patches.values().to_primitive(); let expected_values = PrimitiveArray::from_iter(vec![PI]); assert_arrays_eq!(patch_values, expected_values); @@ -414,14 +426,17 @@ mod tests { .unwrap(); let patches = encoded.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64]); assert_arrays_eq!(chunk_offsets, expected_offsets); + #[expect(deprecated)] let patch_indices = patches.indices().to_primitive(); let expected_indices = PrimitiveArray::from_iter(vec![0u64, 100]); assert_arrays_eq!(patch_indices, expected_indices); + #[expect(deprecated)] let patch_values = patches.values().to_primitive(); let expected_values = PrimitiveArray::from_iter(vec![PI, E]); assert_arrays_eq!(patch_values, expected_values); @@ -524,6 +539,7 @@ mod tests { .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); + #[expect(deprecated)] let decoded = sliced_alp.to_primitive(); let expected_slice = original.slice(512..1024).unwrap(); diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index b6ce03b9eee..99eaa09ff2c 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -61,6 +61,7 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -80,8 +81,10 @@ mod tests { #[test] fn issue_5766_test_cast_alp_with_patches_to_nullable() -> VortexResult<()> { let values = buffer![1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456].into_array(); + #[expect(deprecated)] + let values_primitive = values.to_primitive(); let alp = alp_encode( - values.to_primitive().as_view(), + values_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), )?; @@ -96,7 +99,9 @@ mod tests { let expected = values.cast(nullable_dtype)?; - assert_arrays_eq!(casted.to_canonical()?.into_primitive(), expected); + #[expect(deprecated)] + let casted_prim = casted.to_canonical()?.into_primitive(); + assert_arrays_eq!(casted_prim, expected); Ok(()) } @@ -104,8 +109,10 @@ mod tests { #[test] fn test_cast_alp_f32_to_f64() -> VortexResult<()> { let values = buffer![1.5f32, 2.5, 3.5, 4.5].into_array(); + #[expect(deprecated)] + let values_primitive = values.to_primitive(); let alp = alp_encode( - values.to_primitive().as_view(), + values_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), )?; @@ -118,6 +125,7 @@ mod tests { &DType::Primitive(PType::F64, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_canonical()?.into_primitive(); let values = decoded.as_slice::(); assert_eq!(values.len(), 4); @@ -130,8 +138,10 @@ mod tests { #[test] fn test_cast_alp_to_int() -> VortexResult<()> { let values = buffer![1.0f32, 2.0, 3.0, 4.0].into_array(); + #[expect(deprecated)] + let values_primitive = values.to_primitive(); let alp = alp_encode( - values.to_primitive().as_view(), + values_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), )?; @@ -144,6 +154,7 @@ mod tests { &DType::Primitive(PType::I32, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_canonical()?.into_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i32, 2, 3, 4])); @@ -157,8 +168,10 @@ mod tests { #[case(buffer![42.42f64].into_array())] #[case(buffer![0.0f32, -1.5, 2.5, -3.5, 4.5].into_array())] fn test_cast_alp_conformance(#[case] array: vortex_array::ArrayRef) -> VortexResult<()> { + #[expect(deprecated)] + let array_primitive = array.to_primitive(); let alp = alp_encode( - array.to_primitive().as_view(), + array_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index ebda0a7797c..d0eca850b8c 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -153,6 +153,7 @@ mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -192,10 +193,9 @@ mod tests { ) .unwrap(); assert!(encoded.patches().is_none()); - assert_eq!( - encoded.encoded().to_primitive().as_slice::(), - vec![1234; 1025] - ); + #[expect(deprecated)] + let encoded_prim = encoded.encoded().to_primitive(); + assert_eq!(encoded_prim.as_slice::(), vec![1234; 1025]); let r = alp_scalar_compare(encoded.as_view(), 1.3_f32, CompareOperator::Eq) .unwrap() @@ -220,10 +220,9 @@ mod tests { ) .unwrap(); assert!(encoded.patches().is_none()); - assert_eq!( - encoded.encoded().to_primitive().as_slice::(), - vec![1234; 1025] - ); + #[expect(deprecated)] + let encoded_prim = encoded.encoded().to_primitive(); + assert_eq!(encoded_prim.as_slice::(), vec![1234; 1025]); let r_eq = alp_scalar_compare(encoded.as_view(), 1.234444_f32, CompareOperator::Eq) .unwrap() @@ -248,10 +247,9 @@ mod tests { ) .unwrap(); assert!(encoded.patches().is_none()); - assert_eq!( - encoded.encoded().to_primitive().as_slice::(), - vec![605; 10] - ); + #[expect(deprecated)] + let encoded_prim = encoded.encoded().to_primitive(); + assert_eq!(encoded_prim.as_slice::(), vec![605; 10]); // !(0.0605_f32 >= 0.06051_f32); let r_gte = alp_scalar_compare(encoded.as_view(), 0.06051_f32, CompareOperator::Gte) @@ -292,10 +290,9 @@ mod tests { ) .unwrap(); assert!(encoded.patches().is_none()); - assert_eq!( - encoded.encoded().to_primitive().as_slice::(), - vec![0; 10] - ); + #[expect(deprecated)] + let encoded_prim = encoded.encoded().to_primitive(); + assert_eq!(encoded_prim.as_slice::(), vec![0; 10]); let r_gte = test_alp_compare(encoded.as_view(), -0.00000001_f32, CompareOperator::Gte).unwrap(); diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index c305ea5e99b..2843e72f8e1 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -45,6 +45,7 @@ mod test { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -63,8 +64,10 @@ mod test { 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 ].into_array())] fn test_filter_alp_conformance(#[case] array: ArrayRef) { + #[expect(deprecated)] + let array_primitive = array.to_primitive(); let alp = alp_encode( - array.to_primitive().as_view(), + array_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 7b9757376d9..2123e4d85f8 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -57,6 +57,7 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -79,8 +80,10 @@ mod test { 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0 ].into_array())] fn test_mask_alp_conformance(#[case] array: vortex_array::ArrayRef) { + #[expect(deprecated)] + let array_primitive = array.to_primitive(); let alp = alp_encode( - array.to_primitive().as_view(), + array_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index 3fcfa7c9f77..daeead2a4d0 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -43,6 +43,7 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -57,8 +58,10 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).into_array())] #[case(buffer![42.42f64].into_array())] fn test_take_alp_conformance(#[case] array: vortex_array::ArrayRef) { + #[expect(deprecated)] + let array_primitive = array.to_primitive(); let alp = alp_encode( - array.to_primitive().as_view(), + array_primitive.as_view(), None, &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index ae224c52b7d..a9bce5c7f78 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -418,7 +418,9 @@ impl ALPRDData { let mut patches = patches.cast_values(&left_parts.dtype().as_nonnullable())?; // Force execution of the lazy cast so patch values are materialized // before serialization. - *patches.values_mut() = patches.values().to_canonical()?.into_array(); + #[expect(deprecated)] + let canonical = patches.values().to_canonical()?.into_array(); + *patches.values_mut() = canonical; Ok(patches) }) .transpose() @@ -660,6 +662,7 @@ impl ValidityChild for ALPRD { mod test { use prost::Message; use rstest::rstest; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -693,6 +696,7 @@ mod test { let rd_array = encoder.encode(real_array.as_view()); + #[expect(deprecated)] let decoded = rd_array.as_array().to_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_option_iter(reals)); diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 86b6ff9c331..69d3252ba89 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -50,6 +50,7 @@ impl CastReduce for ALPRD { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; @@ -76,6 +77,7 @@ mod tests { &DType::Primitive(PType::F64, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_primitive(); let f64_values = decoded.as_slice::(); assert_eq!(f64_values.len(), 5); diff --git a/encodings/alp/src/alp_rd/compute/take.rs b/encodings/alp/src/alp_rd/compute/take.rs index 12cf8e935c1..fac46e74d02 100644 --- a/encodings/alp/src/alp_rd/compute/take.rs +++ b/encodings/alp/src/alp_rd/compute/take.rs @@ -58,6 +58,7 @@ impl TakeExecute for ALPRD { mod test { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -86,6 +87,7 @@ mod test { .is_unsigned_int() ); + #[expect(deprecated)] let taken = encoded .take(buffer![0, 2].into_array()) .unwrap() @@ -110,6 +112,7 @@ mod test { .is_unsigned_int() ); + #[expect(deprecated)] let taken = encoded .take(PrimitiveArray::from_option_iter([Some(0), Some(2), None]).into_array()) .unwrap() diff --git a/encodings/datetime-parts/src/compress.rs b/encodings/datetime-parts/src/compress.rs index 2edb6097c7b..bc25f0eb999 100644 --- a/encodings/datetime-parts/src/compress.rs +++ b/encodings/datetime-parts/src/compress.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; @@ -26,9 +27,11 @@ pub struct TemporalParts { /// Splitting the components by granularity creates more small values, which enables better /// cascading compression. pub fn split_temporal(array: TemporalArray) -> VortexResult { + #[expect(deprecated)] let temporal_values = array.temporal_values().to_primitive(); // After this operation, timestamps will be a PrimitiveArray + #[expect(deprecated)] let timestamps = temporal_values .clone() .into_array() @@ -83,6 +86,7 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -119,23 +123,27 @@ mod tests { } = split_temporal(temporal_array).unwrap(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); + #[expect(deprecated)] + let days_prim = days.to_primitive(); assert!( - days.to_primitive() + days_prim .validity() .vortex_expect("days validity should be derivable") .mask_eq(&validity, &mut ctx) .unwrap() ); + #[expect(deprecated)] + let seconds_prim = seconds.to_primitive(); assert!(matches!( - seconds - .to_primitive() + seconds_prim .validity() .vortex_expect("seconds validity should be derivable"), Validity::NonNullable )); + #[expect(deprecated)] + let subseconds_prim = subseconds.to_primitive(); assert!(matches!( - subseconds - .to_primitive() + subseconds_prim .validity() .vortex_expect("subseconds validity should be derivable"), Validity::NonNullable diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 0e56d594dcb..573a0af48d7 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -91,12 +91,14 @@ mod tests { fn test_bad_cast_fails(#[case] validity: Validity) { let array = date_time_array(validity); // Cast to incompatible type - force evaluation via to_canonical + #[expect(deprecated)] let result = array .cast(DType::Bool(Nullability::NonNullable)) .and_then(|a| a.to_canonical().map(|c| c.into_array())); assert!(result.is_err(), "Expected error, got: {result:?}"); // Cast nullable with nulls to non-nullable - force evaluation via to_canonical + #[expect(deprecated)] let result = array .cast(array.dtype().with_nullability(Nullability::NonNullable)) .and_then(|a| a.to_canonical().map(|c| c.into_array())); diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index cd15c466b2a..f5f67836e56 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::dict::TakeExecute; use vortex_array::builtins::ArrayBuiltins; @@ -22,6 +23,7 @@ fn take_datetime_parts( indices: &ArrayRef, ) -> VortexResult { // we go ahead and canonicalize here to avoid worst-case canonicalizing 3 separate times + #[expect(deprecated)] let indices = indices.to_primitive(); let taken_days = array.days().take(indices.clone().into_array())?; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index e554bbfae71..30d7bcd9a5f 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -47,6 +47,7 @@ impl CastReduce for DecimalByteParts { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; @@ -77,6 +78,7 @@ mod tests { ); // Verify the values are preserved + #[expect(deprecated)] let decoded = casted.to_decimal(); assert_eq!(decoded.len(), 4); } @@ -91,6 +93,7 @@ mod tests { .unwrap(); // Cast to non-nullable should fail due to nulls - force evaluation via to_canonical + #[expect(deprecated)] let result = array .into_array() .cast(DType::Decimal(decimal_dtype, Nullability::NonNullable)) diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index 49f518a7949..d43a2d91639 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -12,6 +12,7 @@ use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -58,6 +59,7 @@ fn generate_alp_bit_pack_primitive_array( ) .vortex_expect(""); + #[expect(deprecated)] let encoded = alp.encoded().to_primitive(); let bp = bitpack_to_best_bit_width(&encoded) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 727e69dad04..537e7c07a94 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -389,6 +389,7 @@ pub mod test_harness { use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; @@ -413,6 +414,7 @@ pub mod test_harness { }) .collect::>(); + #[expect(deprecated)] let values = if fraction_null == 0.0 { values.into_array().to_primitive() } else { @@ -430,6 +432,7 @@ mod test { use rand::SeedableRng; use rand::rngs::StdRng; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; @@ -506,6 +509,7 @@ mod test { .collect::>(); let chunked = ChunkedArray::from_iter(chunks).into_array(); + #[expect(deprecated)] let into_ca = chunked.to_primitive(); let mut primitive_builder = PrimitiveBuilder::::with_capacity(chunked.dtype().nullability(), 10 * 100); @@ -538,6 +542,7 @@ mod test { let bitpacked = bitpack_encode(&array, 4, None).unwrap(); let patches = bitpacked.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -561,6 +566,7 @@ mod test { let bitpacked = bitpack_encode(&array, 4, None).unwrap(); let patches = bitpacked.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); @@ -580,6 +586,7 @@ mod test { let bitpacked = bitpack_encode(&array, 4, None).unwrap(); let patches = bitpacked.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -604,6 +611,7 @@ mod test { let bitpacked = bitpack_encode(&array, 4, None).unwrap(); let patches = bitpacked.patches().unwrap(); + #[expect(deprecated)] let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // Single chunk starting at patch index 0. diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 337f82691a6..4ddbd1832cc 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -160,6 +160,7 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; @@ -219,6 +220,7 @@ mod tests { #[test] fn test_all_zeros() -> VortexResult<()> { + #[expect(deprecated)] let zeros = buffer![0u16, 0, 0, 0].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); let actual = unpack(&bitpacked)?; @@ -228,6 +230,7 @@ mod tests { #[test] fn test_simple_patches() -> VortexResult<()> { + #[expect(deprecated)] let zeros = buffer![0u16, 1, 0, 1].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); let actual = unpack(&bitpacked)?; @@ -237,6 +240,7 @@ mod tests { #[test] fn test_one_full_chunk() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter(0u16..1024).into_array().to_primitive(); let bitpacked = encode(&zeros, 10); let actual = unpack(&bitpacked)?; @@ -246,6 +250,7 @@ mod tests { #[test] fn test_three_full_chunks_with_patches() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter((5u16..1029).chain(5u16..1029).chain(5u16..1029)) .into_array() .to_primitive(); @@ -261,6 +266,7 @@ mod tests { #[test] fn test_one_full_chunk_and_one_short_chunk_no_patch() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); let bitpacked = encode(&zeros, 11); assert!(bitpacked.patches().is_none()); @@ -271,6 +277,7 @@ mod tests { #[test] fn test_one_full_chunk_and_one_short_chunk_with_patches() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter(512u16..1537) .into_array() .to_primitive(); @@ -284,6 +291,7 @@ mod tests { #[test] fn test_offset_and_short_chunk_and_patches() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter(512u16..1537) .into_array() .to_primitive(); @@ -304,6 +312,7 @@ mod tests { #[test] fn test_offset_and_short_chunk_with_chunks_between_and_patches() -> VortexResult<()> { + #[expect(deprecated)] let zeros = BufferMut::from_iter(512u16..2741) .into_array() .to_primitive(); diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index dbca8c0e48b..ba5029260a7 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -333,6 +333,7 @@ impl> BitPackedArrayExt for T {} #[cfg(test)] mod test { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -355,7 +356,9 @@ mod test { let uncompressed = PrimitiveArray::from_option_iter(values); let packed = BitPackedData::encode(&uncompressed.into_array(), 1).unwrap(); let expected = PrimitiveArray::from_option_iter(values); - assert_arrays_eq!(packed.as_array().to_primitive(), expected); + #[expect(deprecated)] + let packed_primitive = packed.as_array().to_primitive(); + assert_arrays_eq!(packed_primitive, expected); } #[test] @@ -375,8 +378,10 @@ mod test { let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); assert!(packed_with_patches.patches().is_some()); + #[expect(deprecated)] + let packed_primitive = packed_with_patches.as_array().to_primitive(); assert_arrays_eq!( - packed_with_patches.as_array().to_primitive(), + packed_primitive, PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) ); } diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index aa3ca07d0b0..1b161889560 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -178,6 +178,7 @@ fn filter_with_indices( #[cfg(test)] mod test { use vortex_array::IntoArray as _; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -225,8 +226,10 @@ mod test { let filtered = bitpacked .filter(Mask::from_indices(4096, (0..1024).collect())) .unwrap(); + #[expect(deprecated)] + let filtered_prim = filtered.to_primitive(); assert_arrays_eq!( - filtered.to_primitive(), + filtered_prim, PrimitiveArray::from_iter((0..1024).map(|i| (i % 63) as u8)) ); } @@ -236,6 +239,7 @@ mod test { let values: Buffer = (0..500).collect(); let unpacked = PrimitiveArray::new(values.clone(), Validity::NonNullable); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 9).unwrap(); + #[expect(deprecated)] let filtered = bitpacked .filter(Mask::from_indices(values.len(), (0..250).collect())) .unwrap() @@ -283,6 +287,7 @@ mod test { ); // Filter to include some patched and some non-patched values. + #[expect(deprecated)] let filtered = bitpacked .filter(Mask::from_indices(values.len(), vec![0, 2, 5, 9])) .unwrap() @@ -316,6 +321,7 @@ mod test { // Use low selectivity (only select 2% of values) to avoid full decompression. let indices: Vec = (0..20).collect(); + #[expect(deprecated)] let filtered = bitpacked .filter(Mask::from_indices(values.len(), indices)) .unwrap() diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 94f3feea46f..cdf27dade75 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -8,6 +8,7 @@ use lending_iterator::LendingIterator; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; @@ -57,7 +58,9 @@ fn bitpacked_is_constant( ) -> VortexResult { let mut bit_unpack_iterator = array.unpacked_chunks::()?; let patches = array.patches().map(|p| { + #[expect(deprecated)] let values = p.values().to_primitive(); + #[expect(deprecated)] let indices = p.indices().to_primitive(); let offset = p.offset(); (indices, values, offset) diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 4ff2b484818..b7ea83513c7 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -166,6 +166,7 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -281,9 +282,10 @@ mod test { taken_primitive, PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(4)]) ); + #[expect(deprecated)] + let taken_primitive_prim = taken_primitive.to_primitive(); assert_eq!( - taken_primitive - .to_primitive() + taken_primitive_prim .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), 1 diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 105004c0c71..c13e9f60d4c 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -96,6 +96,7 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -144,6 +145,8 @@ mod tests { array.len(), ) .vortex_expect("Delta array construction should succeed"); - assert_arrays_eq!(packed_delta.as_array().to_primitive(), array); + #[expect(deprecated)] + let packed_delta_prim = packed_delta.as_array().to_primitive(); + assert_arrays_eq!(packed_delta_prim, array); } } diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index d0c0606ea93..72e8c2ab835 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; @@ -16,6 +17,7 @@ impl OperationsVTable for Delta { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { + #[expect(deprecated)] let decompressed = array.array().slice(index..index + 1)?.to_primitive(); decompressed.into_array().execute_scalar(0, ctx) } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index f27379a866f..0be93fa955d 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -52,6 +52,7 @@ mod test { use std::sync::LazyLock; use itertools::Itertools; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -166,6 +167,7 @@ mod test { .unwrap() ); + #[expect(deprecated)] let encoded = compressed .encoded() .to_primitive() diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 92cd8b36888..eba41b07927 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; @@ -35,6 +36,7 @@ impl DynAggregateKernel for FoRIsSortedKernel { return Ok(None); }; + #[expect(deprecated)] let encoded = array.encoded().to_primitive(); let unsigned_array = PrimitiveArray::from_buffer_handle( encoded.buffer_handle().clone(), diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index bed17366676..0041cf99b15 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -7,6 +7,7 @@ pub use bitpacking::*; pub use delta::*; pub use r#for::*; pub use rle::*; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::validity::Validity; @@ -82,6 +83,7 @@ pub(crate) fn fill_forward_nulls( Validity::NonNullable | Validity::AllValid => values, Validity::AllInvalid => Buffer::zeroed(values.len()), Validity::Array(validity_array) => { + #[expect(deprecated)] let bit_buffer = validity_array.to_bool().to_bit_buffer(); let mut last_valid = T::default(); match values.try_into_mut() { diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 5c9ab27c33a..d8f822c19ff 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -142,6 +142,7 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -236,6 +237,7 @@ mod tests { let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) .vortex_expect("RLEData is always valid"); + #[expect(deprecated)] let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); // TODO(joe): replace with compute null count let mut ctx = SESSION.create_execution_ctx(); @@ -364,6 +366,7 @@ mod tests { let rle_array = RLEData::encode(primitive.as_view()).unwrap(); assert_eq!(rle_array.len(), 2048); + #[expect(deprecated)] let original_data = rle_array.as_array().to_primitive(); let ctx = ArrayContext::empty(); @@ -388,6 +391,7 @@ mod tests { ) .unwrap(); + #[expect(deprecated)] let decoded_data = decoded.to_primitive(); assert_arrays_eq!(original_data, decoded_data); @@ -431,7 +435,9 @@ mod tests { ) .unwrap(); + #[expect(deprecated)] let original_data = sliced.as_array().to_primitive(); + #[expect(deprecated)] let decoded_data = decoded.to_primitive(); assert_arrays_eq!(original_data, decoded_data); @@ -459,6 +465,7 @@ mod tests { // Simulate cascading compression: narrow u16->u8 then re-encode with RLE, // matching the path taken by the BtrBlocks compressor. + #[expect(deprecated)] let indices_prim = rle.indices().to_primitive().narrow()?; let re_encoded = RLEData::encode(indices_prim.as_view())?; @@ -475,6 +482,7 @@ mod tests { }; // Decompress — panicked before the fill_forward_nulls chunk-boundary fix. + #[expect(deprecated)] let decoded = reconstructed.as_array().to_primitive(); assert_arrays_eq!(decoded, original); Ok(()) diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index e19ed3efc8b..a73e8ac10de 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -6,6 +6,7 @@ use std::mem; use fastlanes::RLE as FastLanesRLE; use vortex_array::ArrayView; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -138,6 +139,7 @@ fn padded_validity(array: &PrimitiveArray) -> Validity { let mut builder = BitBufferMut::with_capacity(padded_len); + #[expect(deprecated)] let bool_array = validity_array.to_bool(); builder.append_buffer(&bool_array.to_bit_buffer()); builder.append_n(false, padded_len - len); @@ -151,6 +153,7 @@ fn padded_validity(array: &PrimitiveArray) -> Validity { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; @@ -171,6 +174,7 @@ mod tests { let encoded_u8 = RLEData::encode(PrimitiveArray::new(array_u8, Validity::NonNullable).as_view()) .unwrap(); + #[expect(deprecated)] let decoded_u8 = encoded_u8.as_array().to_primitive(); let expected_u8 = PrimitiveArray::from_iter(vec![1u8, 1, 2, 2, 3, 3]); assert_arrays_eq!(decoded_u8, expected_u8); @@ -180,6 +184,7 @@ mod tests { let encoded_u16 = RLEData::encode(PrimitiveArray::new(array_u16, Validity::NonNullable).as_view()) .unwrap(); + #[expect(deprecated)] let decoded_u16 = encoded_u16.as_array().to_primitive(); let expected_u16 = PrimitiveArray::from_iter(vec![100u16, 100, 200, 200]); assert_arrays_eq!(decoded_u16, expected_u16); @@ -189,6 +194,7 @@ mod tests { let encoded_u64 = RLEData::encode(PrimitiveArray::new(array_u64, Validity::NonNullable).as_view()) .unwrap(); + #[expect(deprecated)] let decoded_u64 = encoded_u64.as_array().to_primitive(); let expected_u64 = PrimitiveArray::from_iter(vec![1000u64, 1000, 2000]); assert_arrays_eq!(decoded_u64, expected_u64); @@ -220,6 +226,7 @@ mod tests { RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.values().len(), 2); // 2 chunks, each storing value 42 + #[expect(deprecated)] let decoded = encoded.as_array().to_primitive(); // Verify round-trip let expected = PrimitiveArray::from_iter(vec![42u16; 2000]); assert_arrays_eq!(decoded, expected); @@ -233,6 +240,7 @@ mod tests { RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); assert_eq!(encoded.values().len(), 256); + #[expect(deprecated)] let decoded = encoded.as_array().to_primitive(); // Verify round-trip let expected = PrimitiveArray::from_iter((0u8..=255).collect::>()); assert_arrays_eq!(decoded, expected); @@ -278,8 +286,10 @@ mod tests { #[case::f32((-2000..2000).map(|i| i as f32).collect::>())] #[case::f64((-2000..2000).map(|i| i as f64).collect::>())] fn test_roundtrip_primitive_types(#[case] values: Buffer) { + #[expect(deprecated)] let primitive = values.clone().into_array().to_primitive(); let result = RLEData::encode(primitive.as_view()).unwrap(); + #[expect(deprecated)] let decoded = result.as_array().to_primitive(); let expected = PrimitiveArray::new( values, @@ -297,6 +307,7 @@ mod tests { /// fill-forward default at index 0 and the actual value at index 1), which /// holds whenever the first position of each chunk is null. fn with_masked_constant_indices(rle: &RLEArray) -> VortexResult { + #[expect(deprecated)] let indices_prim = rle.indices().to_primitive(); let masked_indices = MaskedArray::try_new( ConstantArray::new(1u16, indices_prim.len()).into_array(), @@ -374,6 +385,7 @@ mod tests { /// positions, which can happen when indices are further compressed and the /// compressor clobbers invalid entries with arbitrary data. fn with_random_invalid_indices(rle: &RLEArray) -> VortexResult { + #[expect(deprecated)] let indices_prim = rle.indices().to_primitive(); let mut indices_data: Vec = indices_prim.as_slice::().to_vec(); @@ -479,6 +491,7 @@ mod tests { fn test_float_zeros(#[case] values: Vec) { let primitive = PrimitiveArray::from_iter(values); let rle = RLEData::encode(primitive.as_view()).unwrap(); + #[expect(deprecated)] let decoded = rle.as_array().to_primitive(); assert_arrays_eq!(primitive, decoded); } diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index 2123d74f136..fc63f293c3a 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -86,11 +86,12 @@ mod tests { Validity::from_iter([true, false, true, true, false]), ); let encoded = rle(&primitive); - encoded + #[expect(deprecated)] + let result = encoded .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())) - .unwrap(); + .and_then(|a| a.to_canonical().map(|c| c.into_array())); + result.unwrap(); } #[rstest] diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 2776418c231..1273e019dec 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -43,6 +43,7 @@ impl OperationsVTable for RLE { mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -174,6 +175,7 @@ mod tests { let expected: Vec = (0..3000).map(|i| (i / 50) as u16).collect(); let array = values.into_array(); + #[expect(deprecated)] let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); // Access scalars from multiple chunks. @@ -264,6 +266,7 @@ mod tests { #[test] fn test_slice_decode_with_nulls() { let array = fixture::rle_array_with_nulls(); + #[expect(deprecated)] let sliced = array.slice(1..4).unwrap().to_primitive(); // [null, 20, 20] let expected = PrimitiveArray::from_option_iter([Option::::None, Some(20), Some(20)]); @@ -284,6 +287,7 @@ mod tests { let expected: Vec = (0..2100).map(|i| (i / 100) as u32).collect(); let array = values.into_array(); + #[expect(deprecated)] let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); // Slice across first and second chunk. diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 35730c9a1f0..9853cd3956e 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -95,6 +95,7 @@ mod tests { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -185,6 +186,7 @@ mod tests { }; { + #[expect(deprecated)] let arr2 = chunked_arr.as_array().to_varbinview(); let res2 = arr2.with_iterator(|iter| iter.map(|b| b.map(|v| v.to_vec())).collect::>()); diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index 2ae2cf0ff40..f641e52d8bf 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -123,6 +123,7 @@ fn compare_fsst_constant( #[cfg(test)] mod tests { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; @@ -158,6 +159,7 @@ mod tests { let rhs = ConstantArray::new("world", lhs.len()); // Ensure fastpath for Eq exists, and returns correct answer + #[expect(deprecated)] let equals = lhs .clone() .into_array() @@ -173,6 +175,7 @@ mod tests { ); // Ensure fastpath for Eq exists, and returns correct answer + #[expect(deprecated)] let not_equals = lhs .clone() .into_array() diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index e5e8bba9cde..ef050435aba 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::varbin::VarBinArrayExt; @@ -58,6 +59,7 @@ impl LikeKernel for FSST { let negated = options.negated; let codes = array.codes(); + #[expect(deprecated)] let offsets = codes.offsets().to_primitive(); let all_bytes = codes.bytes(); let all_bytes = all_bytes.as_slice(); diff --git a/encodings/fsst/src/tests.rs b/encodings/fsst/src/tests.rs index fb45ed31629..441e217a883 100644 --- a/encodings/fsst/src/tests.rs +++ b/encodings/fsst/src/tests.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::varbin::builder::VarBinBuilder; use vortex_array::assert_arrays_eq; @@ -95,6 +96,7 @@ fn test_fsst_array_ops() { ); // test to_canonical + #[expect(deprecated)] let canonical_array = fsst_array.to_varbinview().into_array(); assert_arrays_eq!(fsst_array, canonical_array); diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 39a67e07ede..8ea7c72c44e 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -30,6 +30,7 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; @@ -262,7 +263,9 @@ fn collect_valid(parray: ArrayView<'_, Primitive>) -> VortexResult VortexError { diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index b21bf121afc..1f318207aec 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -7,6 +7,7 @@ use std::sync::LazyLock; use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -140,6 +141,7 @@ fn test_validity_and_multiple_chunks_and_pages() { let slice = compressed.slice(100..103).unwrap(); assert_nth_scalar!(slice, 0, 100); assert_nth_scalar!(slice, 2, 102); + #[expect(deprecated)] let primitive = slice.to_primitive(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index 65739507fd9..be0c30128a2 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -6,6 +6,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -53,7 +54,11 @@ pub fn runend_encode(array: ArrayView) -> (PrimitiveArray, ArrayRef) ConstantArray::new(Scalar::null(array.dtype().clone()), 1).into_array(), ); } - Validity::Array(a) => Some(a.to_bool().to_bit_buffer()), + Validity::Array(a) => { + #[expect(deprecated)] + let bool_array = a.to_bool(); + Some(bool_array.to_bit_buffer()) + } }; let (ends, values) = match validity { @@ -310,6 +315,7 @@ pub fn runend_decode_varbinview( #[cfg(test)] mod test { + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -325,6 +331,7 @@ mod test { fn encode() { let arr = PrimitiveArray::from_iter([1i32, 1, 2, 2, 2, 3, 3, 3, 3, 3]); let (ends, values) = runend_encode(arr.as_view()); + #[expect(deprecated)] let values = values.to_primitive(); let expected_ends = PrimitiveArray::from_iter(vec![2u8, 5, 10]); @@ -342,6 +349,7 @@ mod test { ])), ); let (ends, values) = runend_encode(arr.as_view()); + #[expect(deprecated)] let values = values.to_primitive(); let expected_ends = PrimitiveArray::from_iter(vec![2u8, 4, 5, 8, 10]); @@ -358,6 +366,7 @@ mod test { Validity::from(BitBuffer::new_unset(5)), ); let (ends, values) = runend_encode(arr.as_view()); + #[expect(deprecated)] let values = values.to_primitive(); let expected_ends = PrimitiveArray::from_iter(vec![5u64]); diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index dfda6b4cce5..4e134195d0e 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -36,6 +36,7 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -69,6 +70,7 @@ mod tests { ); // Verify by decoding to canonical form + #[expect(deprecated)] let decoded = casted.to_primitive(); // RunEnd encoding should expand to [100, 100, 100, 200, 200, 100, 100, 100, 300, 300] assert_eq!(decoded.len(), 10); diff --git a/encodings/runend/src/compute/take.rs b/encodings/runend/src/compute/take.rs index 9bbdbf6784e..cf346ff81b3 100644 --- a/encodings/runend/src/compute/take.rs +++ b/encodings/runend/src/compute/take.rs @@ -7,6 +7,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::TakeExecute; @@ -60,6 +61,7 @@ pub fn take_indices_unchecked>( indices: &[T], validity: &Validity, ) -> VortexResult { + #[expect(deprecated)] let ends = array.ends().to_primitive(); let ends_len = ends.len(); diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs index 4e913f5feae..52fbe7ddcc0 100644 --- a/encodings/runend/src/compute/take_from.rs +++ b/encodings/runend/src/compute/take_from.rs @@ -91,7 +91,9 @@ mod tests { .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([2i32, 2, 2, 3, 3, 2, 2]); - assert_arrays_eq!(result.to_canonical()?.into_array(), expected); + #[expect(deprecated)] + let canonical = result.to_canonical()?.into_array(); + assert_arrays_eq!(canonical, expected); Ok(()) } @@ -114,7 +116,9 @@ mod tests { .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([2i32, 3, 3]); - assert_arrays_eq!(result.to_canonical()?.into_array(), expected); + #[expect(deprecated)] + let canonical = result.to_canonical()?.into_array(); + assert_arrays_eq!(canonical, expected); Ok(()) } @@ -137,7 +141,9 @@ mod tests { .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([3i32, 3, 2, 2]); - assert_arrays_eq!(result.to_canonical()?.into_array(), expected); + #[expect(deprecated)] + let canonical = result.to_canonical()?.into_array(); + assert_arrays_eq!(canonical, expected); Ok(()) } @@ -160,7 +166,9 @@ mod tests { .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([3i32]); - assert_arrays_eq!(result.to_canonical()?.into_array(), expected); + #[expect(deprecated)] + let canonical = result.to_canonical()?.into_array(); + assert_arrays_eq!(canonical, expected); Ok(()) } diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index 69876b0a273..e10c314dc30 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -247,6 +247,7 @@ fn decode_nullable_sequential( #[cfg(test)] mod tests { use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -373,6 +374,7 @@ mod tests { BitBuffer::from(vec![true, false, true, false, true]), Validity::from(BitBuffer::from(vec![true, false, true, false, true])), ); + #[expect(deprecated)] let decoded = runend_decode_bools(ends, values, 0, 10000)?.to_bool(); // Check length and a few values diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index af92cd1ce53..2baaba5df5b 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -153,6 +153,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu mod tests { #[expect(unused_imports)] use itertools::Itertools; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -164,6 +165,7 @@ mod tests { let primitive_array = PrimitiveArray::from_iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); + #[expect(deprecated)] let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); } @@ -173,6 +175,7 @@ mod tests { let primitive_array = PrimitiveArray::from_iter([0]); let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); + #[expect(deprecated)] let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); } @@ -198,6 +201,7 @@ mod tests { let primitive_array = PrimitiveArray::from_iter(0u8..=255); let encoded = sequence_encode(primitive_array.as_view()).unwrap(); assert!(encoded.is_some()); + #[expect(deprecated)] let decoded = encoded.unwrap().to_primitive(); assert_arrays_eq!(decoded, primitive_array); } diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index 1eb9ddf47d8..b53e7188c52 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -89,6 +89,7 @@ impl CastReduce for Sequence { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -130,6 +131,7 @@ mod tests { ); // Verify the values + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([100i64, 110, 120, 130])); } @@ -149,6 +151,7 @@ mod tests { ); // Verify the values + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!( decoded, @@ -172,6 +175,7 @@ mod tests { ); // Verify the values were correctly converted + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!( decoded, diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index a182aa4c781..024975a4e76 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -566,6 +566,7 @@ mod test { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -607,6 +608,7 @@ mod test { let indices = buffer![0u64, 1, 7].into_array(); let values = BoolArray::from_iter([Some(true), None, Some(false)]).into_array(); let sparse_bools = Sparse::try_new(indices, values, 10, Scalar::from(fill_value)).unwrap(); + #[expect(deprecated)] let actual = sparse_bools.as_array().to_bool(); let expected = BoolArray::from_iter([ @@ -635,6 +637,7 @@ mod test { let sparse_ints = Sparse::try_new(indices, values, 10, Scalar::from(fill_value)).unwrap(); assert_eq!(*sparse_ints.dtype(), DType::Primitive(PType::I32, Nullable)); + #[expect(deprecated)] let flat_ints = sparse_ints.as_array().to_primitive(); let expected = PrimitiveArray::from_option_iter([ Some(0i32), @@ -718,6 +721,7 @@ mod test { .unwrap() .into_array(); + #[expect(deprecated)] let actual = sparse_struct.as_array().to_struct(); assert_arrays_eq!(actual, expected); } @@ -785,6 +789,7 @@ mod test { .unwrap() .into_array(); + #[expect(deprecated)] let actual = sparse_struct.as_array().to_struct(); assert_arrays_eq!(actual, expected); } @@ -813,6 +818,7 @@ mod test { .into_arrow_preferred() .unwrap(); + #[expect(deprecated)] let actual = sparse_struct .as_array() .to_decimal() @@ -845,6 +851,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = >::from_iter([ Some("hello"), @@ -886,6 +893,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = >::from_iter([ Some("hello"), @@ -920,6 +928,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = VarBinViewArray::from_iter_str([ "hello", "123", "123", "goodbye", "hello", "bonjour", "123", "123", "你好", @@ -950,6 +959,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = >::from_iter([ Some("hello"), @@ -991,6 +1001,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = VarBinViewArray::from_iter_nullable_bin([ Some(b"hello" as &[u8]), @@ -1037,6 +1048,7 @@ mod test { let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_array(); + #[expect(deprecated)] let result_listview = actual.to_listview(); // Check the structure @@ -1051,6 +1063,7 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] // Verify actual values + #[expect(deprecated)] let elements_array = result_listview.elements().to_primitive(); let elements_slice = elements_array.as_slice::(); @@ -1094,6 +1107,7 @@ mod test { .execute::(&mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("no fail") .into_array(); + #[expect(deprecated)] let result_listview = actual.to_listview(); // Check the structure @@ -1108,6 +1122,7 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] - extra element beyond original slice // Verify actual values + #[expect(deprecated)] let elements_array = result_listview.elements().to_primitive(); let elements_slice = elements_array.as_slice::(); @@ -1139,6 +1154,7 @@ mod test { let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_array(); + #[expect(deprecated)] let result_listview = actual.to_listview(); // Check the structure @@ -1153,6 +1169,7 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] from sparse // Verify actual values + #[expect(deprecated)] let elements_array = result_listview.elements().to_primitive(); let elements_slice = elements_array.as_slice::(); @@ -1211,6 +1228,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let actual = array.as_array().to_varbinview().into_array(); let expected = VarBinViewArray::from_iter_nullable_bin([ Some(b"hello" as &[u8]), @@ -1479,8 +1497,10 @@ mod test { .unwrap() .into_array(); + #[expect(deprecated)] + let actual_listview = actual.to_listview(); assert_eq!( - actual.to_listview().offsets().dtype(), + actual_listview.offsets().dtype(), &DType::Primitive(PType::U16, NonNullable) ); assert_arrays_eq!(&actual, &expected); @@ -1539,6 +1559,7 @@ mod test { .into_array() .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_array(); + #[expect(deprecated)] let result_listview = canonical.to_listview(); // Verify the structure @@ -1551,6 +1572,7 @@ mod test { if size == 0 { vec![] // null/empty list } else { + #[expect(deprecated)] let elements = result_listview.elements().to_primitive(); let slice = elements.as_slice::(); slice[offset..offset + size].to_vec() @@ -1621,6 +1643,7 @@ mod test { .into_array() .execute::(&mut LEGACY_SESSION.create_execution_ctx())? .into_array(); + #[expect(deprecated)] let result_listview = canonical.to_listview(); assert_eq!(result_listview.len(), 5); @@ -1632,6 +1655,7 @@ mod test { if size == 0 { vec![] // null/empty list } else { + #[expect(deprecated)] let elements = result_listview.elements().to_primitive(); let slice = elements.as_slice::(); slice[offset..offset + size].to_vec() diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index a9fe0046646..4eccd273090 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -36,6 +36,7 @@ impl CastReduce for Sparse { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -70,7 +71,9 @@ mod tests { ); let expected = PrimitiveArray::from_iter([0i64, 0, 100, 0, 0, 200, 0, 0, 300, 0]); - assert_arrays_eq!(casted.to_primitive(), expected); + #[expect(deprecated)] + let casted_primitive = casted.to_primitive(); + assert_arrays_eq!(casted_primitive, expected); } #[test] @@ -146,7 +149,9 @@ mod tests { ); let expected = PrimitiveArray::from_iter([10u64, 20, 30, 40, 50]); - assert_arrays_eq!(casted.to_primitive(), expected); + #[expect(deprecated)] + let casted_primitive = casted.to_primitive(); + assert_arrays_eq!(casted_primitive, expected); Ok(()) } diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index ed5d0bf8ecc..61436f80dea 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -22,6 +22,7 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; @@ -453,8 +454,9 @@ impl SparseData { fill.cast(array.dtype())? } else { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array - let (top_pvalue, _) = array - .to_primitive() + #[expect(deprecated)] + let primitive = array.to_primitive(); + let (top_pvalue, _) = primitive .top_value()? .vortex_expect("Non empty or all null array"); @@ -462,13 +464,12 @@ impl SparseData { }; let fill_array = ConstantArray::new(fill.clone(), array.len()).into_array(); - let non_top_mask = Mask::from_buffer( - array - .binary(fill_array.clone(), Operator::NotEq)? - .fill_null(Scalar::bool(true, Nullability::NonNullable))? - .to_bool() - .to_bit_buffer(), - ); + #[expect(deprecated)] + let non_top_bool = array + .binary(fill_array.clone(), Operator::NotEq)? + .fill_null(Scalar::bool(true, Nullability::NonNullable))? + .to_bool(); + let non_top_mask = Mask::from_buffer(non_top_bool.to_bit_buffer()); let non_top_values = array .filter(non_top_mask.clone())? @@ -763,7 +764,9 @@ mod test { true, true, false, true, false, true, false, true, true, false, true, false, ]) ); - assert_arrays_eq!(sparse.to_primitive(), original); + #[expect(deprecated)] + let sparse_primitive = sparse.to_primitive(); + assert_arrays_eq!(sparse_primitive, original); } #[test] diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index 552dcf5fa6e..28bfd6d85bd 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -25,6 +25,7 @@ impl OperationsVTable for Sparse { #[cfg(test)] mod tests { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -42,6 +43,7 @@ mod tests { let mut expected = vec![999u64; 1000]; expected[0] = 0; + #[expect(deprecated)] let values = sliced.to_primitive(); assert_arrays_eq!(values, PrimitiveArray::from_iter(expected)); } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 23e6f66f3eb..d91bf1f2974 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -271,6 +271,7 @@ impl ValidityChild for ZigZag { mod test { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::scalar::Scalar; @@ -281,6 +282,7 @@ mod test { #[test] fn test_compute_statistics() -> VortexResult<()> { + #[expect(deprecated)] let array = buffer![1i32, -5i32, 2, 3, 4, 5, 6, 7, 8, 9, 10] .into_array() .to_primitive(); diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 3c96c8f5778..8668ae6495f 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -78,6 +78,7 @@ where #[cfg(test)] mod test { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::assert_arrays_eq; @@ -90,10 +91,9 @@ mod test { .unwrap() .into_array(); assert!(compressed.is::()); - assert_arrays_eq!( - compressed.to_primitive(), - PrimitiveArray::from_iter(-100_i8..100) - ); + #[expect(deprecated)] + let decompressed = compressed.to_primitive(); + assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i8..100)); } #[test] fn test_compress_i16() { @@ -101,10 +101,9 @@ mod test { .unwrap() .into_array(); assert!(compressed.is::()); - assert_arrays_eq!( - compressed.to_primitive(), - PrimitiveArray::from_iter(-100_i16..100) - ); + #[expect(deprecated)] + let decompressed = compressed.to_primitive(); + assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i16..100)); } #[test] fn test_compress_i32() { @@ -112,10 +111,9 @@ mod test { .unwrap() .into_array(); assert!(compressed.is::()); - assert_arrays_eq!( - compressed.to_primitive(), - PrimitiveArray::from_iter(-100_i32..100) - ); + #[expect(deprecated)] + let decompressed = compressed.to_primitive(); + assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i32..100)); } #[test] fn test_compress_i64() { @@ -123,9 +121,8 @@ mod test { .unwrap() .into_array(); assert!(compressed.is::()); - assert_arrays_eq!( - compressed.to_primitive(), - PrimitiveArray::from_iter(-100_i64..100) - ); + #[expect(deprecated)] + let decompressed = compressed.to_primitive(); + assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i64..100)); } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 6aa5af1f532..4552506077c 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -74,6 +74,7 @@ mod tests { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -188,7 +189,9 @@ mod tests { fn test_take_zigzag_conformance(#[case] array: ArrayRef) -> VortexResult<()> { use vortex_array::compute::conformance::take::test_take_conformance; - let zigzag = zigzag_encode(array.to_primitive().as_view())?; + #[expect(deprecated)] + let array_primitive = array.to_primitive(); + let zigzag = zigzag_encode(array_primitive.as_view())?; test_take_conformance(&zigzag.into_array()); Ok(()) } diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 168f437c8cf..5e3d592acca 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -23,6 +23,7 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -370,7 +371,9 @@ fn collect_valid_primitive(parray: &PrimitiveArray) -> VortexResult VortexResult<(ByteBuffer, Vec)> { @@ -799,7 +802,9 @@ impl ZstdData { } pub fn from_array(array: ArrayRef, level: i32, values_per_frame: usize) -> VortexResult { - Self::from_canonical(&array.to_canonical()?, level, values_per_frame)? + #[expect(deprecated)] + let canonical = array.to_canonical()?; + Self::from_canonical(&canonical, level, values_per_frame)? .ok_or_else(|| vortex_err!("Zstd can only encode Primitive and VarBinView arrays")) } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index 2beb94a1a4e..508fcd6231d 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -88,6 +88,7 @@ impl CastReduce for Zstd { mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -115,6 +116,7 @@ mod tests { &DType::Primitive(PType::I64, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i64, 2, 3, 4, 5])); } @@ -150,6 +152,7 @@ mod tests { &DType::Primitive(PType::U32, Nullability::NonNullable) ); // Verify the values are correct + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([20u32, 30, 40, 50])); } @@ -173,6 +176,7 @@ mod tests { casted.dtype(), &DType::Primitive(PType::U32, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_primitive(); let expected = PrimitiveArray::from_iter([20u32, 30, 40, 50]); assert_arrays_eq!(decoded, expected); diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 84122db22c7..da467d3f2ed 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -4,6 +4,7 @@ use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -78,6 +79,7 @@ fn test_zstd_with_validity_and_multi_frame() { assert_nth_scalar!(compressed, 177, 177); let mut ctx = LEGACY_SESSION.create_execution_ctx(); + #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() .to_primitive(); @@ -94,6 +96,7 @@ fn test_zstd_with_validity_and_multi_frame() { // check slicing works let slice = compressed.slice(176..179).unwrap(); + #[expect(deprecated)] let primitive = slice.to_primitive(); assert_eq!( i32::try_from( @@ -130,6 +133,7 @@ fn test_zstd_with_dict() { assert_nth_scalar!(compressed, 199, 199); let mut ctx = LEGACY_SESSION.create_execution_ctx(); + #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() .to_primitive(); @@ -137,6 +141,7 @@ fn test_zstd_with_dict() { // check slicing works let slice = compressed.slice(176..179).unwrap(); + #[expect(deprecated)] let primitive = slice.to_primitive(); assert_arrays_eq!(primitive, PrimitiveArray::from_iter([176, 177, 178])); } @@ -213,6 +218,7 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 4, "baz"); let mut ctx = LEGACY_SESSION.create_execution_ctx(); + #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() .to_varbinview(); diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 8d503c8c590..d4d3865507d 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -8,6 +8,7 @@ use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; use vortex_array::Canonical; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; @@ -50,9 +51,9 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { .clone() .apply(&filter_expr.clone().unwrap_or_else(|| lit(true))) .vortex_expect("filter expression evaluation should succeed in fuzz test"); - let mask = bool_mask - .to_bool() - .to_mask_fill_null_false(&mut SESSION.create_execution_ctx()); + #[expect(deprecated)] + let bool_mask_bool = bool_mask.to_bool(); + let mask = bool_mask_bool.to_mask_fill_null_false(&mut SESSION.create_execution_ctx()); let filtered = array_data .filter(mask) .vortex_expect("filter operation should succeed in fuzz test"); @@ -110,6 +111,7 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { output_array.dtype() ); + #[expect(deprecated)] let bool_result = expected_array .binary(output_array.clone(), Operator::Eq) .vortex_expect("compare operation should succeed in fuzz test") diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index 503287c2dcf..fdb399bac43 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; @@ -34,21 +35,24 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult() - .iter() - .map(|v| *v as Out) - .collect::>(), - Validity::from_mask( - array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, - target.nullability(), - ), - ) - .into_array() + { + #[expect(deprecated)] + let prim = array.to_primitive(); + PrimitiveArray::new( + prim.as_slice::() + .iter() + .map(|v| *v as Out) + .collect::>(), + Validity::from_mask( + array.validity()?.to_mask( + array.len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?, + target.nullability(), + ), + ) + .into_array() + } }) } ))) @@ -64,31 +68,32 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult Ok(Some( - PrimitiveArray::new( - array - .to_primitive() - .as_slice::() - .iter() - .map(|v| *v as f64) - .collect::>(), - Validity::from_mask( - array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, - target.nullability(), - ), - ) - .into_array(), - )), - (PType::F64, PType::F32) => - { + (PType::F32, PType::F64) => { + #[expect(deprecated)] + let prim = array.to_primitive(); + Ok(Some( + PrimitiveArray::new( + prim.as_slice::() + .iter() + .map(|v| *v as f64) + .collect::>(), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), + ) + .into_array(), + )) + } + (PType::F64, PType::F32) => { + #[expect(deprecated)] + let prim = array.to_primitive(); #[expect(clippy::cast_possible_truncation)] Ok(Some( PrimitiveArray::new( - array - .to_primitive() - .as_slice::() + prim.as_slice::() .iter() .map(|v| *v as f32) .collect::>(), diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index 88784616eda..c45c25d4125 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -40,9 +41,10 @@ pub fn compare_canonical_array( .as_bool() .value() .vortex_expect("nulls handled before"); + #[expect(deprecated)] + let bool_array = array.to_bool(); compare_to( - array - .to_bool() + bool_array .to_bit_buffer() .iter() .zip( @@ -62,6 +64,7 @@ pub fn compare_canonical_array( } DType::Primitive(p, _) => { let primitive = value.as_primitive(); + #[expect(deprecated)] let primitive_array = array.to_primitive(); match_each_native_ptype!(p, |P| { let pval = primitive @@ -90,6 +93,7 @@ pub fn compare_canonical_array( } DType::Decimal(..) => { let decimal = value.as_decimal(); + #[expect(deprecated)] let decimal_array = array.to_decimal(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let dval = decimal @@ -118,26 +122,34 @@ pub fn compare_canonical_array( ) }) } - DType::Utf8(_) => array.to_varbinview().with_iterator(|iter| { - let utf8_value = value.as_utf8(); - compare_to( - iter.map(|v| v.map(|b| unsafe { str::from_utf8_unchecked(b) })), - utf8_value.value().vortex_expect("nulls handled before"), - operator, - result_nullability, - ) - }), - DType::Binary(_) => array.to_varbinview().with_iterator(|iter| { - let binary_value = value.as_binary(); - compare_to( - // Don't understand the lifetime problem here but identity map makes it go away - #[expect(clippy::map_identity)] - iter.map(|v| v), - binary_value.value().vortex_expect("nulls handled before"), - operator, - result_nullability, - ) - }), + DType::Utf8(_) => { + #[expect(deprecated)] + let varbinview = array.to_varbinview(); + varbinview.with_iterator(|iter| { + let utf8_value = value.as_utf8(); + compare_to( + iter.map(|v| v.map(|b| unsafe { str::from_utf8_unchecked(b) })), + utf8_value.value().vortex_expect("nulls handled before"), + operator, + result_nullability, + ) + }) + } + DType::Binary(_) => { + #[expect(deprecated)] + let varbinview = array.to_varbinview(); + varbinview.with_iterator(|iter| { + let binary_value = value.as_binary(); + compare_to( + // Don't understand the lifetime problem here but identity map makes it go away + #[expect(clippy::map_identity)] + iter.map(|v| v), + binary_value.value().vortex_expect("nulls handled before"), + operator, + result_nullability, + ) + }) + } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let scalar_vals: Vec = (0..array.len()) diff --git a/fuzz/src/array/fill_null.rs b/fuzz/src/array/fill_null.rs index 4b4218cd73b..b9705147326 100644 --- a/fuzz/src/array/fill_null.rs +++ b/fuzz/src/array/fill_null.rs @@ -7,6 +7,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -71,7 +72,9 @@ fn fill_bool_array( } Validity::AllInvalid => ConstantArray::new(fill_value.clone(), array.len()).into_array(), Validity::Array(validity_array) => { - let validity_bits = validity_array.to_bool().into_bit_buffer(); + #[expect(deprecated)] + let validity_bool = validity_array.to_bool(); + let validity_bits = validity_bool.into_bit_buffer(); let data_bits = array.into_bit_buffer(); let new_bits = match data_bits.try_into_mut() { @@ -116,6 +119,7 @@ fn fill_primitive_array( ConstantArray::new(fill_value.clone(), array.len()).into_array() } Validity::Array(validity_array) => { + #[expect(deprecated)] let validity_bool_array = validity_array.to_bool(); let validity_bits = validity_bool_array.to_bit_buffer(); let data_slice = array.as_slice::(); @@ -162,6 +166,7 @@ fn fill_decimal_array( ConstantArray::new(fill_value.clone(), array.len()).into_array() } Validity::Array(validity_array) => { + #[expect(deprecated)] let validity_bool_array = validity_array.to_bool(); let validity_bits = validity_bool_array.to_bit_buffer(); let data_buffer = array.buffer::(); @@ -196,6 +201,7 @@ fn fill_varbinview_array( Validity::NonNullable | Validity::AllValid => array.into_array(), Validity::AllInvalid => ConstantArray::new(fill_value.clone(), array.len()).into_array(), Validity::Array(validity_array) => { + #[expect(deprecated)] let validity_bool_array = validity_array.to_bool(); let validity_bits = validity_bool_array.to_bit_buffer(); @@ -223,9 +229,11 @@ fn fill_varbinview_array( let string_refs: Vec<&str> = strings.iter().map(|s| s.as_str()).collect(); let result = VarBinViewArray::from_iter_str(string_refs).into_array(); if result_nullability == Nullability::Nullable { + #[expect(deprecated)] + let result_vbv = result.to_varbinview(); VarBinViewArray::new_handle( - result.to_varbinview().views_handle().clone(), - Arc::clone(result.to_varbinview().data_buffers()), + result_vbv.views_handle().clone(), + Arc::clone(result_vbv.data_buffers()), result.dtype().as_nullable(), result_nullability.into(), ) @@ -257,9 +265,11 @@ fn fill_varbinview_array( let binary_refs: Vec<&[u8]> = binaries.iter().map(|b| b.as_slice()).collect(); let result = VarBinViewArray::from_iter_bin(binary_refs).into_array(); if result_nullability == Nullability::Nullable { + #[expect(deprecated)] + let result_vbv = result.to_varbinview(); VarBinViewArray::new_handle( - result.to_varbinview().views_handle().clone(), - Arc::clone(result.to_varbinview().data_buffers()), + result_vbv.views_handle().clone(), + Arc::clone(result_vbv.data_buffers()), result.dtype().as_nullable(), result_nullability.into(), ) diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 9e5b90c7c05..c96d1044efe 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -43,6 +44,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult match array.dtype() { DType::Bool(_) => { + #[expect(deprecated)] let bool_array = array.to_bool(); Ok(BoolArray::new( BitBuffer::from_iter( @@ -57,6 +59,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult .into_array()) } DType::Primitive(p, _) => match_each_native_ptype!(p, |P| { + #[expect(deprecated)] let primitive_array = array.to_primitive(); Ok(PrimitiveArray::new( filter @@ -70,6 +73,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult .into_array()) }), DType::Decimal(d, _) => { + #[expect(deprecated)] let decimal_array = array.to_decimal(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); @@ -87,6 +91,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult }) } DType::Utf8(_) | DType::Binary(_) => { + #[expect(deprecated)] let utf8 = array.to_varbinview(); let values = utf8.with_iterator(|iter| { iter.zip(filter.iter()) @@ -97,6 +102,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult Ok(VarBinViewArray::from_iter(values, array.dtype().clone()).into_array()) } DType::Struct(..) => { + #[expect(deprecated)] let struct_array = array.to_struct(); let filtered_children = struct_array .iter_unmasked_fields() diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 8f1aba4ed16..75cf60f6d9c 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -7,6 +7,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -44,6 +45,7 @@ pub fn mask_validity(validity: &Validity, mask: &Mask) -> Validity { Validity::from_bit_buffer(make_valid.clone(), Nullability::Nullable) } Validity::Array(is_valid) => { + #[expect(deprecated)] let is_valid = is_valid.to_bool(); Validity::from_bit_buffer( is_valid.to_bit_buffer() & make_valid, diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index 2c72dbcc7e5..6dc126d9254 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -327,24 +327,22 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { } // Sum - returns a scalar, does NOT update current_array (terminal operation) - let sum_result = sum_canonical_array( - current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - &mut ctx, - ) - .vortex_expect("sum_canonical_array should succeed in fuzz test"); + #[expect(deprecated)] + let current_array_canonical = current_array + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + let sum_result = sum_canonical_array(current_array_canonical, &mut ctx) + .vortex_expect("sum_canonical_array should succeed in fuzz test"); (Action::Sum, ExpectedValue::Scalar(sum_result)) } ActionType::MinMax => { // MinMax - returns a scalar, does NOT update current_array (terminal operation) - let min_max_result = min_max_canonical_array( - current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - &mut ctx, - ) - .vortex_expect("min_max_canonical_array should succeed in fuzz test"); + #[expect(deprecated)] + let current_array_canonical = current_array + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + let min_max_result = min_max_canonical_array(current_array_canonical, &mut ctx) + .vortex_expect("min_max_canonical_array should succeed in fuzz test"); (Action::MinMax, ExpectedValue::MinMax(min_max_result)) } ActionType::FillNull => { @@ -370,13 +368,13 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { } // Compute expected result on canonical form - let expected_result = fill_null_canonical_array( - current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - &fill_value, - ) - .vortex_expect("fill_null_canonical_array should succeed in fuzz test"); + #[expect(deprecated)] + let current_array_canonical = current_array + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + let expected_result = + fill_null_canonical_array(current_array_canonical, &fill_value) + .vortex_expect("fill_null_canonical_array should succeed in fuzz test"); // Update current_array to the result for chaining current_array = expected_result.clone(); ( @@ -391,10 +389,12 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { .collect::>>()?; // Compute expected result on canonical form + #[expect(deprecated)] + let current_array_canonical = current_array + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); let expected_result = mask_canonical_array( - current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), + current_array_canonical, &Mask::from_iter(mask.clone()), ) .vortex_expect("mask_canonical_array should succeed in fuzz test"); @@ -424,13 +424,13 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { let expected_scalars: Vec = indices_vec .iter() .map(|&idx| { - scalar_at_canonical_array( - current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - idx, + #[expect(deprecated)] + let canonical = current_array + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + scalar_at_canonical_array(canonical, idx).vortex_expect( + "scalar_at_canonical_array should succeed in fuzz test", ) - .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); @@ -581,6 +581,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { debug!(id = i, action = ?action); match action { Action::Compress(strategy) => { + #[expect(deprecated)] let canonical = current_array .to_canonical() .vortex_expect("to_canonical should succeed in fuzz test"); diff --git a/fuzz/src/array/scalar_at.rs b/fuzz/src/array/scalar_at.rs index 489860c7f59..fcc26c39b12 100644 --- a/fuzz/src/array/scalar_at.rs +++ b/fuzz/src/array/scalar_at.rs @@ -54,12 +54,12 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let list = array.list_elements_at(index)?; let children: Vec = (0..list.len()) .map(|i| { - scalar_at_canonical_array( - list.to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - i, - ) - .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") + #[expect(deprecated)] + let canonical = list + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + scalar_at_canonical_array(canonical, i) + .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); Scalar::list( @@ -72,12 +72,12 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let list = array.fixed_size_list_elements_at(index)?; let children: Vec = (0..list.len()) .map(|i| { - scalar_at_canonical_array( - list.to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - i, - ) - .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") + #[expect(deprecated)] + let canonical = list + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + scalar_at_canonical_array(canonical, i) + .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); Scalar::fixed_size_list(list.dtype().clone(), children, array.dtype().nullability()) @@ -86,20 +86,20 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let field_scalars: Vec = array .iter_unmasked_fields() .map(|field| { - scalar_at_canonical_array( - field - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"), - index, - ) - .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") + #[expect(deprecated)] + let canonical = field + .to_canonical() + .vortex_expect("to_canonical should succeed in fuzz test"); + scalar_at_canonical_array(canonical, index) + .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); Scalar::struct_(array.dtype().clone(), field_scalars) } Canonical::Extension(array) => { - let storage_scalar = - scalar_at_canonical_array(array.storage_array().to_canonical()?, index)?; + #[expect(deprecated)] + let storage_canonical = array.storage_array().to_canonical()?; + let storage_scalar = scalar_at_canonical_array(storage_canonical, index)?; Scalar::extension_ref(array.ext_dtype().clone(), storage_scalar) } Canonical::Variant(_) => unreachable!("Variant arrays are not fuzzed"), diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index d7b9901cc90..f0de94db90a 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -6,6 +6,7 @@ use std::fmt::Debug; use vortex_array::ArrayRef; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -65,6 +66,7 @@ pub fn search_sorted_canonical_array( ) -> VortexResult { match array.dtype() { DType::Bool(_) => { + #[expect(deprecated)] let bool_array = array.to_bool(); let validity = bool_array .as_ref() @@ -84,6 +86,7 @@ pub fn search_sorted_canonical_array( SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side) } DType::Primitive(p, _) => { + #[expect(deprecated)] let primitive_array = array.to_primitive(); let validity = primitive_array .as_ref() @@ -106,6 +109,7 @@ pub fn search_sorted_canonical_array( }) } DType::Decimal(d, _) => { + #[expect(deprecated)] let decimal_array = array.to_decimal(); let validity = decimal_array .as_ref() @@ -138,6 +142,7 @@ pub fn search_sorted_canonical_array( }) } DType::Utf8(_) | DType::Binary(_) => { + #[expect(deprecated)] let utf8 = array.to_varbinview(); let opt_values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index e1e4c388e55..c131f70e9c4 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -41,11 +42,13 @@ pub fn slice_canonical_array( match array.dtype() { DType::Bool(_) => { + #[expect(deprecated)] let bool_array = array.to_bool(); let sliced_bools = bool_array.to_bit_buffer().slice(start..stop); Ok(BoolArray::new(sliced_bools, validity).into_array()) } DType::Primitive(p, _) => { + #[expect(deprecated)] let primitive_array = array.to_primitive(); match_each_native_ptype!(p, |P| { Ok(PrimitiveArray::new( @@ -56,6 +59,7 @@ pub fn slice_canonical_array( }) } DType::Utf8(_) | DType::Binary(_) => { + #[expect(deprecated)] let utf8 = array.to_varbinview(); let values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); @@ -66,6 +70,7 @@ pub fn slice_canonical_array( .into_array()) } DType::Struct(..) => { + #[expect(deprecated)] let struct_array = array.to_struct(); let sliced_children = struct_array .iter_unmasked_fields() @@ -80,6 +85,7 @@ pub fn slice_canonical_array( .map(|a| a.into_array()) } DType::List(..) => { + #[expect(deprecated)] let list_array = array.to_listview(); let offsets = slice_canonical_array(list_array.offsets(), start, stop)?; @@ -98,6 +104,7 @@ pub fn slice_canonical_array( .into_array()) } DType::FixedSizeList(..) => { + #[expect(deprecated)] let fsl_array = array.to_fixed_size_list(); let list_size = fsl_array.list_size() as usize; let elements = @@ -108,6 +115,7 @@ pub fn slice_canonical_array( .map(|a| a.into_array()) } DType::Decimal(decimal_dtype, _) => { + #[expect(deprecated)] let decimal_array = array.to_decimal(); Ok( match_each_decimal_value_type!(decimal_array.values_type(), |D| { diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index 12fb2c9ab0c..dbe9cff9179 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -6,6 +6,7 @@ use std::cmp::Ordering; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -26,6 +27,7 @@ use crate::array::take_canonical_array_non_nullable_indices; pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { match array.dtype() { DType::Bool(_) => { + #[expect(deprecated)] let bool_array = array.to_bool(); let mut opt_values = bool_array .to_bit_buffer() @@ -47,6 +49,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { Ok(BoolArray::from_iter(opt_values).into_array()) } DType::Primitive(p, _) => { + #[expect(deprecated)] let primitive_array = array.to_primitive(); match_each_native_ptype!(p, |P| { let mut opt_values = primitive_array @@ -71,6 +74,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { }) } DType::Decimal(d, _) => { + #[expect(deprecated)] let decimal_array = array.to_decimal(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); @@ -96,6 +100,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { }) } DType::Utf8(_) | DType::Binary(_) => { + #[expect(deprecated)] let utf8 = array.to_varbinview(); let mut opt_values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index e737b33d0cc..d33dcdeaf2c 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -64,6 +65,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort match array.dtype() { DType::Bool(_) => { + #[expect(deprecated)] let bool_array = array.to_bool(); let vec_values = bool_array.to_bit_buffer().iter().collect::>(); Ok(BoolArray::new( @@ -76,6 +78,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort .into_array()) } DType::Primitive(p, _) => { + #[expect(deprecated)] let primitive_array = array.to_primitive(); match_each_native_ptype!(p, |P| { Ok(take_primitive::

( @@ -86,6 +89,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort }) } DType::Decimal(d, _) => { + #[expect(deprecated)] let decimal_array = array.to_decimal(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { @@ -98,6 +102,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort }) } DType::Utf8(_) | DType::Binary(_) => { + #[expect(deprecated)] let utf8 = array.to_varbinview(); let values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); @@ -110,6 +115,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort .into_array()) } DType::Struct(..) => { + #[expect(deprecated)] let struct_array = array.to_struct(); let taken_children = struct_array .iter_unmasked_fields() diff --git a/fuzz/src/compress.rs b/fuzz/src/compress.rs index ab1d2cc38f9..59e2b2d7c88 100644 --- a/fuzz/src/compress.rs +++ b/fuzz/src/compress.rs @@ -71,6 +71,7 @@ pub fn run_compress_roundtrip(fuzz: FuzzCompressRoundtrip) -> crate::error::Vort let original_dtype = array.dtype().clone(); // Try to canonicalize - this is the main thing we're testing + #[expect(deprecated)] let canonical = match array.to_canonical() { Ok(c) => c, Err(e) => { diff --git a/results.json b/results.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vortex-array/benches/take_patches.rs b/vortex-array/benches/take_patches.rs index 85d7f6ffcd0..f929cbd3b69 100644 --- a/vortex-array/benches/take_patches.rs +++ b/vortex-array/benches/take_patches.rs @@ -11,7 +11,8 @@ use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -use vortex_array::ToCanonical; +#[expect(deprecated)] +use vortex_array::ToCanonical as _; use vortex_array::VortexSessionExecute; use vortex_array::patches::Patches; use vortex_buffer::Buffer; @@ -49,7 +50,9 @@ fn take_search(bencher: Bencher, (patches_sparsity, index_multiple): (f64, f64)) bencher .with_inputs(|| (&patches, &indices, LEGACY_SESSION.create_execution_ctx())) .bench_refs(|(patches, indices, ctx)| { - patches.take_search(indices.to_primitive(), false, ctx) + #[expect(deprecated)] + let prim = indices.to_primitive(); + patches.take_search(prim, false, ctx) }); } @@ -66,7 +69,9 @@ fn take_search_chunked(bencher: Bencher, (patches_sparsity, index_multiple): (f6 bencher .with_inputs(|| (&patches, &indices, LEGACY_SESSION.create_execution_ctx())) .bench_refs(|(patches, indices, ctx)| { - patches.take_search(indices.to_primitive(), false, ctx) + #[expect(deprecated)] + let prim = indices.to_primitive(); + patches.take_search(prim, false, ctx) }); } @@ -82,7 +87,11 @@ fn take_map(bencher: Bencher, (patches_sparsity, index_multiple): (f64, f64)) { bencher .with_inputs(|| (&patches, &indices, LEGACY_SESSION.create_execution_ctx())) - .bench_refs(|(patches, indices, ctx)| patches.take_map(indices.to_primitive(), false, ctx)); + .bench_refs(|(patches, indices, ctx)| { + #[expect(deprecated)] + let prim = indices.to_primitive(); + patches.take_map(prim, false, ctx) + }); } fn fixture(len: usize, sparsity: f64, rng: &mut StdRng) -> Patches { diff --git a/vortex-array/benches/varbinview_compact.rs b/vortex-array/benches/varbinview_compact.rs index ae763055db1..99243ce3606 100644 --- a/vortex-array/benches/varbinview_compact.rs +++ b/vortex-array/benches/varbinview_compact.rs @@ -7,7 +7,8 @@ use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; -use vortex_array::ToCanonical; +#[expect(deprecated)] +use vortex_array::ToCanonical as _; use vortex_array::arrays::VarBinViewArray; use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; @@ -45,6 +46,7 @@ fn compact_impl(bencher: Bencher, (output_size, utilization_pct): (usize, usize) .into_array() .take(indices) .vortex_expect("operation should succeed in benchmark"); + #[expect(deprecated)] let array = taken.to_varbinview(); bencher.with_inputs(|| &array).bench_refs(|array| { @@ -61,6 +63,7 @@ fn compact_sliced_impl(bencher: Bencher, (output_size, utilization_pct): (usize, .into_array() .slice(0..output_size) .vortex_expect("slice should succeed"); + #[expect(deprecated)] let array = sliced.to_varbinview(); bencher.with_inputs(|| &array).bench_refs(|array| { diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index 3fe72921b7c..84367076314 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -301,13 +301,17 @@ impl ArrayRef { } /// Returns the canonical representation of the array. + #[deprecated(note = "use `array.execute::(ctx)` instead")] pub fn into_canonical(self) -> VortexResult { self.execute(&mut LEGACY_SESSION.create_execution_ctx()) } /// Returns the canonical representation of the array. + #[deprecated(note = "use `array.execute::(ctx)` instead")] pub fn to_canonical(&self) -> VortexResult { - self.clone().into_canonical() + #[expect(deprecated)] + let result = self.clone().into_canonical(); + result } /// Writes the array into the canonical builder. diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 7fb1248aab9..49de1f7f17c 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -395,7 +395,9 @@ impl Array { #[deprecated(note = "Use Array::::execute::() instead")] pub fn to_canonical(&self) -> VortexResult { - self.inner.to_canonical() + #[expect(deprecated)] + let result = self.inner.to_canonical(); + result } pub fn nbytes(&self) -> u64 { diff --git a/vortex-array/src/arrays/arbitrary.rs b/vortex-array/src/arrays/arbitrary.rs index 16099d5c2c1..fc5919d3e9b 100644 --- a/vortex-array/src/arrays/arbitrary.rs +++ b/vortex-array/src/arrays/arbitrary.rs @@ -13,7 +13,8 @@ use vortex_error::VortexExpect; use crate::ArrayRef; use crate::IntoArray; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::arrays::BoolArray; use crate::arrays::ChunkedArray; use crate::arrays::NullArray; @@ -105,10 +106,14 @@ fn random_array_chunk( PType::I16 => random_primitive::(u, *n, chunk_len), PType::I32 => random_primitive::(u, *n, chunk_len), PType::I64 => random_primitive::(u, *n, chunk_len), - PType::F16 => Ok(random_primitive::(u, *n, chunk_len)? - .to_primitive() - .reinterpret_cast(PType::F16) - .into_array()), + PType::F16 => { + #[expect(deprecated)] + let prim = random_primitive::(u, *n, chunk_len)? + .to_primitive() + .reinterpret_cast(PType::F16) + .into_array(); + Ok(prim) + } PType::F32 => random_primitive::(u, *n, chunk_len), PType::F64 => random_primitive::(u, *n, chunk_len), }, diff --git a/vortex-array/src/arrays/bool/compute/fill_null.rs b/vortex-array/src/arrays/bool/compute/fill_null.rs index f2489f9d073..59b9ca0b717 100644 --- a/vortex-array/src/arrays/bool/compute/fill_null.rs +++ b/vortex-array/src/arrays/bool/compute/fill_null.rs @@ -51,7 +51,8 @@ mod tests { use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -65,6 +66,7 @@ mod tests { BitBuffer::from_iter([true, true, false, false]), Validity::from_iter([true, false, true, false]), ); + #[expect(deprecated)] let non_null_array = bool_array .into_array() .fill_null(Scalar::from(fill_value)) diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index b3b3a269795..f7129d08366 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -84,7 +84,8 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray as _; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::BoolArray; use crate::arrays::PrimitiveArray; use crate::arrays::bool::BoolArrayExt; @@ -102,6 +103,7 @@ mod test { Some(false), ]); + #[expect(deprecated)] let b = reference .take(buffer![0, 3, 4].into_array()) .unwrap() diff --git a/vortex-array/src/arrays/bool/vtable/operations.rs b/vortex-array/src/arrays/bool/vtable/operations.rs index ece00b038ea..11147b810b2 100644 --- a/vortex-array/src/arrays/bool/vtable/operations.rs +++ b/vortex-array/src/arrays/bool/vtable/operations.rs @@ -28,7 +28,8 @@ mod tests { use std::iter; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::assert_arrays_eq; @@ -36,6 +37,7 @@ mod tests { #[test] fn test_slice_hundred_elements() { let arr = BoolArray::from_iter(iter::repeat_n(Some(true), 100)); + #[expect(deprecated)] let sliced_arr = arr.into_array().slice(8..16).unwrap().to_bool(); assert_eq!(sliced_arr.len(), 8); assert_eq!(sliced_arr.to_bit_buffer().len(), 8); @@ -45,6 +47,7 @@ mod tests { #[test] fn test_slice() { let arr = BoolArray::from_iter([Some(true), Some(true), None, Some(false), None]); + #[expect(deprecated)] let sliced_arr = arr.into_array().slice(1..4).unwrap().to_bool(); assert_arrays_eq!( diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 97be61cb524..78d0c4caa52 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -192,14 +192,14 @@ impl Array { || new_chunk_n_elements + n_elements > target_rowsize) && !chunks_to_combine.is_empty() { - new_chunks.push( - unsafe { - Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) - } - .into_array() - .to_canonical()? - .into_array(), - ); + #[expect(deprecated)] + let canonical = unsafe { + Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) + } + .into_array() + .to_canonical()? + .into_array(); + new_chunks.push(canonical); new_chunk_n_bytes = 0; new_chunk_n_elements = 0; @@ -216,12 +216,13 @@ impl Array { } if !chunks_to_combine.is_empty() { - new_chunks.push( + #[expect(deprecated)] + let canonical = unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? - .into_array(), - ); + .into_array(); + new_chunks.push(canonical); } unsafe { Ok(Self::new_unchecked(new_chunks, self.dtype().clone())) } diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 38b0f4313ea..afa400de612 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -126,7 +126,8 @@ mod test { use vortex_error::VortexResult; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::BoolArray; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; @@ -284,6 +285,7 @@ mod test { let result = arr.take(indices_arr.into_array())?; // Verify every element. + #[expect(deprecated)] let result = result.to_primitive(); let result_vals = result.as_slice::(); for (pos, &idx) in indices.iter().enumerate() { diff --git a/vortex-array/src/arrays/chunked/compute/zip.rs b/vortex-array/src/arrays/chunked/compute/zip.rs index 0979cb15031..1d7873888a3 100644 --- a/vortex-array/src/arrays/chunked/compute/zip.rs +++ b/vortex-array/src/arrays/chunked/compute/zip.rs @@ -52,7 +52,8 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -102,6 +103,7 @@ mod tests { assert_eq!(zipped.nchunks(), 4); let mut values: Vec = Vec::new(); for chunk in zipped.chunks() { + #[expect(deprecated)] let primitive = chunk.to_primitive(); values.extend_from_slice(primitive.as_slice::()); } diff --git a/vortex-array/src/arrays/chunked/paired_chunks.rs b/vortex-array/src/arrays/chunked/paired_chunks.rs index d81ee13c3b8..2145c88dbbd 100644 --- a/vortex-array/src/arrays/chunked/paired_chunks.rs +++ b/vortex-array/src/arrays/chunked/paired_chunks.rs @@ -121,6 +121,8 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::ChunkedArray; use crate::arrays::chunked::paired_chunks::PairedChunksExt; use crate::dtype::DType; @@ -136,11 +138,12 @@ mod tests { left: &ChunkedArray, right: &ChunkedArray, ) -> VortexResult, Vec, std::ops::Range)>> { - use crate::ToCanonical; let mut result = Vec::new(); for pair in left.paired_chunks(right) { let pair = pair?; + #[expect(deprecated)] let l: Vec = pair.left.to_primitive().as_slice::().to_vec(); + #[expect(deprecated)] let r: Vec = pair.right.to_primitive().as_slice::().to_vec(); result.push((l, r, pair.pos)); } diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index f3f9b45afc9..6cc2049ec64 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -19,7 +19,8 @@ use crate::arrays::VarBinViewArray; use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -190,8 +191,11 @@ pub fn pack_nested_structs() { ) .unwrap() .into_array(); + #[expect(deprecated)] let canonical_struct = chunked.to_struct(); + #[expect(deprecated)] let canonical_varbin = canonical_struct.unmasked_fields()[0].to_varbinview(); + #[expect(deprecated)] let original_varbin = struct_array.unmasked_fields()[0].to_varbinview(); let orig_values = original_varbin.with_iterator(|it| it.map(|a| a.map(|v| v.to_vec())).collect::>()); @@ -224,6 +228,7 @@ pub fn pack_nested_lists() { ), ); + #[expect(deprecated)] let canon_values = chunked_list.unwrap().as_array().to_listview(); assert_eq!( diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index c3d7b5115e9..1e32f7295bc 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -220,7 +220,8 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::accessor::ArrayAccessor; use crate::arrays::ChunkedArray; @@ -274,8 +275,11 @@ mod tests { ) .unwrap() .into_array(); + #[expect(deprecated)] let canonical_struct = chunked.to_struct(); + #[expect(deprecated)] let canonical_varbin = canonical_struct.unmasked_field(0).to_varbinview(); + #[expect(deprecated)] let original_varbin = struct_array.unmasked_field(0).to_varbinview(); let orig_values = original_varbin .with_iterator(|it| it.map(|a| a.map(|v| v.to_vec())).collect::>()); @@ -305,6 +309,7 @@ mod tests { List(Arc::new(Primitive(I32, NonNullable)), NonNullable), ); + #[expect(deprecated)] let canon_values = chunked_list.unwrap().as_array().to_listview(); assert_eq!( diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index c0bc63bfdb6..5e9c4a4d817 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -21,7 +21,8 @@ use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayParts; @@ -186,6 +187,7 @@ impl VTable for Chunked { &DType::Primitive(PType::U64, Nullability::NonNullable), nchunks + 1, )?; + #[expect(deprecated)] let chunk_offsets_buf = chunk_offsets.to_primitive().to_buffer::(); let chunk_offsets_usize = chunk_offsets_buf .iter() diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index e01a37f7caa..15e67a02929 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -72,7 +72,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; @@ -100,6 +101,7 @@ mod tests { taken.dtype() ); assert_arrays_eq!( + #[expect(deprecated)] taken.to_primitive(), PrimitiveArray::new( buffer![42i32, 42, 42], @@ -128,6 +130,7 @@ mod tests { taken.dtype() ); assert_arrays_eq!( + #[expect(deprecated)] taken.to_primitive(), PrimitiveArray::new(buffer![42i32, 42, 42], Validity::AllValid) ); diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index fbffbfe4576..8d4955c2261 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -333,7 +333,8 @@ mod tests { use crate::arrays::listview::ListViewRebuildMode; use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; @@ -346,6 +347,7 @@ mod tests { #[test] fn test_canonicalize_null() { let const_null = ConstantArray::new(Scalar::null(DType::Null), 42); + #[expect(deprecated)] let actual = const_null.as_array().to_null(); assert_eq!(actual.len(), 42); assert_eq!( @@ -375,6 +377,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(); + #[expect(deprecated)] let canonical = const_array.to_canonical()?.into_array(); let canonical_stats = canonical.statistics(); @@ -400,6 +403,7 @@ mod tests { // Create a ConstantArray with the f16 scalar let const_array = ConstantArray::new(f16_scalar.clone(), 1).into_array(); + #[expect(deprecated)] let canonical_const = const_array.to_primitive(); // Verify the scalar value is preserved through canonicalization @@ -419,17 +423,21 @@ mod tests { Nullability::NonNullable, ); let const_array = ConstantArray::new(list_scalar, 2).into_array(); + #[expect(deprecated)] let canonical_const = const_array.to_listview(); let list_array = canonical_const.rebuild(ListViewRebuildMode::MakeZeroCopyToList)?; assert_arrays_eq!( + #[expect(deprecated)] list_array.elements().to_primitive(), PrimitiveArray::from_iter([1u64, 2, 1, 2]) ); assert_arrays_eq!( + #[expect(deprecated)] list_array.offsets().to_primitive(), PrimitiveArray::from_iter([0u64, 2]) ); assert_arrays_eq!( + #[expect(deprecated)] list_array.sizes().to_primitive(), PrimitiveArray::from_iter([2u64, 2]) ); @@ -444,13 +452,18 @@ mod tests { Nullability::NonNullable, ); let const_array = ConstantArray::new(list_scalar, 2).into_array(); + #[expect(deprecated)] let canonical_const = const_array.to_listview(); - assert!(canonical_const.elements().to_primitive().is_empty()); + #[expect(deprecated)] + let elements_prim = canonical_const.elements().to_primitive(); + assert!(elements_prim.is_empty()); assert_arrays_eq!( + #[expect(deprecated)] canonical_const.offsets().to_primitive(), PrimitiveArray::from_iter([0u64, 0]) ); assert_arrays_eq!( + #[expect(deprecated)] canonical_const.sizes().to_primitive(), PrimitiveArray::from_iter([0u64, 0]) ); @@ -463,13 +476,18 @@ mod tests { Nullability::Nullable, )); let const_array = ConstantArray::new(list_scalar, 2).into_array(); + #[expect(deprecated)] let canonical_const = const_array.to_listview(); - assert!(canonical_const.elements().to_primitive().is_empty()); + #[expect(deprecated)] + let elements_prim = canonical_const.elements().to_primitive(); + assert!(elements_prim.is_empty()); assert_arrays_eq!( + #[expect(deprecated)] canonical_const.offsets().to_primitive(), PrimitiveArray::from_iter([0u64, 0]) ); assert_arrays_eq!( + #[expect(deprecated)] canonical_const.sizes().to_primitive(), PrimitiveArray::from_iter([0u64, 0]) ); @@ -488,6 +506,7 @@ mod tests { 3, ); + #[expect(deprecated)] let struct_array = array.as_array().to_struct(); assert_eq!(struct_array.len(), 3); assert_eq!( @@ -521,6 +540,7 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 4).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 4); @@ -530,6 +550,7 @@ mod tests { // Check that each list is [10, 20, 30]. for i in 0..4 { let list = canonical.fixed_size_list_elements_at(i).unwrap(); + #[expect(deprecated)] let list_primitive = list.to_primitive(); assert_arrays_eq!(list_primitive, PrimitiveArray::from_iter([10i32, 20, 30])); } @@ -548,6 +569,7 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 3).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 3); @@ -555,6 +577,7 @@ mod tests { assert!(matches!(canonical.validity(), Ok(Validity::AllValid))); // Check elements. + #[expect(deprecated)] let elements = canonical.elements().to_primitive(); assert_arrays_eq!( elements, @@ -572,6 +595,7 @@ mod tests { )); let const_array = ConstantArray::new(fsl_scalar, 5).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 5); @@ -579,6 +603,7 @@ mod tests { assert!(matches!(canonical.validity(), Ok(Validity::AllInvalid))); // Elements should be defaults (zeros). + #[expect(deprecated)] let elements = canonical.elements().to_primitive(); assert_eq!(elements.len(), 20); // 5 lists * 4 elements each assert!(elements.as_slice::().iter().all(|&x| x == 0)); @@ -594,6 +619,7 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 10).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 10); @@ -614,12 +640,14 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 2).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 2); assert_eq!(canonical.list_size(), 2); // Check elements are repeated correctly. + #[expect(deprecated)] let elements = canonical.elements().to_varbinview(); assert_eq!( elements @@ -657,11 +685,13 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 1).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 1); assert_eq!(canonical.list_size(), 1); + #[expect(deprecated)] let elements = canonical.elements().to_primitive(); assert_arrays_eq!(elements, PrimitiveArray::from_iter([42i16])); } @@ -680,6 +710,7 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 3).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 3); @@ -687,6 +718,7 @@ mod tests { assert!(matches!(canonical.validity(), Ok(Validity::NonNullable))); // Check elements including nulls. + #[expect(deprecated)] let elements = canonical.elements().to_primitive(); assert_eq!( elements @@ -737,11 +769,13 @@ mod tests { ); let const_array = ConstantArray::new(fsl_scalar, 1000).into_array(); + #[expect(deprecated)] let canonical = const_array.to_fixed_size_list(); assert_eq!(canonical.len(), 1000); assert_eq!(canonical.list_size(), 5); + #[expect(deprecated)] let elements = canonical.elements().to_primitive(); assert_eq!(elements.len(), 5000); diff --git a/vortex-array/src/arrays/datetime/test.rs b/vortex-array/src/arrays/datetime/test.rs index 3d270ec7053..2b1a681e7fb 100644 --- a/vortex-array/src/arrays/datetime/test.rs +++ b/vortex-array/src/arrays/datetime/test.rs @@ -8,7 +8,8 @@ use vortex_error::VortexResult; use crate::IntoArray; use crate::Precision; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::arrays::PrimitiveArray; use crate::arrays::datetime::TemporalData; use crate::assert_arrays_eq; @@ -195,11 +196,10 @@ fn test_validity_preservation(#[case] validity: Validity) { let temporal_array = TemporalData::new_timestamp(milliseconds, TimeUnit::Milliseconds, Some("UTC".into())); + #[expect(deprecated)] + let prim = temporal_array.temporal_values().to_primitive(); assert!( - temporal_array - .temporal_values() - .to_primitive() - .validity() + prim.validity() .vortex_expect("temporal validity should be derivable") .array_eq(&validity, Precision::Ptr) ); diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index f602b827c03..80155954214 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -150,7 +150,8 @@ mod tests { use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::DecimalDType; @@ -169,6 +170,7 @@ mod tests { // Cast to nullable let nullable_dtype = DType::Decimal(decimal_dtype, Nullability::Nullable); + #[expect(deprecated)] let casted = array .into_array() .cast(nullable_dtype.clone()) @@ -189,6 +191,7 @@ mod tests { // Cast to non-nullable let non_nullable_dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable); + #[expect(deprecated)] let casted = array .into_array() .cast(non_nullable_dtype.clone()) @@ -209,11 +212,12 @@ mod tests { // Attempt to cast to non-nullable should fail let non_nullable_dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable); - array + #[expect(deprecated)] + let result = array .into_array() .cast(non_nullable_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())) - .unwrap(); + .and_then(|a| a.to_canonical().map(|c| c.into_array())); + result.unwrap(); } #[test] @@ -226,6 +230,7 @@ mod tests { // Try to cast to different scale - not supported let different_dtype = DType::Decimal(DecimalDType::new(15, 3), Nullability::NonNullable); + #[expect(deprecated)] let result = array .into_array() .cast(different_dtype) @@ -250,6 +255,7 @@ mod tests { // Try to downcast precision - not supported let smaller_dtype = DType::Decimal(DecimalDType::new(10, 2), Nullability::NonNullable); + #[expect(deprecated)] let result = array .into_array() .cast(smaller_dtype) @@ -274,6 +280,7 @@ mod tests { // Cast to higher precision with same scale - should succeed let wider_dtype = DType::Decimal(DecimalDType::new(38, 2), Nullability::NonNullable); + #[expect(deprecated)] let casted = array.into_array().cast(wider_dtype).unwrap().to_decimal(); assert_eq!(casted.precision(), 38); @@ -292,6 +299,7 @@ mod tests { ); // Try to cast to non-decimal type - should fail since no kernel can handle it + #[expect(deprecated)] let result = array .into_array() .cast(DType::Utf8(Nullability::NonNullable)) diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index de01076b231..684e5aa61e4 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -94,7 +94,8 @@ mod tests { use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DecimalDType; use crate::dtype::Nullability; use crate::scalar::DecimalValue; @@ -108,6 +109,7 @@ mod tests { [None, Some(800i128), None, Some(1000i128), None], decimal_dtype, ); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( @@ -144,6 +146,7 @@ mod tests { decimal_dtype, ); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( @@ -165,6 +168,7 @@ mod tests { let decimal_dtype = DecimalDType::new(3, 0); let arr = DecimalArray::from_option_iter([None, Some(10i8), None], decimal_dtype); // i8 max is 127, so 200 doesn't fit — the array should be widened to i16. + #[expect(deprecated)] let result = arr .into_array() .fill_null(Scalar::decimal( @@ -189,6 +193,7 @@ mod tests { decimal_dtype, Validity::NonNullable, ); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::decimal( diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index d3ef863990c..f1f6db96f72 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -13,7 +13,8 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; @@ -145,6 +146,7 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { let codes_validity = codes .validity()? .to_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; + #[expect(deprecated)] let codes_primitive = self.codes().to_primitive(); let values_len = self.values().len(); @@ -270,7 +272,8 @@ mod test { use crate::ArrayRef; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ChunkedArray; use crate::arrays::DictArray; @@ -433,6 +436,7 @@ mod test { ); array.append_to_builder(builder.as_mut(), &mut LEGACY_SESSION.create_execution_ctx())?; + #[expect(deprecated)] let into_prim = array.to_primitive(); let prim_into = builder.finish_into_canonical().into_primitive(); diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index 07de42dd384..c9575c32ae2 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -57,7 +57,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::Dict; use crate::arrays::PrimitiveArray; use crate::arrays::dict::DictArraySlotsExt; @@ -83,6 +84,7 @@ mod tests { &DType::Primitive(PType::I64, Nullability::NonNullable) ); + #[expect(deprecated)] let decoded = casted.to_primitive(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i64, 2, 3, 2, 1])); } @@ -168,7 +170,9 @@ mod tests { ); // Verify values are unchanged + #[expect(deprecated)] let original_values = dict.as_array().to_primitive(); + #[expect(deprecated)] let final_values = back_to_non_nullable.to_primitive(); assert_arrays_eq!(original_values, final_values); } @@ -217,9 +221,8 @@ mod tests { casted.dtype(), &DType::Primitive(PType::F64, Nullability::NonNullable) ); - assert_arrays_eq!( - casted.to_primitive(), - PrimitiveArray::from_iter([1.0f64, 3.0, 1.0]) - ); + #[expect(deprecated)] + let casted_prim = casted.to_primitive(); + assert_arrays_eq!(casted_prim, PrimitiveArray::from_iter([1.0f64, 3.0, 1.0])); } } diff --git a/vortex-array/src/arrays/dict/compute/fill_null.rs b/vortex-array/src/arrays/dict/compute/fill_null.rs index 8627759256a..189d8240050 100644 --- a/vortex-array/src/arrays/dict/compute/fill_null.rs +++ b/vortex-array/src/arrays/dict/compute/fill_null.rs @@ -95,7 +95,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::DictArray; use crate::arrays::PrimitiveArray; @@ -121,6 +122,7 @@ mod tests { .into_array() .fill_null(Scalar::primitive(20, Nullability::NonNullable)) .vortex_expect("operation should succeed in test"); + #[expect(deprecated)] let filled_primitive = filled.to_primitive(); assert_arrays_eq!(filled_primitive, PrimitiveArray::from_iter([10, 20, 20])); assert!( diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index 84facbd34bf..c56cc8ef367 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -60,7 +60,8 @@ mod test { use crate::ArrayRef; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; @@ -89,6 +90,7 @@ mod test { let dict = dict_encode(&PrimitiveArray::from_option_iter(values.clone()).into_array()).unwrap(); + #[expect(deprecated)] let actual = dict.as_array().to_primitive(); let expected = PrimitiveArray::from_option_iter(values); @@ -102,6 +104,7 @@ mod test { let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 32])); let dict = dict_encode(&expected.clone().into_array()).unwrap(); + #[expect(deprecated)] let actual = dict.as_array().to_primitive(); assert_arrays_eq!(actual, expected); @@ -113,6 +116,7 @@ mod test { let expected = PrimitiveArray::from_iter((0..1000).map(|i| unique_values[i % 100])); let dict = dict_encode(&expected.clone().into_array()).unwrap(); + #[expect(deprecated)] let actual = dict.as_array().to_primitive(); assert_arrays_eq!(actual, expected); @@ -126,6 +130,7 @@ mod test { ); assert_eq!(reference.len(), 6); let dict = dict_encode(&reference.clone().into_array()).unwrap(); + #[expect(deprecated)] let flattened_dict = dict.as_array().to_varbinview(); assert_eq!( flattened_dict.with_iterator(|iter| iter diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index cf1c26491b0..a67deff4c45 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -198,7 +198,7 @@ impl VTable for Dict { let values = array.values().clone(); debug_assert!(values.is_canonical()); // TODO: add canonical owned cast. - let values = values.to_canonical()?; + let values = values.execute::(ctx)?; Ok(ExecutionResult::done(take_canonical(values, &codes, ctx)?)) } diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index eda55a73f50..af42fd5c9ea 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -105,12 +105,12 @@ mod tests { let storage = buffer![1i64].into_array(); let arr = ExtensionArray::new(original_dtype, storage); - assert!( - arr.into_array() - .cast(DType::Extension(target_dtype)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())) - .is_err() - ); + #[expect(deprecated)] + let result = arr + .into_array() + .cast(DType::Extension(target_dtype)) + .and_then(|a| a.to_canonical().map(|c| c.into_array())); + assert!(result.is_err()); } #[test] diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 3f5295f729e..6a58e4838be 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -56,7 +56,8 @@ mod tests { use vortex_mask::Mask; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::ConstantArray; use crate::arrays::Extension; use crate::arrays::ExtensionArray; @@ -143,7 +144,9 @@ mod tests { assert_eq!(ext_result.ext_dtype(), &ext_dtype); // Check the storage values - let storage_result: &[i64] = &ext_result.storage_array().to_primitive().to_buffer::(); + #[expect(deprecated)] + let storage_prim = ext_result.storage_array().to_primitive(); + let storage_result: &[i64] = &storage_prim.to_buffer::(); assert_eq!(storage_result, &[1, 3, 5]); } @@ -169,6 +172,7 @@ mod tests { assert_eq!(ext_result.len(), 3); // Check values: should be [Some(1), None, None] + #[expect(deprecated)] let canonical = ext_result.storage_array().to_primitive(); assert_eq!(canonical.len(), 3); } diff --git a/vortex-array/src/arrays/filter/execute/bool.rs b/vortex-array/src/arrays/filter/execute/bool.rs index 1a35d1e74aa..a22e55366fc 100644 --- a/vortex-array/src/arrays/filter/execute/bool.rs +++ b/vortex-array/src/arrays/filter/execute/bool.rs @@ -31,7 +31,8 @@ mod test { use crate::IntoArray; use crate::arrays::filter::execute::bool::BoolArray; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::compute::conformance::filter::test_filter_conformance; #[test] @@ -39,6 +40,7 @@ mod test { let arr = BoolArray::from_iter([true, true, false]); let mask = Mask::from_iter([true, false, true]); + #[expect(deprecated)] let filtered = arr.filter(mask).unwrap().to_bool(); assert_eq!(2, filtered.len()); diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index 4f7dbbeab2a..1ee018f3168 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -76,7 +76,8 @@ mod test { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -186,6 +187,7 @@ mod test { // Filter to keep only 2 lists. let mask = Mask::from_iter([true, false, false, true, false]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2, "Wrong number of filtered lists"); @@ -218,6 +220,7 @@ mod test { // Filter to keep lists with gaps and overlaps. let mask = Mask::from_iter([false, true, true, true, false]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 3, "Wrong filter result length"); @@ -261,6 +264,7 @@ mod test { let mask1 = Mask::from_iter([true, false, true, false]); let result1 = const_offset_list.filter(mask1).unwrap(); + #[expect(deprecated)] let result1_list = result1.to_listview(); assert_eq!(result1_list.len(), 2); @@ -284,6 +288,7 @@ mod test { let mask2 = Mask::from_iter([true, false, true]); let result2 = both_const_list.filter(mask2).unwrap(); + #[expect(deprecated)] let result2_list = result2.to_listview(); assert_eq!(result2_list.len(), 2); @@ -310,6 +315,7 @@ mod test { // Filter to keep only 2 lists, demonstrating we keep all 10000 elements. let mask = Mask::from_iter([false, true, false, false, true]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2); @@ -345,6 +351,7 @@ mod test { // Test sparse selection from large dataset. let sparse_mask = Mask::from_iter((0..5).map(|i| i == 0 || i == 4)); let sparse_result = listview.filter(sparse_mask).unwrap(); + #[expect(deprecated)] let sparse_list = sparse_result.to_listview(); assert_eq!(sparse_list.len(), 2); diff --git a/vortex-array/src/arrays/filter/execute/primitive.rs b/vortex-array/src/arrays/filter/execute/primitive.rs index 4a1b6367428..0aae5738da7 100644 --- a/vortex-array/src/arrays/filter/execute/primitive.rs +++ b/vortex-array/src/arrays/filter/execute/primitive.rs @@ -35,7 +35,8 @@ mod test { use crate::IntoArray; use crate::arrays::PrimitiveArray; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::compute::conformance::filter::LARGE_SIZE; use crate::compute::conformance::filter::MEDIUM_SIZE; use crate::compute::conformance::filter::test_filter_conformance; @@ -45,6 +46,7 @@ mod test { let mask = [true, true, false, true, true, true, false, true]; let arr = PrimitiveArray::from_iter([1u32, 24, 54, 2, 3, 2, 3, 2]); + #[expect(deprecated)] let filtered = arr.filter(Mask::from_iter(mask)).unwrap().to_primitive(); assert_eq!( filtered.len(), diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index 465b2817166..00c2eb8a198 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -10,8 +10,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; -use crate::ToCanonical; use crate::array::ArrayView; +use crate::arrays::BoolArray; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; use crate::arrays::Primitive; @@ -162,7 +162,7 @@ fn take_nullable_fsl( { Validity::NonNullable | Validity::AllValid => Mask::new_true(indices_len), Validity::AllInvalid => Mask::new_false(indices_len), - Validity::Array(a) => a.to_bool().to_mask(ctx), + Validity::Array(a) => a.execute::(ctx)?.to_mask(ctx), }; // We must use placeholder zeros for null lists to maintain the array length without diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs index 81fd9afe0c5..efd6f630580 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs @@ -7,7 +7,8 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::PrimitiveArray; @@ -84,6 +85,7 @@ fn test_fsl_of_fsl_basic() { let first_outer_list = outer_fsl.fixed_size_list_elements_at(0).unwrap(); // Check first inner list [1,2]. + #[expect(deprecated)] let inner_list_0 = first_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(0) @@ -102,6 +104,7 @@ fn test_fsl_of_fsl_basic() { ); // Check second inner list [3,4]. + #[expect(deprecated)] let inner_list_1 = first_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(1) @@ -120,6 +123,7 @@ fn test_fsl_of_fsl_basic() { ); // Check third inner list [5,6]. + #[expect(deprecated)] let inner_list_2 = first_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(2) @@ -141,6 +145,7 @@ fn test_fsl_of_fsl_basic() { let second_outer_list = outer_fsl.fixed_size_list_elements_at(1).unwrap(); // Check first inner list [7,8]. + #[expect(deprecated)] let inner_list_0 = second_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(0) @@ -159,6 +164,7 @@ fn test_fsl_of_fsl_basic() { ); // Check second inner list [9,10]. + #[expect(deprecated)] let inner_list_1 = second_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(1) @@ -177,6 +183,7 @@ fn test_fsl_of_fsl_basic() { ); // Check third inner list [11,12]. + #[expect(deprecated)] let inner_list_2 = second_outer_list .to_fixed_size_list() .fixed_size_list_elements_at(2) @@ -301,16 +308,19 @@ fn test_deeply_nested_fsl() { // Check the actual deeply nested values. // Structure: [[[1,2],[3,4]],[[5,6],[7,8]]]. let top_level = level3.fixed_size_list_elements_at(0).unwrap(); + #[expect(deprecated)] let level2_0 = top_level .to_fixed_size_list() .fixed_size_list_elements_at(0) .unwrap(); + #[expect(deprecated)] let level2_1 = top_level .to_fixed_size_list() .fixed_size_list_elements_at(1) .unwrap(); // First level-2 list: [[1,2],[3,4]]. + #[expect(deprecated)] let level1_0_0 = level2_0 .to_fixed_size_list() .fixed_size_list_elements_at(0) @@ -328,6 +338,7 @@ fn test_deeply_nested_fsl() { 2i32.into() ); + #[expect(deprecated)] let level1_0_1 = level2_0 .to_fixed_size_list() .fixed_size_list_elements_at(1) @@ -346,6 +357,7 @@ fn test_deeply_nested_fsl() { ); // Second level-2 list: [[5,6],[7,8]]. + #[expect(deprecated)] let level1_1_0 = level2_1 .to_fixed_size_list() .fixed_size_list_elements_at(0) @@ -363,6 +375,7 @@ fn test_deeply_nested_fsl() { 6i32.into() ); + #[expect(deprecated)] let level1_1_1 = level2_1 .to_fixed_size_list() .fixed_size_list_elements_at(1) diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 55b143f644f..033845088b5 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -327,7 +327,9 @@ pub trait ListArrayExt: TypedArrayRef { fn reset_offsets(&self, recurse: bool) -> VortexResult> { let mut elements = self.sliced_elements()?; if recurse && elements.is_canonical() { - elements = elements.to_canonical()?.compact()?.into_array(); + #[expect(deprecated)] + let compacted = elements.to_canonical()?.compact()?.into_array(); + elements = compacted; } else if recurse && let Some(child_list_array) = elements.as_opt::() { elements = child_list_array .into_owned() diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index bac84916390..c9813cfe92d 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -87,10 +87,11 @@ mod tests { let target_dtype = DType::Primitive(PType::U64, Nullability::NonNullable); // can't cast list to u64 - let result = list - .into_array() - .cast(target_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + let result = list.into_array().cast(target_dtype).and_then(|a| { + #[expect(deprecated)] + let canonical = a.to_canonical().map(|c| c.into_array()); + canonical + }); assert!(result.is_err()); } @@ -111,10 +112,11 @@ mod tests { Nullability::NonNullable, ); - let result = list - .into_array() - .cast(target_dtype) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + let result = list.into_array().cast(target_dtype).and_then(|a| { + #[expect(deprecated)] + let canonical = a.to_canonical().map(|c| c.into_array()); + canonical + }); assert!(result.is_err()); // Nulls in list element array — the inner cast error is deferred until diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 52c2296dec8..33551227b68 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -200,7 +200,8 @@ mod test { use crate::IntoArray as _; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ListArray; @@ -235,6 +236,7 @@ mod test { ) ); + #[expect(deprecated)] let result = result.to_listview(); assert_eq!(result.len(), 4); @@ -337,6 +339,7 @@ mod test { ) ); + #[expect(deprecated)] let result = result.to_listview(); assert_eq!(result.len(), 3); @@ -467,6 +470,7 @@ mod test { assert_eq!(result.len(), 2); + #[expect(deprecated)] let result_view = result.to_listview(); assert_eq!(result_view.len(), 2); assert!( @@ -496,6 +500,7 @@ mod test { assert_eq!(result.len(), 3); + #[expect(deprecated)] let result_view = result.to_listview(); assert_eq!(result_view.len(), 3); assert!( diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index f440b360951..cae55bf0828 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -14,7 +14,8 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; @@ -249,7 +250,9 @@ impl ListViewData { // Skip host-only validation when offsets/sizes are not host-resident. if offsets.is_host() && sizes.is_host() { + #[expect(deprecated)] let offsets_primitive = offsets.to_primitive(); + #[expect(deprecated)] let sizes_primitive = sizes.to_primitive(); // Validate the `offsets` and `sizes` arrays. @@ -382,12 +385,11 @@ pub trait ListViewArrayExt: TypedArrayRef { } fn verify_is_zero_copy_to_list(&self) -> bool { - validate_zctl( - self.elements(), - self.offsets().to_primitive(), - self.sizes().to_primitive(), - ) - .is_ok() + #[expect(deprecated)] + let offsets_primitive = self.offsets().to_primitive(); + #[expect(deprecated)] + let sizes_primitive = self.sizes().to_primitive(); + validate_zctl(self.elements(), offsets_primitive, sizes_primitive).is_ok() } } impl> ListViewArrayExt for T {} @@ -456,12 +458,12 @@ impl Array { /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { if cfg!(debug_assertions) && is_zctl { - validate_zctl( - self.elements(), - self.offsets().to_primitive(), - self.sizes().to_primitive(), - ) - .vortex_expect("Failed to validate zero-copy to list flag"); + #[expect(deprecated)] + let offsets_primitive = self.offsets().to_primitive(); + #[expect(deprecated)] + let sizes_primitive = self.sizes().to_primitive(); + validate_zctl(self.elements(), offsets_primitive, sizes_primitive) + .vortex_expect("Failed to validate zero-copy to list flag"); } let dtype = self.dtype().clone(); let len = self.len(); diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index 54571ecdf21..54ee8c2705c 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -8,7 +8,8 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; use crate::arrays::ListArray; @@ -149,6 +150,7 @@ unsafe fn build_list_offsets_from_list_view( // Create uninit range for direct memory access. let mut offsets_range = offsets_builder.uninit_range(len + 1); + #[expect(deprecated)] let offsets = list_view.offsets().to_primitive(); let offsets_slice = offsets.as_slice::(); debug_assert!(offsets_slice.is_sorted()); @@ -187,6 +189,7 @@ pub fn recursive_list_from_list_view(array: ArrayRef) -> VortexResult return Ok(array); } + #[expect(deprecated)] let canonical = array.to_canonical()?; Ok(match canonical { diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index d0a88a750e6..d2ab4d92df7 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -8,7 +8,8 @@ use vortex_error::VortexResult; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::arrays::ConstantArray; @@ -116,6 +117,7 @@ impl ListViewArray { fn naive_rebuild( &self, ) -> VortexResult { + #[expect(deprecated)] let sizes_canonical = self.sizes().to_primitive(); let total: u64 = sizes_canonical .as_slice::() @@ -149,8 +151,10 @@ impl ListViewArray { fn rebuild_with_take( &self, ) -> VortexResult { + #[expect(deprecated)] let offsets_canonical = self.offsets().to_primitive(); let offsets_slice = offsets_canonical.as_slice::(); + #[expect(deprecated)] let sizes_canonical = self.sizes().to_primitive(); let sizes_slice = sizes_canonical.as_slice::(); @@ -202,8 +206,10 @@ impl ListViewArray { .as_list_element_opt() .vortex_expect("somehow had a canonical list that was not a list"); + #[expect(deprecated)] let offsets_canonical = self.offsets().to_primitive(); let offsets_slice = offsets_canonical.as_slice::(); + #[expect(deprecated)] let sizes_canonical = self.sizes().to_primitive(); let sizes_slice = sizes_canonical.as_slice::(); @@ -217,6 +223,7 @@ impl ListViewArray { let mut new_sizes = BufferMut::::with_capacity(len); // Canonicalize the elements up front as we will be slicing the elements quite a lot. + #[expect(deprecated)] let elements_canonical = self .elements() .to_canonical() @@ -379,7 +386,8 @@ mod tests { use super::ListViewRebuildMode; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -502,6 +510,7 @@ mod tests { ); // Note that element at index 2 (97) is preserved as a gap. + #[expect(deprecated)] let all_elements = trimmed.elements().to_primitive(); assert_eq!( all_elements diff --git a/vortex-array/src/arrays/listview/tests/filter.rs b/vortex-array/src/arrays/listview/tests/filter.rs index 271ea746482..26a5190e517 100644 --- a/vortex-array/src/arrays/listview/tests/filter.rs +++ b/vortex-array/src/arrays/listview/tests/filter.rs @@ -12,7 +12,8 @@ use super::common::create_nullable_listview; use super::common::create_overlapping_listview; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; @@ -49,6 +50,7 @@ fn test_filter_preserves_unreferenced_elements() { // Filter to keep only 2 lists. let mask = Mask::from_iter([true, false, false, true, false]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2, "Wrong number of filtered lists"); @@ -80,6 +82,7 @@ fn test_filter_with_gaps() { // Filter to keep lists with gaps and overlaps. let mask = Mask::from_iter([false, true, true, true, false]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 3, "Wrong filter result length"); @@ -123,6 +126,7 @@ fn test_filter_constant_arrays() { let mask1 = Mask::from_iter([true, false, true, false]); let result1 = const_offset_list.filter(mask1).unwrap(); + #[expect(deprecated)] let result1_list = result1.to_listview(); assert_eq!(result1_list.len(), 2); @@ -146,6 +150,7 @@ fn test_filter_constant_arrays() { let mask2 = Mask::from_iter([true, false, true]); let result2 = both_const_list.filter(mask2).unwrap(); + #[expect(deprecated)] let result2_list = result2.to_listview(); assert_eq!(result2_list.len(), 2); @@ -171,6 +176,7 @@ fn test_filter_extreme_offsets() { // Filter to keep only 2 lists, demonstrating we keep all 10000 elements. let mask = Mask::from_iter([false, true, false, false, true]); let result = listview.filter(mask).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2); @@ -206,6 +212,7 @@ fn test_filter_extreme_offsets() { // Test sparse selection from large dataset. let sparse_mask = Mask::from_iter((0..5).map(|i| i == 0 || i == 4)); let sparse_result = listview.filter(sparse_mask).unwrap(); + #[expect(deprecated)] let sparse_list = sparse_result.to_listview(); assert_eq!(sparse_list.len(), 2); diff --git a/vortex-array/src/arrays/listview/tests/operations.rs b/vortex-array/src/arrays/listview/tests/operations.rs index b890c346d87..235caf53caa 100644 --- a/vortex-array/src/arrays/listview/tests/operations.rs +++ b/vortex-array/src/arrays/listview/tests/operations.rs @@ -12,7 +12,8 @@ use super::common::create_large_listview; use super::common::create_nullable_listview; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::is_constant::is_constant; use crate::arrays::BoolArray; @@ -251,6 +252,7 @@ fn test_cast_numeric_types(#[case] from_ptype: PType, #[case] to_ptype: PType) { let result = listview.cast(target_dtype.clone()).unwrap(); assert_eq!(result.dtype(), &target_dtype); + #[expect(deprecated)] let result_list = result.to_listview(); assert!( result_list.len() == 3 || result_list.len() == 2, @@ -287,6 +289,7 @@ fn test_cast_with_nulls() { let result = listview.cast(target_dtype.clone()).unwrap(); assert_eq!(result.dtype(), &target_dtype); + #[expect(deprecated)] let result_list = result.to_listview(); assert!( result_list @@ -337,6 +340,7 @@ fn test_cast_special_patterns(#[case] expected_sizes: Vec, #[case] list_c }; let result = listview.cast(target_dtype).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), list_count); @@ -369,6 +373,7 @@ fn test_cast_large_dataset() { ); let result = listview.cast(target_dtype).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 20); @@ -537,6 +542,7 @@ fn test_mask_preserves_structure() { let result = listview.mask((!&selection).into_array()).unwrap(); assert_eq!(result.len(), 4); // Length is preserved. + #[expect(deprecated)] let result_list = result.to_listview(); // Check validity: true in selection means null. @@ -590,6 +596,7 @@ fn test_mask_with_existing_nulls() { // Mask additional elements. let selection = Mask::from_iter([false, true, true]); let result = listview.mask((!&selection).into_array()).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); // Check combined validity: @@ -622,6 +629,7 @@ fn test_mask_with_gaps() { let selection = Mask::from_iter([true, false, false]); let result = listview.mask((!&selection).into_array()).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 3); @@ -666,6 +674,7 @@ fn test_mask_constant_arrays() { let selection = Mask::from_iter([false, true, false]); let result = const_list.mask((!&selection).into_array()).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 3); diff --git a/vortex-array/src/arrays/listview/tests/take.rs b/vortex-array/src/arrays/listview/tests/take.rs index b1134f97a04..a162d61b217 100644 --- a/vortex-array/src/arrays/listview/tests/take.rs +++ b/vortex-array/src/arrays/listview/tests/take.rs @@ -11,7 +11,8 @@ use super::common::create_nullable_listview; use super::common::create_overlapping_listview; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::ListViewArray; @@ -48,6 +49,7 @@ fn test_take_preserves_unreferenced_elements() { // Take only 2 lists. let indices = buffer![1u32, 3].into_array(); let result = listview.take(indices).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2); @@ -76,6 +78,7 @@ fn test_take_with_gaps() { let indices = buffer![1u32, 3, 4, 2].into_array(); let result = listview.take(indices).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); // Verify the entire elements array is preserved including gaps. @@ -111,6 +114,7 @@ fn test_take_constant_arrays() { let indices = buffer![3u32, 0, 2].into_array(); let result = const_offset_list.take(indices).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 3); @@ -135,6 +139,7 @@ fn test_take_constant_arrays() { let indices2 = buffer![2u32, 0].into_array(); let result2 = both_const_list.take(indices2).unwrap(); + #[expect(deprecated)] let result2_list = result2.to_listview(); assert_eq!(result2_list.len(), 2); @@ -160,6 +165,7 @@ fn test_take_extreme_offsets() { // Take only 2 lists, demonstrating we keep all 10000 elements. let indices = buffer![1u32, 4].into_array(); let result = listview.take(indices).unwrap(); + #[expect(deprecated)] let result_list = result.to_listview(); assert_eq!(result_list.len(), 2); diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index ed108a77bf7..a2db8c07f1a 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -9,6 +9,7 @@ use super::*; use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; +#[expect(deprecated)] use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; @@ -62,6 +63,7 @@ fn test_masked_child_with_validity() { let array = MaskedArray::try_new(child, Validity::from_iter([true, false, true, false, true])).unwrap(); + #[expect(deprecated)] let prim = array.as_array().to_primitive(); // Positions where validity is false should be null in masked_child. diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index 1d787369d96..502bb8fcc06 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -16,7 +16,8 @@ mod test { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::NullArray; use crate::compute::conformance::consistency::test_array_consistency; @@ -28,6 +29,7 @@ mod test { #[test] fn test_slice_nulls() { let nulls = NullArray::new(10); + #[expect(deprecated)] let sliced = nulls.slice(0..4).unwrap().to_null(); assert_eq!(sliced.len(), 4); @@ -45,6 +47,7 @@ mod test { #[test] fn test_take_nulls() { let nulls = NullArray::new(10); + #[expect(deprecated)] let taken = nulls .take(buffer![0u64, 2, 4, 6, 8].into_array()) .unwrap() diff --git a/vortex-array/src/arrays/null/compute/take.rs b/vortex-array/src/arrays/null/compute/take.rs index 76ae248eded..5352ae67653 100644 --- a/vortex-array/src/arrays/null/compute/take.rs +++ b/vortex-array/src/arrays/null/compute/take.rs @@ -6,7 +6,8 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::array::ArrayView; use crate::arrays::Null; use crate::arrays::NullArray; @@ -18,6 +19,7 @@ use crate::optimizer::rules::ParentRuleSet; impl TakeReduce for Null { #[expect(clippy::cast_possible_truncation)] fn take(array: ArrayView<'_, Null>, indices: &ArrayRef) -> VortexResult> { + #[expect(deprecated)] let indices = indices.to_primitive(); // Enforce all indices are valid diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index ef3ba7fbbfe..893d18db1ac 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -173,6 +173,7 @@ mod tests { // Take indices [0, 1, 2, 3, 4] - should get [0, 10, 0, 30, 0] let indices = buffer![0u32, 1, 2, 3, 4].into_array(); + #[expect(deprecated)] let result = array.take(indices)?.to_canonical()?.into_array(); let expected = PrimitiveArray::from_iter([0u16, 10, 0, 30, 0]).into_array(); @@ -186,6 +187,7 @@ mod tests { let array = make_patched_array(&[0; 10], &[1, 3], &[100, 200], 2..10)?; let indices = buffer![0u32, 1, 2, 3, 7].into_array(); + #[expect(deprecated)] let result = array.take(indices)?.to_canonical()?.into_array(); let expected = PrimitiveArray::from_iter([0u16, 200, 0, 0, 0]).into_array(); @@ -201,6 +203,7 @@ mod tests { // Take indices in reverse order let indices = buffer![4u32, 3, 2, 1, 0].into_array(); + #[expect(deprecated)] let result = array.take(indices)?.to_canonical()?.into_array(); let expected = PrimitiveArray::from_iter([0u16, 30, 0, 10, 0]).into_array(); @@ -216,9 +219,11 @@ mod tests { // Take the same patched index multiple times let indices = buffer![2u32, 2, 0, 2].into_array(); + #[expect(deprecated)] let result = array.take(indices)?.to_canonical()?.into_array(); // execute the array. + #[expect(deprecated)] let _canonical = result.to_canonical()?.into_primitive(); let expected = PrimitiveArray::from_iter([99u16, 99, 0, 99]).into_array(); @@ -250,6 +255,7 @@ mod tests { .into_array(), ), ); + #[expect(deprecated)] let result = array .take(indices.into_array())? .to_canonical()? diff --git a/vortex-array/src/arrays/primitive/array/accessor.rs b/vortex-array/src/arrays/primitive/array/accessor.rs index 5a3665c71ed..f26b201ea2b 100644 --- a/vortex-array/src/arrays/primitive/array/accessor.rs +++ b/vortex-array/src/arrays/primitive/array/accessor.rs @@ -5,7 +5,8 @@ use std::iter; use vortex_error::VortexExpect; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; @@ -26,6 +27,7 @@ impl ArrayAccessor for PrimitiveArray { } Validity::AllInvalid => f(&mut iter::repeat_n(None, self.len())), Validity::Array(v) => { + #[expect(deprecated)] let validity = v.to_bool().into_bit_buffer(); let mut iter = self .as_slice::() diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index c1bd0130eb9..21b474984a4 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -16,7 +16,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; @@ -179,46 +180,58 @@ pub trait PrimitiveArrayExt: TypedArrayRef { if min < 0 || max < 0 { // Signed if min >= i8::MIN as i64 && max <= i8::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::I8, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } if min >= i16::MIN as i64 && max <= i16::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::I16, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } if min >= i32::MIN as i64 && max <= i32::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::I32, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } } else { // Unsigned if max <= u8::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::U8, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } if max <= u16::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::U16, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } if max <= u32::MAX as i64 { - return Ok(self + #[expect(deprecated)] + let result = self .as_ref() .cast(DType::Primitive(PType::U32, nullability))? - .to_primitive()); + .to_primitive(); + return Ok(result); } } @@ -483,6 +496,7 @@ impl Array { BufferMut::::from_iter(buf_iter.zip(iter::repeat(false)).map(f)) } Validity::Array(val) => { + #[expect(deprecated)] let val = val.to_bool().into_bit_buffer(); BufferMut::::from_iter(buf_iter.zip(val.iter()).map(f)) } @@ -536,6 +550,7 @@ impl PrimitiveData { Validity::AllValid | Validity::NonNullable => valid_elems_buffer.aligned(alignment), Validity::AllInvalid => ByteBuffer::zeroed_aligned(n_rows * byte_width, alignment), Validity::Array(is_valid) => { + #[expect(deprecated)] let bool_array = is_valid.to_bool(); let bool_buffer = bool_array.to_bit_buffer(); let mut bytes = ByteBufferMut::zeroed_aligned(n_rows * byte_width, alignment); diff --git a/vortex-array/src/arrays/primitive/array/patch.rs b/vortex-array/src/arrays/primitive/array/patch.rs index 51fae5e9cd0..197db4d3854 100644 --- a/vortex-array/src/arrays/primitive/array/patch.rs +++ b/vortex-array/src/arrays/primitive/array/patch.rs @@ -135,7 +135,8 @@ mod tests { use vortex_buffer::buffer; use super::*; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::assert_arrays_eq; use crate::validity::Validity; @@ -174,8 +175,10 @@ mod tests { fn patch_sliced() { let input = PrimitiveArray::new(buffer![2u32; 10], Validity::AllValid); let sliced = input.slice(2..8).unwrap(); + #[expect(deprecated)] + let sliced_primitive = sliced.to_primitive(); assert_arrays_eq!( - sliced.to_primitive(), + sliced_primitive, PrimitiveArray::new(buffer![2u32; 6], Validity::AllValid) ); } diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index e5258bccf65..f1c9d3bcdbc 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -121,7 +121,8 @@ mod test { use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; @@ -133,11 +134,13 @@ mod test { let arr = buffer![0u32, 10, 200].into_array(); // cast from u32 to u8 + #[expect(deprecated)] let p = arr.cast(PType::U8.into()).unwrap().to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([0u8, 10, 200])); assert!(matches!(p.validity(), Ok(Validity::NonNullable))); // to nullable + #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::Nullable)) @@ -150,6 +153,7 @@ mod test { assert!(matches!(p.validity(), Ok(Validity::AllValid))); // back to non-nullable + #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) @@ -159,6 +163,7 @@ mod test { assert!(matches!(p.validity(), Ok(Validity::NonNullable))); // to nullable u32 + #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U32, Nullability::Nullable)) @@ -171,6 +176,7 @@ mod test { assert!(matches!(p.validity(), Ok(Validity::AllValid))); // to non-nullable u8 + #[expect(deprecated)] let p = p .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) @@ -183,6 +189,7 @@ mod test { #[test] fn cast_u32_f32() { let arr = buffer![0u32, 10, 200].into_array(); + #[expect(deprecated)] let u8arr = arr.cast(PType::F32.into()).unwrap().to_primitive(); assert_arrays_eq!(u8arr, PrimitiveArray::from_iter([0.0f32, 10., 200.])); } @@ -190,6 +197,7 @@ mod test { #[test] fn cast_i32_u32() { let arr = buffer![-1i32].into_array(); + #[expect(deprecated)] let error = arr .cast(PType::U32.into()) .and_then(|a| a.to_canonical().map(|c| c.into_array())) @@ -201,6 +209,7 @@ mod test { #[test] fn cast_array_with_nulls_to_nonnullable() { let arr = PrimitiveArray::from_option_iter([Some(-1i32), None, Some(10)]); + #[expect(deprecated)] let err = arr .into_array() .cast(PType::I32.into()) @@ -220,6 +229,7 @@ mod test { buffer![-1i32, 0, 10], Validity::from_iter([false, true, true]), ); + #[expect(deprecated)] let p = arr .into_array() .cast(DType::Primitive(PType::U32, Nullability::Nullable)) @@ -246,6 +256,7 @@ mod test { let src = PrimitiveArray::from_iter([0u32, 10, 100]); let src_ptr = src.as_slice::().as_ptr(); + #[expect(deprecated)] let dst = src.into_array().cast(PType::I32.into())?.to_primitive(); let dst_ptr = dst.as_slice::().as_ptr(); @@ -260,6 +271,7 @@ mod test { #[test] fn cast_same_width_int_out_of_range_errors() { let arr = buffer![u32::MAX].into_array(); + #[expect(deprecated)] let err = arr .cast(PType::I32.into()) .and_then(|a| a.to_canonical().map(|c| c.into_array())) @@ -272,6 +284,7 @@ mod test { #[test] fn cast_same_width_all_null() -> vortex_error::VortexResult<()> { let arr = PrimitiveArray::new(buffer![0xFFu8, 0xFF], Validity::AllInvalid); + #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::I8, Nullability::Nullable))? @@ -291,6 +304,7 @@ mod test { buffer![u32::MAX, 0u32, 42u32], Validity::from_iter([false, true, true]), ); + #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::I32, Nullability::Nullable))? @@ -308,6 +322,7 @@ mod test { buffer![1000u32, 10u32, 42u32], Validity::from_iter([false, true, true]), ); + #[expect(deprecated)] let casted = arr .into_array() .cast(DType::Primitive(PType::U8, Nullability::Nullable))? diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index 3ee48ce4c05..5862874a37d 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -57,13 +57,15 @@ mod test { use crate::arrays::primitive::compute::fill_null::BoolArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::scalar::Scalar; use crate::validity::Validity; #[test] fn fill_null_leading_none() { let arr = PrimitiveArray::from_option_iter([None, Some(8u8), None, Some(10), None]); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(42u8)) @@ -84,6 +86,7 @@ mod test { fn fill_null_all_none() { let arr = PrimitiveArray::from_option_iter([Option::::None, None, None, None, None]); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(255u8)) @@ -106,6 +109,7 @@ mod test { buffer![8u8, 10, 12, 14, 16], Validity::Array(BoolArray::from_iter([true, true, true, true, true]).into_array()), ); + #[expect(deprecated)] let p = arr .into_array() .fill_null(Scalar::from(255u8)) @@ -125,6 +129,7 @@ mod test { #[test] fn fill_null_non_nullable() { let arr = buffer![8u8, 10, 12, 14, 16].into_array(); + #[expect(deprecated)] let p = arr.fill_null(Scalar::from(255u8)).unwrap().to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16])); assert!( diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 5e63cdc2092..6c9a5f1edca 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -91,7 +91,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; use crate::arrays::VarBinArray; @@ -209,7 +210,9 @@ mod tests { .unwrap(); assert_eq!(result.dtype(), &target_dtype); assert_eq!(result.len(), 3); - assert_eq!(result.to_struct().struct_fields().nfields(), 2); + #[expect(deprecated)] + let nfields = result.to_struct().struct_fields().nfields(); + assert_eq!(nfields, 2); } #[test] @@ -238,6 +241,8 @@ mod tests { .unwrap(); assert_eq!(result.dtype(), &target_dtype); assert_eq!(result.len(), 3); - assert_eq!(result.to_struct().struct_fields().nfields(), 3); + #[expect(deprecated)] + let nfields = result.to_struct().struct_fields().nfields(); + assert_eq!(nfields, 3); } } diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index 91518ca7693..8a802969d39 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -153,7 +153,8 @@ mod tests { use crate::arrays::struct_::compute::rules::ConstantArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::FieldNames; use crate::dtype::Nullability; @@ -187,6 +188,7 @@ mod tests { // Use `ArrayBuiltins::cast` which goes through the optimizer and applies // `StructCastPushDownRule`. + #[expect(deprecated)] let result = source.into_array().cast(target).unwrap().to_struct(); assert_arrays_eq!( result.unmasked_field_by_name("a").unwrap(), @@ -255,6 +257,7 @@ mod tests { Nullability::NonNullable, ); + #[expect(deprecated)] let result = source.into_array().cast(target).unwrap().to_struct(); assert_eq!(result.unmasked_fields().len(), 2); assert_arrays_eq!( @@ -286,6 +289,7 @@ mod tests { Nullability::NonNullable, ); + #[expect(deprecated)] let result = source.into_array().cast(target).unwrap().to_struct(); assert_eq!( result.unmasked_field_by_name("val").unwrap().dtype(), diff --git a/vortex-array/src/arrays/varbin/accessor.rs b/vortex-array/src/arrays/varbin/accessor.rs index aa84ac64421..ee68f94bd55 100644 --- a/vortex-array/src/arrays/varbin/accessor.rs +++ b/vortex-array/src/arrays/varbin/accessor.rs @@ -5,7 +5,8 @@ use std::iter; use vortex_error::VortexExpect; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinArray; use crate::arrays::varbin::VarBinArrayExt; @@ -17,6 +18,7 @@ impl ArrayAccessor<[u8]> for VarBinArray { where F: for<'a> FnOnce(&mut dyn Iterator>) -> R, { + #[expect(deprecated)] let offsets = self.offsets().to_primitive(); let validity = self .validity() @@ -38,6 +40,7 @@ impl ArrayAccessor<[u8]> for VarBinArray { } Validity::AllInvalid => f(&mut iter::repeat_n(None, self.len())), Validity::Array(v) => { + #[expect(deprecated)] let validity = v.to_bool().into_bit_buffer(); let mut iter = offsets .windows(2) diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index fb8b2d54790..8d22f2ea451 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -13,7 +13,8 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; @@ -245,6 +246,7 @@ impl VarBinData { && matches!(dtype, DType::Utf8(_)) && let Some(bytes) = bytes.as_host_opt() { + #[expect(deprecated)] let primitive_offsets = offsets.to_primitive(); match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| { let offsets_slice = primitive_offsets.as_slice::(); diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 776f77a8796..963e67fdde5 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -146,7 +146,8 @@ mod test { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::VarBinArray; @@ -164,6 +165,7 @@ mod test { [Some(b"abc".to_vec()), None, Some(b"def".to_vec())], DType::Binary(Nullability::Nullable), ); + #[expect(deprecated)] let result = array .into_array() .binary( @@ -206,6 +208,7 @@ mod test { [None, None, Some(b"def".to_vec())], DType::Binary(Nullability::Nullable), ); + #[expect(deprecated)] let result = array .into_array() .binary(vbv.into_array(), Operator::Eq) diff --git a/vortex-array/src/arrays/varbin/vtable/canonical.rs b/vortex-array/src/arrays/varbin/vtable/canonical.rs index ecf651b3ea4..fc30ee2c94e 100644 --- a/vortex-array/src/arrays/varbin/vtable/canonical.rs +++ b/vortex-array/src/arrays/varbin/vtable/canonical.rs @@ -53,7 +53,8 @@ mod tests { use crate::arrays::VarBinViewArray; use crate::arrays::varbin::builder::VarBinBuilder; use crate::assert_arrays_eq; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; @@ -72,6 +73,7 @@ mod tests { let varbin = varbin.slice(1..4).unwrap(); + #[expect(deprecated)] let canonical = varbin.to_varbinview(); assert_eq!(canonical.dtype(), &dtype); @@ -95,6 +97,7 @@ mod tests { #[case(DType::Binary(Nullability::NonNullable))] fn test_canonical_varbin_unsliced(#[case] dtype: DType) { let varbin = VarBinArray::from_iter_nonnull(["foo", "bar", "baz"], dtype.clone()); + #[expect(deprecated)] let canonical = varbin.as_array().to_varbinview(); let expected = match dtype { DType::Utf8(_) => VarBinViewArray::from_iter_str(["foo", "bar", "baz"]), @@ -108,6 +111,7 @@ mod tests { fn test_canonical_varbin_empty() { let varbin = VarBinArray::from_iter_nonnull([] as [&str; 0], DType::Utf8(Nullability::NonNullable)); + #[expect(deprecated)] let canonical = varbin.as_array().to_varbinview(); assert_eq!(canonical.len(), 0); } diff --git a/vortex-array/src/arrays/varbinview/accessor.rs b/vortex-array/src/arrays/varbinview/accessor.rs index ef1470a090d..cd42c3f2b58 100644 --- a/vortex-array/src/arrays/varbinview/accessor.rs +++ b/vortex-array/src/arrays/varbinview/accessor.rs @@ -5,7 +5,8 @@ use std::iter; use vortex_error::VortexExpect; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinViewArray; use crate::validity::Validity; @@ -39,6 +40,7 @@ impl ArrayAccessor<[u8]> for VarBinViewArray { } Validity::AllInvalid => f(&mut iter::repeat_n(None, views.len())), Validity::Array(v) => { + #[expect(deprecated)] let validity = v.to_bool().into_bit_buffer(); let mut iter = views.iter().zip(validity.iter()).map(|(view, valid)| { if valid { diff --git a/vortex-array/src/arrays/varbinview/compute/mod.rs b/vortex-array/src/arrays/varbinview/compute/mod.rs index 772221f13ef..b69efad1e88 100644 --- a/vortex-array/src/arrays/varbinview/compute/mod.rs +++ b/vortex-array/src/arrays/varbinview/compute/mod.rs @@ -15,7 +15,8 @@ mod tests { use crate::IntoArray; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinViewArray; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; #[test] fn take_nullable() { let arr = VarBinViewArray::from_iter_nullable_str([ @@ -30,12 +31,12 @@ mod tests { let taken = arr.take(buffer![0, 3].into_array()).unwrap(); assert!(taken.dtype().is_nullable()); - assert_eq!( - taken.to_varbinview().with_iterator(|it| it - .map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) - .collect::>()), - [Some("one".to_string()), Some("four".to_string())] - ); + #[expect(deprecated)] + let result = taken.to_varbinview().with_iterator(|it| { + it.map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) + .collect::>() + }); + assert_eq!(result, [Some("one".to_string()), Some("four".to_string())]); } // Consistency tests use rstest::rstest; diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index e7b93bc79d6..ff3d9052b0b 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -95,7 +95,8 @@ mod tests { use crate::accessor::ArrayAccessor; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::compute::take::PrimitiveArray; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::compute::conformance::take::test_take_conformance; use crate::dtype::DType; use crate::dtype::Nullability::NonNullable; @@ -115,12 +116,12 @@ mod tests { let taken = arr.take(buffer![0, 3].into_array()).unwrap(); assert!(taken.dtype().is_nullable()); - assert_eq!( - taken.to_varbinview().with_iterator(|it| it - .map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) - .collect::>()), - [Some("one".to_string()), Some("four".to_string())] - ); + #[expect(deprecated)] + let result = taken.to_varbinview().with_iterator(|it| { + it.map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) + .collect::>() + }); + assert_eq!(result, [Some("one".to_string()), Some("four".to_string())]); } #[test] @@ -136,12 +137,12 @@ mod tests { let taken = arr.take(indices.into_array()).unwrap(); assert!(taken.dtype().is_nullable()); - assert_eq!( - taken.to_varbinview().with_iterator(|it| it - .map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) - .collect::>()), - [Some("two".to_string()), None] - ); + #[expect(deprecated)] + let result = taken.to_varbinview().with_iterator(|it| { + it.map(|v| v.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })) + .collect::>() + }); + assert_eq!(result, [Some("two".to_string()), None]); } #[rstest] diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index 57f8aab8067..2be832b500f 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -214,7 +214,8 @@ mod tests { use crate::accessor::ArrayAccessor; use crate::arrays::VarBinViewArray; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; @@ -246,6 +247,7 @@ mod tests { let mask = Mask::from_iter([true, false, true, false, false, true]); + #[expect(deprecated)] let zipped = mask .clone() .into_array() diff --git a/vortex-array/src/arrays/varbinview/tests.rs b/vortex-array/src/arrays/varbinview/tests.rs index ab19e33dd59..2892ff16283 100644 --- a/vortex-array/src/arrays/varbinview/tests.rs +++ b/vortex-array/src/arrays/varbinview/tests.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::BinaryView; use crate::assert_arrays_eq; @@ -31,6 +32,7 @@ pub fn slice_array() { #[test] pub fn flatten_array() { let binary_arr = VarBinViewArray::from_iter_str(["string1", "string2"]); + #[expect(deprecated)] let var_bin = binary_arr.as_array().to_varbinview(); assert_arrays_eq!( var_bin, diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index 5cf2f0fa9fe..0e784aca67d 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -20,7 +20,8 @@ use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -115,6 +116,7 @@ impl ArrayBuilder for BoolBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let bool_array = array.to_bool(); self.inner.append_buffer(&bool_array.to_bit_buffer()); @@ -168,7 +170,8 @@ mod tests { use crate::builders::BoolBuilder; use crate::builders::bool::BoolArray; use crate::builders::builder_with_capacity; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -202,7 +205,9 @@ mod tests { .clone() .append_to_builder(builder.as_mut(), &mut ctx)?; + #[expect(deprecated)] let canon_into = builder.finish().to_bool(); + #[expect(deprecated)] let into_canon = chunk.to_bool(); assert!( diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index d009fd451c9..90c077b083c 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -14,7 +14,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builders::ArrayBuilder; @@ -195,6 +196,7 @@ impl ArrayBuilder for DecimalBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let decimal_array = array.to_decimal(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { diff --git a/vortex-array/src/builders/dict/bytes.rs b/vortex-array/src/builders/dict/bytes.rs index 155d0b1db9a..d66870d6ba5 100644 --- a/vortex-array/src/builders/dict/bytes.rs +++ b/vortex-array/src/builders/dict/bytes.rs @@ -25,7 +25,8 @@ use crate::arrays::VarBin; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::build_views::BinaryView; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::PType; use crate::dtype::UnsignedPType; @@ -172,7 +173,9 @@ impl DictEncoder for BytesDictBuilder { } else { // NOTE(aduffy): it is very rare that this path would be taken, only e.g. // if we're performing dictionary encoding downstream of some other compression. - self.encode_bytes(&array.to_varbinview(), len) + #[expect(deprecated)] + let varbinview = array.to_varbinview(); + self.encode_bytes(&varbinview, len) } } @@ -204,7 +207,8 @@ mod test { use std::str; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::VarBinArray; use crate::arrays::dict::DictArraySlotsExt; @@ -214,11 +218,12 @@ mod test { fn encode_varbin() { let arr = VarBinArray::from(vec!["hello", "world", "hello", "again", "world"]); let dict = dict_encode(&arr.into_array()).unwrap(); - assert_eq!( - dict.codes().to_primitive().as_slice::(), - &[0, 1, 0, 2, 1] - ); - dict.values().to_varbinview().with_iterator(|iter| { + #[expect(deprecated)] + let codes = dict.codes().to_primitive(); + assert_eq!(codes.as_slice::(), &[0, 1, 0, 2, 1]); + #[expect(deprecated)] + let values = dict.values().to_varbinview(); + values.with_iterator(|iter| { assert_eq!( iter.flatten() .map(|b| unsafe { str::from_utf8_unchecked(b) }) @@ -243,11 +248,12 @@ mod test { .into_iter() .collect(); let dict = dict_encode(&arr.into_array()).unwrap(); - assert_eq!( - dict.codes().to_primitive().as_slice::(), - &[0, 1, 2, 0, 1, 3, 2, 1] - ); - dict.values().to_varbinview().with_iterator(|iter| { + #[expect(deprecated)] + let codes = dict.codes().to_primitive(); + assert_eq!(codes.as_slice::(), &[0, 1, 2, 0, 1, 3, 2, 1]); + #[expect(deprecated)] + let values = dict.values().to_varbinview(); + values.with_iterator(|iter| { assert_eq!( iter.map(|b| b.map(|v| unsafe { str::from_utf8_unchecked(v) })) .collect::>(), @@ -260,7 +266,9 @@ mod test { fn repeated_values() { let arr = VarBinArray::from(vec!["a", "a", "b", "b", "a", "b", "a", "b"]); let dict = dict_encode(&arr.into_array()).unwrap(); - dict.values().to_varbinview().with_iterator(|iter| { + #[expect(deprecated)] + let values = dict.values().to_varbinview(); + values.with_iterator(|iter| { assert_eq!( iter.flatten() .map(|b| unsafe { str::from_utf8_unchecked(b) }) @@ -268,9 +276,8 @@ mod test { vec!["a", "b"] ); }); - assert_eq!( - dict.codes().to_primitive().as_slice::(), - &[0, 0, 1, 1, 0, 1, 0, 1] - ); + #[expect(deprecated)] + let codes = dict.codes().to_primitive(); + assert_eq!(codes.as_slice::(), &[0, 0, 1, 1, 0, 1, 0, 1]); } } diff --git a/vortex-array/src/builders/dict/mod.rs b/vortex-array/src/builders/dict/mod.rs index aa24000ac20..7cfcfbbfb8a 100644 --- a/vortex-array/src/builders/dict/mod.rs +++ b/vortex-array/src/builders/dict/mod.rs @@ -9,7 +9,8 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::IntoArray; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::arrays::DictArray; use crate::arrays::Primitive; use crate::arrays::VarBin; @@ -66,7 +67,9 @@ pub fn dict_encode_with_constraints( constraints: &DictConstraints, ) -> VortexResult { let mut encoder = dict_encoder(array, constraints); - let codes = encoder.encode(array).to_primitive().narrow()?; + let encoded = encoder.encode(array); + #[expect(deprecated)] + let codes = encoded.to_primitive().narrow()?; // SAFETY: The encoding process will produce a value set of codes and values // All values in the dictionary are guaranteed to be referenced by at least one code // since we build the dictionary from the codes we observe during encoding diff --git a/vortex-array/src/builders/dict/primitive.rs b/vortex-array/src/builders/dict/primitive.rs index d4ae633c202..03d930dbef1 100644 --- a/vortex-array/src/builders/dict/primitive.rs +++ b/vortex-array/src/builders/dict/primitive.rs @@ -15,7 +15,8 @@ use super::DictConstraints; use super::DictEncoder; use crate::ArrayRef; use crate::IntoArray; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::accessor::ArrayAccessor; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; @@ -125,7 +126,9 @@ where fn encode(&mut self, array: &ArrayRef) -> ArrayRef { let mut codes = BufferMut::::with_capacity(array.len()); - array.to_primitive().with_iterator(|it| { + #[expect(deprecated)] + let prim = array.to_primitive(); + prim.with_iterator(|it| { for value in it { let Some(code) = self.encode_value(value.copied()) else { break; diff --git a/vortex-array/src/builders/extension.rs b/vortex-array/src/builders/extension.rs index ae540c1a6c2..d947877e297 100644 --- a/vortex-array/src/builders/extension.rs +++ b/vortex-array/src/builders/extension.rs @@ -15,7 +15,8 @@ use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; use crate::scalar::ExtScalar; @@ -99,6 +100,7 @@ impl ArrayBuilder for ExtensionBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let ext_array = array.to_extension(); self.storage.extend_from_array(ext_array.storage_array()) } diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index b561cb3ed5f..36024f04340 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -22,7 +22,8 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::scalar::ListScalar; @@ -237,6 +238,7 @@ impl ArrayBuilder for FixedSizeListBuilder { /// This will increase the capacity if extending with this `array` would go past the original /// capacity. unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let fsl = array.to_fixed_size_list(); if fsl.is_empty() { return; @@ -282,7 +284,8 @@ mod tests { use super::FixedSizeListBuilder; use crate::IntoArray as _; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -335,6 +338,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 2); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.elements().len(), 6); assert_eq!(fsl_array.list_size(), 3); @@ -358,6 +362,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 100); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 0); // The elements array should be empty since list_size is 0. @@ -387,6 +392,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 100); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); @@ -416,6 +422,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 5); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.elements().len(), 10); } @@ -429,6 +436,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 0); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 100000000); assert_eq!(fsl_array.elements().len(), 0); @@ -461,6 +469,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 3); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert!( fsl_array @@ -524,6 +533,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 2); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.elements().len(), 6); } @@ -538,11 +548,13 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 5); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 3); assert_eq!(fsl_array.elements().len(), 15); // Check that all elements are zeros. + #[expect(deprecated)] let elements_array = fsl_array.elements().to_primitive(); let elements = elements_array.as_slice::(); assert!(elements.iter().all(|&x| x == 0)); @@ -561,6 +573,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 3); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 2); @@ -591,6 +604,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 1); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 2); @@ -616,6 +630,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 1000); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); @@ -667,6 +682,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 6); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.elements().len(), 12); @@ -742,6 +758,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 5); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.list_size(), 0); assert_eq!(fsl_array.elements().len(), 0); @@ -854,6 +871,7 @@ mod tests { let fsl = builder.finish(); assert_eq!(fsl.len(), 6); + #[expect(deprecated)] let fsl_array = fsl.to_fixed_size_list(); assert_eq!(fsl_array.elements().len(), 12); @@ -1017,6 +1035,7 @@ mod tests { assert_eq!(fsl.list_size(), 3); // Verify elements array: [1, 2, 3, 10, 11, 12, 4, 5, 6, 20, 21, 22]. + #[expect(deprecated)] let elements = fsl.elements().to_primitive(); assert_eq!( elements.as_slice::(), diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 511f2120f1f..8e2c0e8283b 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -23,7 +23,8 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::PrimitiveBuilder; use crate::builders::builder_with_capacity; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::Nullability; @@ -217,6 +218,7 @@ impl ArrayBuilder for ListBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let list = array.to_listview(); if list.is_empty() { return; @@ -233,7 +235,9 @@ impl ArrayBuilder for ListBuilder { // Note that `ListViewArray` has `n` offsets and sizes, not `n+1` offsets like `ListArray`. let elements = list.elements(); + #[expect(deprecated)] let offsets = list.offsets().to_primitive(); + #[expect(deprecated)] let sizes = list.sizes().to_primitive(); fn extend_inner( @@ -305,7 +309,9 @@ impl ArrayBuilder for ListBuilder { } fn finish_into_canonical(&mut self) -> Canonical { - Canonical::List(self.finish_into_list().into_array().to_listview()) + #[expect(deprecated)] + let listview = self.finish_into_list().into_array().to_listview(); + Canonical::List(listview) } } @@ -320,7 +326,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::arrays::list::ListArrayExt; @@ -376,6 +383,7 @@ mod tests { let list = builder.finish(); assert_eq!(list.len(), 2); + #[expect(deprecated)] let list_array = list.to_listview(); assert_eq!(list_array.list_elements_at(0).unwrap().len(), 3); @@ -428,6 +436,7 @@ mod tests { let list = builder.finish(); assert_eq!(list.len(), 3); + #[expect(deprecated)] let list_array = list.to_listview(); assert_eq!(list_array.list_elements_at(0).unwrap().len(), 3); @@ -451,6 +460,7 @@ mod tests { builder.extend_from_array(&list.slice(0..0).unwrap()); builder.extend_from_array(&list.slice(1..3).unwrap()); + #[expect(deprecated)] let expected = ListArray::from_iter_opt_slow::( [ Some(vec![0, 1, 2]), @@ -524,6 +534,7 @@ mod tests { DType::List(Arc::new(DType::Primitive(I32, NonNullable)), NonNullable), ); + #[expect(deprecated)] let canon_values = chunked_list.unwrap().as_array().to_listview(); assert_eq!( diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index c2f5c1471ee..d8217729996 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -21,7 +21,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::ListViewArray; @@ -293,6 +294,7 @@ impl ArrayBuilder for ListViewBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let listview = array.to_listview(); if listview.is_empty() { return; @@ -341,6 +343,7 @@ impl ArrayBuilder for ListViewBuilder { let uninit_range = self.offsets_builder.uninit_range(extend_length); // This should be cheap because we didn't compress after rebuilding. + #[expect(deprecated)] let new_offsets = listview.offsets().to_primitive(); match_each_integer_ptype!(new_offsets.ptype(), |A| { diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index 93c50b130ca..8d422ac7d2c 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -19,7 +19,8 @@ use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -177,6 +178,7 @@ impl ArrayBuilder for PrimitiveBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let array = array.to_primitive(); // This should be checked in `extend_from_array` but we can check it again. diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index f37b88d7c80..b9832bba2a5 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -22,7 +22,8 @@ use crate::builders::DEFAULT_BUILDER_CAPACITY; use crate::builders::LazyBitBufferBuilder; use crate::builders::builder_with_capacity; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::StructFields; @@ -168,6 +169,7 @@ impl ArrayBuilder for StructBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let array = array.to_struct(); for (a, builder) in array diff --git a/vortex-array/src/builders/tests.rs b/vortex-array/src/builders/tests.rs index 749b5abdad2..a46f97f1966 100644 --- a/vortex-array/src/builders/tests.rs +++ b/vortex-array/src/builders/tests.rs @@ -237,6 +237,7 @@ where // Get canonical arrays using both methods. let canonical_direct = builder1.finish_into_canonical(); + #[expect(deprecated)] let canonical_indirect = builder2 .finish() .to_canonical() diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index c1e5fd6f6d6..c125e5086fe 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -28,7 +28,8 @@ use crate::arrays::varbinview::compact::BufferUtilization; use crate::builders::ArrayBuilder; use crate::builders::LazyBitBufferBuilder; use crate::canonical::Canonical; -use crate::canonical::ToCanonical; +#[expect(deprecated)] +use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::scalar::Scalar; @@ -294,6 +295,7 @@ impl ArrayBuilder for VarBinViewBuilder { } unsafe fn extend_from_array_unchecked(&mut self, array: &ArrayRef) { + #[expect(deprecated)] let array = array.to_varbinview(); self.flush_in_progress(); diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index ffae20f09c8..e8edd0488fc 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -425,94 +425,105 @@ impl IntoArray for Canonical { /// # Canonicalization /// /// This trait has a blanket implementation for all types implementing [ToCanonical]. +#[deprecated(note = "use `array.execute::(ctx)` instead")] pub trait ToCanonical { /// Canonicalize into a [`NullArray`] if the target is [`Null`](DType::Null) typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_null(&self) -> NullArray; /// Canonicalize into a [`BoolArray`] if the target is [`Bool`](DType::Bool) typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_bool(&self) -> BoolArray; /// Canonicalize into a [`PrimitiveArray`] if the target is [`Primitive`](DType::Primitive) /// typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_primitive(&self) -> PrimitiveArray; /// Canonicalize into a [`DecimalArray`] if the target is [`Decimal`](DType::Decimal) /// typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_decimal(&self) -> DecimalArray; /// Canonicalize into a [`StructArray`] if the target is [`Struct`](DType::Struct) typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_struct(&self) -> StructArray; /// Canonicalize into a [`ListViewArray`] if the target is [`List`](DType::List) typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_listview(&self) -> ListViewArray; /// Canonicalize into a [`FixedSizeListArray`] if the target is [`List`](DType::FixedSizeList) /// typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_fixed_size_list(&self) -> FixedSizeListArray; /// Canonicalize into a [`VarBinViewArray`] if the target is [`Utf8`](DType::Utf8) /// or [`Binary`](DType::Binary) typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_varbinview(&self) -> VarBinViewArray; /// Canonicalize into an [`ExtensionArray`] if the array is [`Extension`](DType::Extension) /// typed. + #[deprecated(note = "use `array.execute::(ctx)` instead")] fn to_extension(&self) -> ExtensionArray; } // Blanket impl for all Array encodings. +#[expect(deprecated)] impl ToCanonical for ArrayRef { fn to_null(&self) -> NullArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_null() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_null() } fn to_bool(&self) -> BoolArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_bool() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_bool() } fn to_primitive(&self) -> PrimitiveArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_primitive() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_primitive() } fn to_decimal(&self) -> DecimalArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_decimal() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_decimal() } fn to_struct(&self) -> StructArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_struct() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_struct() } fn to_listview(&self) -> ListViewArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_listview() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_listview() } fn to_fixed_size_list(&self) -> FixedSizeListArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_fixed_size_list() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_fixed_size_list() } fn to_varbinview(&self) -> VarBinViewArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_varbinview() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_varbinview() } fn to_extension(&self) -> ExtensionArray { - self.to_canonical() - .vortex_expect("to_canonical failed") - .into_extension() + #[expect(deprecated)] + let result = self.to_canonical().vortex_expect("to_canonical failed"); + result.into_extension() } } diff --git a/vortex-array/src/compute/conformance/binary_numeric.rs b/vortex-array/src/compute/conformance/binary_numeric.rs index 5f5588ec273..ce3eba940df 100644 --- a/vortex-array/src/compute/conformance/binary_numeric.rs +++ b/vortex-array/src/compute/conformance/binary_numeric.rs @@ -31,7 +31,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::RecursiveCanonical; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::builtins::ArrayBuiltins; @@ -92,6 +93,7 @@ fn test_standard_binary_numeric(array: ArrayRef) where Scalar: From, { + #[expect(deprecated)] let canonicalized_array = array.to_primitive(); let original_values = to_vec_of_scalar(&canonicalized_array.into_array()); @@ -328,6 +330,7 @@ where T: NativePType + Num + Copy + std::fmt::Debug, Scalar: From, { + #[expect(deprecated)] let canonicalized_array = array.to_primitive(); let original_values = to_vec_of_scalar(&canonicalized_array.into_array()); diff --git a/vortex-array/src/compute/conformance/consistency.rs b/vortex-array/src/compute/conformance/consistency.rs index 1817588c1f2..a6d2996b091 100644 --- a/vortex-array/src/compute/conformance/consistency.rs +++ b/vortex-array/src/compute/conformance/consistency.rs @@ -1029,6 +1029,7 @@ fn test_slice_aggregate_consistency(array: &ArrayRef) { let sliced = array .slice(start..end) .vortex_expect("slice should succeed in conformance test"); + #[expect(deprecated)] let canonical = array.to_canonical().vortex_expect("to_canonical failed"); let canonical_sliced = canonical .into_array() @@ -1130,6 +1131,7 @@ fn test_cast_slice_consistency(array: &ArrayRef) { let end = 7.min(len - 2).max(start + 1); // Ensure we have at least 1 element // Get canonical form of the original array + #[expect(deprecated)] let canonical = array.to_canonical().vortex_expect("to_canonical failed"); // Choose appropriate target dtype based on the array's type @@ -1254,10 +1256,10 @@ fn test_cast_slice_consistency(array: &ArrayRef) { .vortex_expect("slice should succeed in conformance test"); // Try to cast the sliced array (force execution via to_canonical) - let slice_then_cast = match sliced - .cast(target_dtype.clone()) - .and_then(|a| a.to_canonical().map(|c| c.into_array())) - { + let slice_then_cast = match sliced.cast(target_dtype.clone()).and_then(|a| { + #[expect(deprecated)] + a.to_canonical().map(|c| c.into_array()) + }) { Ok(result) => result, Err(_) => continue, // Skip if cast fails }; @@ -1306,11 +1308,10 @@ fn test_cast_slice_consistency(array: &ArrayRef) { } // Also test the other way: cast then slice - let casted = match array - .clone() - .cast(target_dtype.clone()) - .and_then(|a| a.to_canonical().map(|c| c.into_array())) - { + let casted = match array.clone().cast(target_dtype.clone()).and_then(|a| { + #[expect(deprecated)] + a.to_canonical().map(|c| c.into_array()) + }) { Ok(result) => result, Err(_) => continue, // Skip if cast fails }; diff --git a/vortex-array/src/compute/conformance/take.rs b/vortex-array/src/compute/conformance/take.rs index aa41da6942a..911a2ae7295 100644 --- a/vortex-array/src/compute/conformance/take.rs +++ b/vortex-array/src/compute/conformance/take.rs @@ -64,14 +64,15 @@ fn test_take_all(array: &ArrayRef) { assert_eq!(result.dtype(), array.dtype()); // Verify elements match - match ( - array - .to_canonical() - .vortex_expect("to_canonical failed on array"), - result - .to_canonical() - .vortex_expect("to_canonical failed on result"), - ) { + #[expect(deprecated)] + let array_canonical = array + .to_canonical() + .vortex_expect("to_canonical failed on array"); + #[expect(deprecated)] + let result_canonical = result + .to_canonical() + .vortex_expect("to_canonical failed on result"); + match (array_canonical, result_canonical) { (Canonical::Primitive(orig_prim), Canonical::Primitive(result_prim)) => { assert_eq!( orig_prim.buffer_handle().to_host_sync(), diff --git a/vortex-array/src/display/mod.rs b/vortex-array/src/display/mod.rs index 861d24e3538..8907667bd6e 100644 --- a/vortex-array/src/display/mod.rs +++ b/vortex-array/src/display/mod.rs @@ -579,7 +579,8 @@ impl ArrayRef { } #[cfg(feature = "table-display")] DisplayOptions::TableDisplay => { - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; let mut builder = tabled::builder::Builder::default(); @@ -600,6 +601,7 @@ impl ArrayRef { return write!(f, "{table}"); }; + #[expect(deprecated)] let struct_ = self.to_struct(); builder.push_record(sf.names().iter().map(|name| name.to_string())); diff --git a/vortex-array/src/normalize.rs b/vortex-array/src/normalize.rs index f4c1d8c5a8f..2833bf467d6 100644 --- a/vortex-array/src/normalize.rs +++ b/vortex-array/src/normalize.rs @@ -219,8 +219,10 @@ mod tests { assert_eq!(normalized.len(), 3); // Verify the data: codes [1,0,1] -> values [20, 10, 20] + #[expect(deprecated)] + let normalized_canonical = normalized.to_canonical()?; assert_arrays_eq!( - normalized.to_canonical()?, + normalized_canonical, PrimitiveArray::from_iter(vec![20i32, 10, 20]) ); diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 000ea495815..007ffa43a5d 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -22,7 +22,8 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; +#[expect(deprecated)] +use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builtins::ArrayBuiltins; @@ -407,6 +408,7 @@ impl Patches { /// with the insertion point if not found. fn search_index_binary_search(indices: &ArrayRef, needle: usize) -> VortexResult { if indices.is_canonical() { + #[expect(deprecated)] let primitive = indices.to_primitive(); match_each_integer_ptype!(primitive.ptype(), |T| { let Ok(needle) = T::try_from(needle) else { @@ -1115,7 +1117,8 @@ mod test { use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::assert_arrays_eq; use crate::patches::Patches; @@ -1165,7 +1168,9 @@ mod test { ) .unwrap() .unwrap(); + #[expect(deprecated)] let primitive_values = taken.values().to_primitive(); + #[expect(deprecated)] let primitive_indices = taken.indices().to_primitive(); assert_eq!(taken.array_len(), 2); assert_arrays_eq!( @@ -1207,6 +1212,7 @@ mod test { .unwrap() .unwrap(); + #[expect(deprecated)] let primitive_values = taken.values().to_primitive(); assert_eq!(taken.array_len(), 2); assert_arrays_eq!( @@ -1499,6 +1505,7 @@ mod test { assert_arrays_eq!(masked.indices(), PrimitiveArray::from_iter([5u64, 8])); // Values should be the null and 300 + #[expect(deprecated)] let masked_values = masked.values().to_primitive(); assert_eq!(masked_values.len(), 2); assert!( @@ -1674,6 +1681,7 @@ mod test { ) .unwrap(); + #[expect(deprecated)] let values = patches.values().to_primitive(); assert_eq!( i32::try_from( diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index 0cf591824e5..127985af93a 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -330,7 +330,8 @@ mod tests { use super::*; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::DecimalArray; @@ -387,6 +388,7 @@ mod tests { let array = buffer![1, 0, 1, 0, 1].into_array(); let upper = buffer![2, 1, 1, 0, 0].into_array(); + #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -416,6 +418,7 @@ mod tests { ) .into_array(); + #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -434,6 +437,7 @@ mod tests { // upper is a fixed constant let upper = ConstantArray::new(Scalar::from(2), 5).into_array(); + #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -452,6 +456,7 @@ mod tests { // lower is also a constant let lower = ConstantArray::new(Scalar::from(0), 5).into_array(); + #[expect(deprecated)] let matches = between_canonical( &array, &lower, diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/fns/binary/boolean.rs index b6894fd2887..10d37202f64 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/fns/binary/boolean.rs @@ -110,7 +110,8 @@ mod tests { use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::builtins::ArrayBuiltins; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::scalar_fn::fns::operators::Operator; #[rstest] @@ -124,6 +125,7 @@ mod tests { )] fn test_or(#[case] lhs: ArrayRef, #[case] rhs: ArrayRef) { let r = lhs.binary(rhs, Operator::Or).unwrap(); + #[expect(deprecated)] let r = r.to_bool().into_array(); let v0 = r @@ -163,6 +165,7 @@ mod tests { BoolArray::from_iter([Some(true), Some(true), Some(false), Some(false)]).into_array(), )] fn test_and(#[case] lhs: ArrayRef, #[case] rhs: ArrayRef) { + #[expect(deprecated)] let r = lhs .binary(rhs, Operator::And) .unwrap() diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 6447e0c1638..2b9d971006d 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -250,7 +250,8 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; use crate::LEGACY_SESSION; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::ListArray; @@ -286,6 +287,7 @@ mod tests { Validity::from_iter([false, true, true, true, true]), ); + #[expect(deprecated)] let matches = arr .clone() .into_array() @@ -294,6 +296,7 @@ mod tests { .to_bool(); assert_eq!(to_int_indices(matches).unwrap(), [1u64, 2, 3, 4]); + #[expect(deprecated)] let matches = arr .clone() .into_array() @@ -308,6 +311,7 @@ mod tests { Validity::from_iter([false, true, true, true, true]), ); + #[expect(deprecated)] let matches = arr .clone() .into_array() @@ -316,6 +320,7 @@ mod tests { .to_bool(); assert_eq!(to_int_indices(matches).unwrap(), [2u64, 3, 4]); + #[expect(deprecated)] let matches = arr .clone() .into_array() @@ -324,6 +329,7 @@ mod tests { .to_bool(); assert_eq!(to_int_indices(matches).unwrap(), [4u64]); + #[expect(deprecated)] let matches = other .clone() .into_array() @@ -332,6 +338,7 @@ mod tests { .to_bool(); assert_eq!(to_int_indices(matches).unwrap(), [2u64, 3, 4]); + #[expect(deprecated)] let matches = other .into_array() .binary(arr.into_array(), Operator::Gt) diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 27d60663c1d..2a79a0c6af2 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -461,7 +461,8 @@ mod tests { use crate::arrays::ListArray; use crate::arrays::VarBinArray; use crate::assert_arrays_eq; - use crate::canonical::ToCanonical; + #[expect(deprecated)] + use crate::canonical::ToCanonical as _; use crate::dtype::DType; use crate::dtype::Field; use crate::dtype::FieldPath; @@ -711,10 +712,15 @@ mod tests { // -- Tests migrated from compute/list_contains.rs -- fn nonnull_strings(values: Vec>) -> ArrayRef { - ListArray::from_iter_slow::(values, Arc::new(DType::Utf8(Nullability::NonNullable))) - .unwrap() - .to_listview() - .into_array() + #[expect(deprecated)] + let result = ListArray::from_iter_slow::( + values, + Arc::new(DType::Utf8(Nullability::NonNullable)), + ) + .unwrap() + .to_listview() + .into_array(); + result } fn null_strings(values: Vec>>) -> ArrayRef { @@ -733,11 +739,13 @@ mod tests { let elements = VarBinArray::from_iter(elements, DType::Utf8(Nullability::Nullable)).into_array(); - ListArray::try_new(elements, offsets, Validity::NonNullable) + #[expect(deprecated)] + let result = ListArray::try_new(elements, offsets, Validity::NonNullable) .unwrap() .as_array() .to_listview() - .into_array() + .into_array(); + result } fn bool_array(values: Vec, validity: Validity) -> BoolArray { diff --git a/vortex-array/src/scalar_fn/fns/merge.rs b/vortex-array/src/scalar_fn/fns/merge.rs index 01445eb6c54..0c79ef86731 100644 --- a/vortex-array/src/scalar_fn/fns/merge.rs +++ b/vortex-array/src/scalar_fn/fns/merge.rs @@ -291,7 +291,8 @@ mod tests { use crate::ArrayRef; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::PrimitiveArray; use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; @@ -317,11 +318,16 @@ mod tests { vortex_bail!("empty field path"); }; + #[expect(deprecated)] let mut array = array.to_struct().unmasked_field_by_name(field)?.clone(); for field in field_path { - array = array.to_struct().unmasked_field_by_name(field)?.clone(); + #[expect(deprecated)] + let next = array.to_struct().unmasked_field_by_name(field)?.clone(); + array = next; } - Ok(array.to_primitive()) + #[expect(deprecated)] + let result = array.to_primitive(); + Ok(result) } #[test] @@ -491,13 +497,16 @@ mod tests { ]) .unwrap() .into_array(); + #[expect(deprecated)] let actual_array = test_array.apply(&expr).unwrap().to_struct(); + #[expect(deprecated)] + let inner_struct = actual_array + .unmasked_field_by_name("a") + .unwrap() + .to_struct(); assert_eq!( - actual_array - .unmasked_field_by_name("a") - .unwrap() - .to_struct() + inner_struct .names() .iter() .map(|name| name.as_ref()) @@ -532,6 +541,7 @@ mod tests { ]) .unwrap() .into_array(); + #[expect(deprecated)] let actual_array = test_array.apply(&expr).unwrap().to_struct(); assert_eq!(actual_array.names(), ["a", "c", "b", "d"]); diff --git a/vortex-array/src/scalar_fn/fns/not/mod.rs b/vortex-array/src/scalar_fn/fns/not/mod.rs index bdf7fdd9e76..66953709640 100644 --- a/vortex-array/src/scalar_fn/fns/not/mod.rs +++ b/vortex-array/src/scalar_fn/fns/not/mod.rs @@ -122,7 +122,8 @@ impl ScalarFnVTable for Not { #[cfg(test)] mod tests { use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::bool::BoolArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; @@ -137,15 +138,10 @@ mod tests { fn invert_booleans() { let not_expr = not(root()); let bools = BoolArray::from_iter([false, true, false, false, true, true]); + #[expect(deprecated)] + let result = bools.into_array().apply(¬_expr).unwrap().to_bool(); assert_eq!( - bools - .into_array() - .apply(¬_expr) - .unwrap() - .to_bool() - .to_bit_buffer() - .iter() - .collect::>(), + result.to_bit_buffer().iter().collect::>(), vec![true, false, true, true, false, false] ); } diff --git a/vortex-array/src/scalar_fn/fns/pack.rs b/vortex-array/src/scalar_fn/fns/pack.rs index 13626bb107b..0a24f721064 100644 --- a/vortex-array/src/scalar_fn/fns/pack.rs +++ b/vortex-array/src/scalar_fn/fns/pack.rs @@ -168,7 +168,8 @@ mod tests { use super::PackOptions; use crate::ArrayRef; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::PrimitiveArray; use crate::arrays::struct_::StructArrayExt; use crate::assert_arrays_eq; @@ -195,11 +196,16 @@ mod tests { vortex_bail!("empty field path"); }; + #[expect(deprecated)] let mut array = array.to_struct().unmasked_field_by_name(field)?.clone(); for field in field_path { - array = array.to_struct().unmasked_field_by_name(field)?.clone(); + #[expect(deprecated)] + let next = array.to_struct().unmasked_field_by_name(field)?.clone(); + array = next; } - Ok(array.to_primitive()) + #[expect(deprecated)] + let result = array.to_primitive(); + Ok(result) } #[test] @@ -215,7 +221,9 @@ mod tests { let test_array = test_array(); let actual_array = test_array.clone().apply(&expr).unwrap(); assert_eq!(actual_array.len(), test_array.len()); - assert_eq!(actual_array.to_struct().struct_fields().nfields(), 0); + #[expect(deprecated)] + let nfields = actual_array.to_struct().struct_fields().nfields(); + assert_eq!(nfields, 0); } #[test] @@ -228,6 +236,7 @@ mod tests { [col("a"), col("b"), col("a")], ); + #[expect(deprecated)] let actual_array = test_array().apply(&expr).unwrap().to_struct(); assert_eq!(actual_array.names(), ["one", "two", "three"]); @@ -267,6 +276,7 @@ mod tests { ], ); + #[expect(deprecated)] let actual_array = test_array().apply(&expr).unwrap().to_struct(); assert_eq!(actual_array.names(), ["one", "two", "three"]); @@ -299,6 +309,7 @@ mod tests { [col("a"), col("b"), col("a")], ); + #[expect(deprecated)] let actual_array = test_array().apply(&expr).unwrap().to_struct(); assert_eq!(actual_array.names(), ["one", "two", "three"]); diff --git a/vortex-array/src/scalar_fn/fns/select.rs b/vortex-array/src/scalar_fn/fns/select.rs index c08f88f23cb..2ca070c2bc4 100644 --- a/vortex-array/src/scalar_fn/fns/select.rs +++ b/vortex-array/src/scalar_fn/fns/select.rs @@ -307,7 +307,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; - use crate::ToCanonical; + #[expect(deprecated)] + use crate::ToCanonical as _; use crate::arrays::struct_::StructArrayExt; use crate::dtype::DType; use crate::dtype::FieldName; @@ -335,6 +336,7 @@ mod tests { pub fn include_columns() { let st = test_array(); let select = select(vec![FieldName::from("a")], root()); + #[expect(deprecated)] let selected = st.into_array().apply(&select).unwrap().to_struct(); let selected_names = selected.names().clone(); assert_eq!(selected_names.as_ref(), &["a"]); @@ -344,6 +346,7 @@ mod tests { pub fn exclude_columns() { let st = test_array(); let select = select_exclude(vec![FieldName::from("a")], root()); + #[expect(deprecated)] let selected = st.into_array().apply(&select).unwrap().to_struct(); let selected_names = selected.names().clone(); assert_eq!(selected_names.as_ref(), &["b"]); diff --git a/vortex-bench/src/datasets/tpch_l_comment.rs b/vortex-bench/src/datasets/tpch_l_comment.rs index 00a141f03ba..3488ef92979 100644 --- a/vortex-bench/src/datasets/tpch_l_comment.rs +++ b/vortex-bench/src/datasets/tpch_l_comment.rs @@ -9,6 +9,7 @@ use futures::TryStreamExt; use glob::glob; use vortex::array::ArrayRef; use vortex::array::IntoArray; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::ChunkedArray; use vortex::dtype::Nullability::NonNullable; @@ -76,7 +77,11 @@ impl Dataset for TPCHLCommentChunked { let file_chunks: Vec<_> = file .scan()? .with_projection(pack(vec![("l_comment", col("l_comment"))], NonNullable)) - .map(|a| Ok(a.to_canonical()?.into_array())) + .map(|a| { + #[expect(deprecated)] + let canonical = a.to_canonical()?; + Ok(canonical.into_array()) + }) .into_array_stream()? .try_collect() .await?; @@ -100,11 +105,9 @@ impl Dataset for TPCHLCommentCanonical { } async fn to_vortex_array(&self) -> Result { - let comments_canonical = TPCHLCommentChunked - .to_vortex_array() - .await? - .to_struct() - .into_array(); + let comments_chunked = TPCHLCommentChunked.to_vortex_array().await?; + #[expect(deprecated)] + let comments_canonical = comments_chunked.to_struct().into_array(); Ok(ChunkedArray::from_iter([comments_canonical]).into_array()) } diff --git a/vortex-btrblocks/benches/compress.rs b/vortex-btrblocks/benches/compress.rs index 8b6832ea630..a82dafacf3b 100644 --- a/vortex-btrblocks/benches/compress.rs +++ b/vortex-btrblocks/benches/compress.rs @@ -13,6 +13,7 @@ mod benchmarks { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; @@ -39,6 +40,7 @@ mod benchmarks { #[divan::bench] fn btrblocks(bencher: Bencher) { + #[expect(deprecated)] let array = make_clickbench_window_name().to_primitive(); let compressor = BtrBlocksCompressor::default(); bencher diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index 3b9def06633..a801b9bfd9b 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -6,6 +6,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::decimal::narrowed_decimal; @@ -59,6 +60,7 @@ impl Scheme for DecimalScheme { ) -> VortexResult { // TODO(joe): add support splitting i128/256 buffers into chunks of primitive values // for compression. 2 for i128 and 4 for i256. + #[expect(deprecated)] let decimal = data.array().clone().to_decimal(); let decimal = narrowed_decimal(decimal); let validity = decimal.validity()?; diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index f6d2389f08b..9e7e56fd7a5 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -14,6 +14,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; @@ -253,6 +254,7 @@ impl Scheme for NullDominatedSparseScheme { let sparse_encoded = Sparse::encode(data.array(), None)?; if let Some(sparse) = sparse_encoded.as_opt::() { + #[expect(deprecated)] let indices = sparse.patches().indices().to_primitive().narrow()?; let compressed_indices = compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index cb765fbec5a..cde4f7c7556 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -7,6 +7,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; @@ -186,8 +187,10 @@ impl Scheme for FoRScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { + #[expect(deprecated)] let primitive = data.array().to_primitive(); let for_array = FoR::encode(primitive)?; + #[expect(deprecated)] let biased = for_array.encoded().to_primitive(); // Immediately bitpack. If any other scheme was preferable, it would be chosen instead @@ -290,6 +293,7 @@ impl Scheme for ZigZagScheme { ) -> VortexResult { // Zigzag encode the values, then recursively compress the inner values. let zag = zigzag_encode(data.array_as_primitive())?; + #[expect(deprecated)] let encoded = zag.encoded().to_primitive(); let compressed = compressor.compress_child(&encoded.into_array(), &ctx, self.id(), 0)?; @@ -521,13 +525,16 @@ impl Scheme for SparseScheme { )?; if let Some(sparse) = sparse_encoded.as_opt::() { + #[expect(deprecated)] + let sparse_values_primitive = sparse.patches().values().to_primitive(); let compressed_values = compressor.compress_child( - &sparse.patches().values().to_primitive().into_array(), + &sparse_values_primitive.into_array(), &ctx, self.id(), 0, )?; + #[expect(deprecated)] let indices = sparse.patches().indices().to_primitive().narrow()?; let compressed_indices = @@ -624,8 +631,10 @@ impl Scheme for RunEndScheme { // Run-end encode the ends. let (ends, values) = runend_encode(data.array_as_primitive()); + #[expect(deprecated)] + let values_primitive = values.to_primitive(); let compressed_values = - compressor.compress_child(&values.to_primitive().into_array(), &ctx, self.id(), 0)?; + compressor.compress_child(&values_primitive.into_array(), &ctx, self.id(), 0)?; let compressed_ends = compressor.compress_child(&ends.into_array(), &ctx, self.id(), 1)?; @@ -776,41 +785,36 @@ pub(crate) fn rle_compress( ) -> VortexResult { let rle_array = RLE::encode(data.array_as_primitive())?; - let compressed_values = compressor.compress_child( - &rle_array.values().to_primitive().into_array(), - &ctx, - scheme.id(), - 0, - )?; + #[expect(deprecated)] + let rle_values_primitive = rle_array.values().to_primitive(); + let compressed_values = + compressor.compress_child(&rle_values_primitive.into_array(), &ctx, scheme.id(), 0)?; // Delta is an unstable encoding, once we deem it stable we can switch over to this always. #[cfg(feature = "unstable_encodings")] - let compressed_indices = try_compress_delta( - compressor, - &rle_array.indices().to_primitive().narrow()?.into_array(), - &ctx, - scheme.id(), - 1, - )?; + let compressed_indices = { + #[expect(deprecated)] + let rle_indices_primitive = rle_array.indices().to_primitive().narrow()?; + try_compress_delta( + compressor, + &rle_indices_primitive.into_array(), + &ctx, + scheme.id(), + 1, + )? + }; #[cfg(not(feature = "unstable_encodings"))] - let compressed_indices = compressor.compress_child( - &rle_array.indices().to_primitive().narrow()?.into_array(), - &ctx, - scheme.id(), - 1, - )?; - - let compressed_offsets = compressor.compress_child( - &rle_array - .values_idx_offsets() - .to_primitive() - .narrow()? - .into_array(), - &ctx, - scheme.id(), - 2, - )?; + let compressed_indices = { + #[expect(deprecated)] + let rle_indices_primitive = rle_array.indices().to_primitive().narrow()?; + compressor.compress_child(&rle_indices_primitive.into_array(), &ctx, scheme.id(), 1)? + }; + + #[expect(deprecated)] + let rle_offsets_primitive = rle_array.values_idx_offsets().to_primitive().narrow()?; + let compressed_offsets = + compressor.compress_child(&rle_offsets_primitive.into_array(), &ctx, scheme.id(), 2)?; // SAFETY: Recursive compression doesn't affect the invariants. unsafe { @@ -833,8 +837,10 @@ fn try_compress_delta( parent_id: SchemeId, child_index: usize, ) -> VortexResult { + #[expect(deprecated)] + let child_primitive = child.to_primitive(); let (bases, deltas) = - vortex_fastlanes::delta_compress(&child.to_primitive(), &mut compressor.execution_ctx())?; + vortex_fastlanes::delta_compress(&child_primitive, &mut compressor.execution_ctx())?; let compressed_bases = compressor.compress_child(&bases.into_array(), parent_ctx, parent_id, child_index)?; diff --git a/vortex-btrblocks/src/schemes/patches.rs b/vortex-btrblocks/src/schemes/patches.rs index f094b92895e..3513ea5ad06 100644 --- a/vortex-btrblocks/src/schemes/patches.rs +++ b/vortex-btrblocks/src/schemes/patches.rs @@ -4,6 +4,7 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; @@ -15,6 +16,7 @@ use vortex_error::VortexResult; /// Compresses the given patches by downscaling integers and checking for constant values. pub fn compress_patches(patches: Patches) -> VortexResult { // Downscale the patch indices. + #[expect(deprecated)] let indices = patches.indices().to_primitive().narrow()?.into_array(); // Check if the values are constant. @@ -32,7 +34,11 @@ pub fn compress_patches(patches: Patches) -> VortexResult { let chunk_offsets = patches .chunk_offsets() .as_ref() - .map(|offsets| Ok::(offsets.to_primitive().narrow()?.into_array())) + .map(|offsets| { + #[expect(deprecated)] + let offsets_primitive = offsets.to_primitive().narrow()?.into_array(); + Ok::(offsets_primitive) + }) .transpose()?; Patches::new( diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 5b21d6c1d59..60bee2cf099 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -6,6 +6,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -88,23 +89,19 @@ impl Scheme for FSSTScheme { let compressor_fsst = fsst_train_compressor(&utf8); let fsst = fsst_compress(&utf8, utf8.len(), utf8.dtype(), &compressor_fsst); + #[expect(deprecated)] + let uncompressed_lengths_primitive = fsst.uncompressed_lengths().to_primitive().narrow()?; let compressed_original_lengths = compressor.compress_child( - &fsst - .uncompressed_lengths() - .to_primitive() - .narrow()? - .into_array(), + &uncompressed_lengths_primitive.into_array(), &ctx, self.id(), 0, )?; - let compressed_codes_offsets = compressor.compress_child( - &fsst.codes().offsets().to_primitive().narrow()?.into_array(), - &ctx, - self.id(), - 1, - )?; + #[expect(deprecated)] + let codes_offsets_primitive = fsst.codes().offsets().to_primitive().narrow()?; + let compressed_codes_offsets = + compressor.compress_child(&codes_offsets_primitive.into_array(), &ctx, self.id(), 1)?; let compressed_codes = VarBinArray::try_new( compressed_codes_offsets, fsst.codes().bytes().clone(), @@ -186,6 +183,7 @@ impl Scheme for NullDominatedSparseScheme { if let Some(sparse) = sparse_encoded.as_opt::() { // Compress the indices only (not the values for strings). + #[expect(deprecated)] let indices = sparse.patches().indices().to_primitive().narrow()?; let compressed_indices = compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index e14de1aa9c0..c0ab1f3881c 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -7,6 +7,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; @@ -75,6 +76,7 @@ impl Scheme for TemporalScheme { ctx: CompressorContext, ) -> VortexResult { let array = data.array().clone(); + #[expect(deprecated)] let ext_array = array.to_extension(); let temporal_array = TemporalArray::try_from(ext_array.clone().into_array())?; @@ -99,24 +101,17 @@ impl Scheme for TemporalScheme { subseconds, } = split_temporal(temporal_array)?; - let days = compressor.compress_child( - &days.to_primitive().narrow()?.into_array(), - &ctx, - self.id(), - 0, - )?; - let seconds = compressor.compress_child( - &seconds.to_primitive().narrow()?.into_array(), - &ctx, - self.id(), - 1, - )?; - let subseconds = compressor.compress_child( - &subseconds.to_primitive().narrow()?.into_array(), - &ctx, - self.id(), - 2, - )?; + #[expect(deprecated)] + let days_primitive = days.to_primitive().narrow()?; + let days = compressor.compress_child(&days_primitive.into_array(), &ctx, self.id(), 0)?; + #[expect(deprecated)] + let seconds_primitive = seconds.to_primitive().narrow()?; + let seconds = + compressor.compress_child(&seconds_primitive.into_array(), &ctx, self.id(), 1)?; + #[expect(deprecated)] + let subseconds_primitive = subseconds.to_primitive().narrow()?; + let subseconds = + compressor.compress_child(&subseconds_primitive.into_array(), &ctx, self.id(), 2)?; Ok(DateTimeParts::try_new(dtype, days, seconds, subseconds)?.into_array()) } diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 18d5c216aa9..a18926961e1 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -242,6 +242,7 @@ impl_encode!(f64, u64); #[cfg(test)] mod tests { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -276,6 +277,7 @@ mod tests { Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); + #[expect(deprecated)] let undict = dict_array.as_array().to_primitive().into_array(); assert_arrays_eq!(undict, expected); } diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 0693612aaee..135dfd8080c 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -249,6 +249,7 @@ impl_encode!(i64); #[cfg(test)] mod tests { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -282,6 +283,7 @@ mod tests { Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); + #[expect(deprecated)] let undict = dict_array.as_array().to_primitive().into_array(); assert_arrays_eq!(undict, expected); } diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index db0803a6ee8..e23353f43df 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -13,6 +13,7 @@ use vortex_array::CanonicalValidity; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; @@ -476,10 +477,10 @@ impl CascadingCompressor { // Record the root scheme with the offsets child index so root exclusion rules apply. let offset_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); - let compressed_offsets = self.compress_canonical( - Canonical::Primitive(list_array.offsets().to_primitive().narrow()?), - offset_ctx, - )?; + #[expect(deprecated)] + let list_offsets_primitive = list_array.offsets().to_primitive().narrow()?; + let compressed_offsets = + self.compress_canonical(Canonical::Primitive(list_offsets_primitive), offset_ctx)?; Ok( ListArray::try_new(compressed_elems, compressed_offsets, list_array.validity()?)? @@ -499,16 +500,18 @@ impl CascadingCompressor { let offset_ctx = ctx .clone() .descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); + #[expect(deprecated)] + let list_view_offsets_primitive = list_view.offsets().to_primitive().narrow()?; let compressed_offsets = self.compress_canonical( - Canonical::Primitive(list_view.offsets().to_primitive().narrow()?), + Canonical::Primitive(list_view_offsets_primitive), offset_ctx, )?; let sizes_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::SIZES); - let compressed_sizes = self.compress_canonical( - Canonical::Primitive(list_view.sizes().to_primitive().narrow()?), - sizes_ctx, - )?; + #[expect(deprecated)] + let list_view_sizes_primitive = list_view.sizes().to_primitive().narrow()?; + let compressed_sizes = + self.compress_canonical(Canonical::Primitive(list_view_sizes_primitive), sizes_ctx)?; Ok(ListViewArray::try_new( compressed_elems, diff --git a/vortex-compressor/src/stats/cache.rs b/vortex-compressor/src/stats/cache.rs index 30860b93bbe..28d7d0bc390 100644 --- a/vortex-compressor/src/stats/cache.rs +++ b/vortex-compressor/src/stats/cache.rs @@ -8,6 +8,7 @@ use std::any::TypeId; use vortex_array::ArrayRef; use vortex_array::ArrayView; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinView; @@ -141,7 +142,9 @@ impl ArrayAndStats { let array = self.array.clone(); self.cache.get_or_insert_with::(|| { - BoolStats::generate(&array.to_bool()).vortex_expect("BoolStats shouldn't fail") + #[expect(deprecated)] + let bool_array = array.to_bool(); + BoolStats::generate(&bool_array).vortex_expect("BoolStats shouldn't fail") }) } @@ -153,7 +156,9 @@ impl ArrayAndStats { let opts = self.opts; self.cache.get_or_insert_with::(|| { - IntegerStats::generate_opts(&array.to_primitive(), opts) + #[expect(deprecated)] + let primitive = array.to_primitive(); + IntegerStats::generate_opts(&primitive, opts) }) } @@ -163,7 +168,9 @@ impl ArrayAndStats { let opts = self.opts; self.cache.get_or_insert_with::(|| { - FloatStats::generate_opts(&array.to_primitive(), opts) + #[expect(deprecated)] + let primitive = array.to_primitive(); + FloatStats::generate_opts(&primitive, opts) }) } @@ -173,7 +180,9 @@ impl ArrayAndStats { let opts = self.opts; self.cache.get_or_insert_with::(|| { - StringStats::generate_opts(&array.to_varbinview(), opts) + #[expect(deprecated)] + let varbinview = array.to_varbinview(); + StringStats::generate_opts(&varbinview, opts) }) } diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 7a37c1b7c34..74b1a9828b3 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -272,6 +272,7 @@ where #[cfg(test)] mod tests { use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; @@ -282,6 +283,7 @@ mod tests { #[test] fn test_float_stats() { let floats = buffer![0.0f32, 1.0f32, 2.0f32].into_array(); + #[expect(deprecated)] let floats = floats.to_primitive(); let stats = FloatStats::generate_opts( diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 917fd1b7cd4..c814c03f3fe 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -19,6 +19,7 @@ use cudarc::driver::PushKernelArg; use cudarc::driver::sys::CUevent_flags; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; @@ -382,6 +383,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { ) .vortex_expect("alp_encode"); assert!(alp.patches().is_none()); + #[expect(deprecated)] let for_arr = FoRData::encode(alp.encoded().to_primitive()).vortex_expect("for encode"); let bp = BitPackedData::encode(for_arr.encoded(), bit_width).vortex_expect("bitpack encode"); diff --git a/vortex-cuda/gpu-scan-cli/src/main.rs b/vortex-cuda/gpu-scan-cli/src/main.rs index 4c343742000..b58b8248894 100644 --- a/vortex-cuda/gpu-scan-cli/src/main.rs +++ b/vortex-cuda/gpu-scan-cli/src/main.rs @@ -18,7 +18,8 @@ use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use vortex::VortexSessionDefault; -use vortex::array::ToCanonical; +#[expect(deprecated)] +use vortex::array::ToCanonical as _; use vortex::array::arrays::Dict; use vortex::array::arrays::struct_::StructArrayExt; use vortex::buffer::ByteBufferMut; @@ -178,6 +179,7 @@ async fn cmd_scan(path: PathBuf, gpu_file: bool, json_output: bool) -> VortexRes let mut chunk = 0; while let Some(next) = batches.next().await.transpose()? { + #[expect(deprecated)] let record = next.to_struct(); for (field, field_name) in record diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 660c2872259..3baf6fce9e3 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use futures::future::BoxFuture; use vortex::array::ArrayRef; use vortex::array::Canonical; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolDataParts; @@ -116,6 +117,7 @@ fn export_canonical( vortex_bail!("only support temporal extension types currently"); } + #[expect(deprecated)] let values = extension.storage_array().to_primitive(); let len = extension.len(); diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index a0f93feb23f..c0dda0c1fd5 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -500,6 +500,7 @@ mod tests { use cudarc::driver::PushKernelArg; use rstest::rstest; use vortex::array::IntoArray; + #[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; @@ -886,6 +887,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), )?; assert!(alp.patches().is_none()); + #[expect(deprecated)] let for_arr = FoR::encode(alp.encoded().to_primitive())?; let bp = BitPacked::encode(for_arr.encoded(), 6)?; diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 2b1db3dd0a1..e8fafa3087f 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -103,6 +103,7 @@ mod tests { use cudarc::driver::DeviceRepr; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; + #[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; @@ -199,10 +200,12 @@ mod tests { } fn force_cast(array: PrimitiveArray) -> PrimitiveArray { - array + #[expect(deprecated)] + let result = array .into_array() .cast(DType::Primitive(T::PTYPE, Nullability::NonNullable)) .unwrap() - .to_primitive() + .to_primitive(); + result } } diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index c63687037a4..70c6b60128d 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -376,6 +376,7 @@ mod tests { use std::ffi::CString; use vortex::array::LEGACY_SESSION; + #[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; @@ -429,6 +430,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -452,6 +454,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -475,6 +478,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -503,6 +507,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -545,6 +550,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -568,6 +574,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); + #[expect(deprecated)] let vortex_values = vortex_array.temporal_values().to_primitive(); let values_slice = vortex_values.as_slice::(); @@ -590,6 +597,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_bool(); let expected = BoolArray::new(BitBuffer::from(values), Validity::AllValid); assert_arrays_eq!(vortex_array, expected); @@ -616,6 +624,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_primitive(); let vortex_slice = vortex_array.as_slice::(); @@ -658,6 +667,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_listview(); assert_eq!(vortex_array.len(), len); @@ -686,6 +696,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_fixed_size_list(); assert_eq!(vortex_array.len(), len); @@ -704,6 +715,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_struct(); assert_eq!(vortex_array.len(), len); @@ -739,6 +751,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_struct(); assert_eq!(vortex_array.len(), len); @@ -788,6 +801,7 @@ mod tests { // Test conversion - the old bug would compute child length as 0+0=0 instead of // max(4,0)=4. let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_listview(); assert_eq!(vortex_array.len(), len); @@ -841,6 +855,7 @@ mod tests { // Test conversion - the old bug would compute child length as 0+2=2 instead of // max(4,2)=4. let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_listview(); assert_eq!(vortex_array.len(), len); @@ -896,6 +911,7 @@ mod tests { // Test conversion. The old code would compute child_min_length as 9999+9999=19998, which // would panic when trying to read that much data from the child vector. let result = flat_vector_to_vortex(&vector, len).unwrap(); + #[expect(deprecated)] let vortex_array = result.to_listview(); assert_eq!(vortex_array.len(), len); @@ -911,7 +927,9 @@ mod tests { ); // Verify the null entry has sanitized offset/size (offset=2, size=0) rather than garbage. + #[expect(deprecated)] let offsets = vortex_array.offsets().to_primitive(); + #[expect(deprecated)] let sizes = vortex_array.sizes().to_primitive(); assert_eq!(offsets.as_slice::()[1], 2); // Previous end (0+2). assert_eq!(sizes.as_slice::()[1], 0); diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index d576a85c148..ff30dddc6c1 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -77,6 +77,7 @@ pub(crate) fn new_exporter_with_flatten( let canonical = match canonical { Some(c) => c, None => { + #[expect(deprecated)] let canonical = values.to_canonical()?; cache .canonical_cache diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 1eeca276eac..d2a7923e5ad 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -11,6 +11,7 @@ use paste::paste; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::NullArray; @@ -204,6 +205,7 @@ pub unsafe extern "C-unwind" fn vx_array_get_field( try_or_default(error_out, || { let array = vx_array::as_ref(array); + #[expect(deprecated)] let field_array = array .to_struct() .unmasked_fields() @@ -770,7 +772,9 @@ mod tests { assert!(!res.is_null()); { let res = vx_array::as_ref(res); - let buffer = res.to_bool().to_bit_buffer(); + #[expect(deprecated)] + let bool_array = res.to_bool(); + let buffer = bool_array.to_bit_buffer(); let expected = BoolArray::from_iter(vec![false, false, true, true]); assert_eq!(buffer, expected.to_bit_buffer()); } diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 00a9df35675..37f9b602cae 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -290,6 +290,7 @@ mod tests { use std::sync::Arc; use vortex::array::IntoArray; + #[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::BoolArray; use vortex::array::arrays::ListArray; @@ -357,7 +358,9 @@ mod tests { { let applied_array = vx_array::as_ref(applied_array); let expected: Buffer = ages_array.to_buffer(); - assert_eq!(applied_array.to_primitive().to_buffer(), expected); + #[expect(deprecated)] + let prim = applied_array.to_primitive(); + assert_eq!(prim.to_buffer(), expected); } vx_array_free(applied_array); @@ -461,6 +464,7 @@ mod tests { assert!(error.is_null()); assert!(!applied_array.is_null()); { + #[expect(deprecated)] let array = vx_array::as_ref(applied_array).to_bool(); let expected = BoolArray::from_iter([false, false, true, false]); assert_eq!(array.to_bit_buffer(), expected.to_bit_buffer()); @@ -474,6 +478,7 @@ mod tests { assert!(error.is_null()); assert!(!applied_array.is_null()); { + #[expect(deprecated)] let array = vx_array::as_ref(applied_array).to_bool(); let expected = BoolArray::from_iter([true, true, true, false]); assert_eq!(array.to_bit_buffer(), expected.to_bit_buffer()); @@ -528,6 +533,7 @@ mod tests { assert!(error.is_null()); assert!(!applied.is_null()); { + #[expect(deprecated)] let applied = vx_array::as_ref(applied).to_bool(); let expected = BoolArray::from_iter([true, false, false]); assert_eq!(applied.to_bit_buffer(), expected.to_bit_buffer()); diff --git a/vortex-ffi/src/struct_array.rs b/vortex-ffi/src/struct_array.rs index 33409531be0..8141ce70e91 100644 --- a/vortex-ffi/src/struct_array.rs +++ b/vortex-ffi/src/struct_array.rs @@ -132,6 +132,7 @@ mod tests { use std::sync::Arc; use vortex::array::IntoArray; + #[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -236,6 +237,7 @@ mod tests { assert!(!array.is_null()); { + #[expect(deprecated)] let array = vx_array::as_ref(array).to_struct(); assert_arrays_eq!(array, struct_array); } diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index e87fc4f731d..e42d5671a00 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -12,6 +12,7 @@ use futures::TryStreamExt; use futures::pin_mut; use vortex_array::ArrayRef; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -307,6 +308,7 @@ async fn test_read_projection() { ) ); + #[expect(deprecated)] let actual = array.to_struct().unmasked_field(0).clone(); let expected = VarBinArray::from(strings_expected.to_vec()).into_array(); assert_arrays_eq!(actual, expected); @@ -329,6 +331,7 @@ async fn test_read_projection() { ) ); + #[expect(deprecated)] let actual = array.to_struct().unmasked_field(0).clone(); let expected = Buffer::copy_from(numbers_expected).into_array(); assert_arrays_eq!(actual, expected); @@ -373,6 +376,7 @@ async fn unequal_batches() { let array = array.unwrap(); item_count += array.len(); + #[expect(deprecated)] let numbers = array .to_struct() .unmasked_field_by_name("numbers") @@ -542,12 +546,14 @@ async fn filter_string() { .unwrap(); assert_eq!(result.len(), 1); + #[expect(deprecated)] let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter(vec![Some("Joseph")], DType::Utf8(Nullability::Nullable)) .into_array(); assert_arrays_eq!(names_actual, names_expected); + #[expect(deprecated)] let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32)]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); @@ -597,6 +603,7 @@ async fn filter_or() { .unwrap(); assert_eq!(result.len(), 1); + #[expect(deprecated)] let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), Some("Angela")], @@ -605,6 +612,7 @@ async fn filter_or() { .into_array(); assert_arrays_eq!(names_actual, names_expected); + #[expect(deprecated)] let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), None]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); @@ -651,6 +659,7 @@ async fn filter_and() { .unwrap(); assert_eq!(result.len(), 1); + #[expect(deprecated)] let names_actual = result[0].to_struct().unmasked_field(0).clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), None], @@ -659,6 +668,7 @@ async fn filter_and() { .into_array(); assert_arrays_eq!(names_actual, names_expected); + #[expect(deprecated)] let ages_actual = result[0].to_struct().unmasked_field(1).clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), Some(31i32)]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); @@ -691,6 +701,7 @@ async fn test_with_indices_simple() { let file = SESSION.open_options().open_buffer(buf).unwrap(); // test no indices + #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -707,6 +718,7 @@ async fn test_with_indices_simple() { // test a few indices let kept_indices = [0_u64, 3, 99, 100, 101, 399, 400, 401, 499]; + #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -717,6 +729,7 @@ async fn test_with_indices_simple() { .await .unwrap() .to_struct(); + #[expect(deprecated)] let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); let expected_kept_numbers: Vec = kept_indices @@ -727,6 +740,7 @@ async fn test_with_indices_simple() { assert_arrays_eq!(actual_kept_numbers_array, expected_array); // test all indices + #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -770,6 +784,7 @@ async fn test_with_indices_on_two_columns() { let file = SESSION.open_options().open_buffer(buf).unwrap(); let kept_indices = [0_u64, 3, 7]; + #[expect(deprecated)] let array = file .scan() .unwrap() @@ -824,6 +839,7 @@ async fn test_with_indices_and_with_row_filter_simple() { let file = SESSION.open_options().open_buffer(buf).unwrap(); + #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -841,6 +857,7 @@ async fn test_with_indices_and_with_row_filter_simple() { // test a few indices let kept_indices = [0u64, 3, 99, 100, 101, 399, 400, 401, 499]; + #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -853,6 +870,7 @@ async fn test_with_indices_and_with_row_filter_simple() { .unwrap() .to_struct(); + #[expect(deprecated)] let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); let expected_kept_numbers: Buffer = kept_indices @@ -864,6 +882,7 @@ async fn test_with_indices_and_with_row_filter_simple() { assert_arrays_eq!(actual_kept_numbers_array, expected_array); // test all indices + #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -924,6 +943,7 @@ async fn filter_string_chunked() { let file = SESSION.open_options().open_buffer(buf).unwrap(); + #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -1012,6 +1032,7 @@ async fn test_pruning_with_or() { let file = SESSION.open_options().open_buffer(buf).unwrap(); + #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -1085,6 +1106,7 @@ async fn test_repeated_projection() { let file = SESSION.open_options().open_buffer(buf).unwrap(); + #[expect(deprecated)] let actual = file .scan() .unwrap() @@ -1241,6 +1263,7 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { )? .into_array(); + #[expect(deprecated)] let result = round_trip(&array, Ok).await?.to_struct(); assert_eq!(result.len(), 3); @@ -1248,6 +1271,7 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); assert!(result.all_valid(&mut ctx)?); + #[expect(deprecated)] let nested_struct = result.unmasked_field_by_name("struct")?.to_struct(); assert_eq!(nested_struct.dtype(), &nested_dtype); assert_eq!(nested_struct.len(), 3); @@ -1381,6 +1405,7 @@ async fn test_writer_multiple_pushes() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 9); + #[expect(deprecated)] let numbers = result .to_struct() .unmasked_field_by_name("numbers")? @@ -1415,6 +1440,7 @@ async fn test_writer_push_stream() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 6); + #[expect(deprecated)] let numbers = result .to_struct() .unmasked_field_by_name("numbers")? @@ -1479,6 +1505,7 @@ async fn test_writer_empty_chunks() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 2); + #[expect(deprecated)] let numbers = result .to_struct() .unmasked_field_by_name("numbers")? @@ -1517,6 +1544,7 @@ async fn test_writer_mixed_push_and_stream() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 6); + #[expect(deprecated)] let numbers = result .to_struct() .unmasked_field_by_name("numbers")? @@ -1559,10 +1587,12 @@ async fn test_writer_with_complex_types() -> VortexResult<()> { assert_eq!(result.len(), 3); assert_eq!(result.dtype(), &dtype); + #[expect(deprecated)] let strings_field = result .to_struct() .unmasked_field_by_name("strings") .cloned()?; + #[expect(deprecated)] let strings = strings_field.to_varbinview().with_iterator(|iter| { iter.map(|s| s.map(|st| unsafe { String::from_utf8_unchecked(st.to_vec()) })) .collect::>() diff --git a/vortex-file/tests/test_write_table.rs b/vortex-file/tests/test_write_table.rs index aa924ae44b9..06f491e9491 100644 --- a/vortex-file/tests/test_write_table.rs +++ b/vortex-file/tests/test_write_table.rs @@ -9,6 +9,7 @@ use std::sync::LazyLock; use futures::StreamExt; use futures::pin_mut; use vortex_array::IntoArray; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DictArray; @@ -102,7 +103,9 @@ async fn test_file_roundtrip() { while let Some(next) = stream.next().await { let next = next.expect("next"); + #[expect(deprecated)] let next = next.to_struct(); + #[expect(deprecated)] let a = next.unmasked_field_by_name("a").unwrap().to_struct(); let b = next.unmasked_field_by_name("b").unwrap(); diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index fe5496521b6..0ad60ae80ac 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -29,6 +29,7 @@ use jni::sys::jstring; use vortex::array::ArrayRef; use vortex::array::ArrayView; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::VarBin; @@ -234,6 +235,7 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getField( let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { + #[expect(deprecated)] let struct_array = array_ref.inner.to_struct(); let idx = index as usize; if idx >= struct_array.struct_fields().nfields() { @@ -303,14 +305,12 @@ macro_rules! get_primitive { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { let scalar_value = if array_ref.is_extension { - array_ref - .inner - .to_extension() - .storage_array() - .execute_scalar( - index as usize, - &mut LEGACY_SESSION.create_execution_ctx(), - )? + #[expect(deprecated)] + let ext = array_ref.inner.to_extension(); + ext.storage_array().execute_scalar( + index as usize, + &mut LEGACY_SESSION.create_execution_ctx(), + )? } else { array_ref.inner.execute_scalar( index as usize, @@ -348,10 +348,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBigDecimal( let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { let scalar_value = if array_ref.is_extension { - array_ref - .inner - .to_extension() - .storage_array() + #[expect(deprecated)] + let ext = array_ref.inner.to_extension(); + ext.storage_array() .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())? } else { array_ref diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index c301a696784..9807f97b5ad 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -524,13 +524,12 @@ mod tests { .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .into_array(); - assert_arrays_eq!( - actual - .to_canonical() - .vortex_expect("to_canonical failed") - .into_array(), - expected - ); + #[expect(deprecated)] + let actual_canonical = actual + .to_canonical() + .vortex_expect("to_canonical failed") + .into_array(); + assert_arrays_eq!(actual_canonical, expected); }) } } diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index f6b779b431d..d08c5933fb5 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -9,6 +9,7 @@ use itertools::Itertools; use parking_lot::Mutex; use vortex_array::ArrayRef; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical as _; use vortex_array::VortexSessionExecute; use vortex_array::arrays::struct_::StructArrayExt; @@ -95,6 +96,7 @@ impl FileStatsAccumulator { ) -> VortexResult<(SequenceId, ArrayRef)> { let (sequence_id, chunk) = chunk?; if chunk.dtype().is_struct() { + #[expect(deprecated)] let chunk = chunk.to_struct(); for (acc, field) in self .accumulators diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index bf6993d72b7..46cde10789f 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -201,6 +201,7 @@ mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -404,24 +405,22 @@ mod tests { .bit_buffer(), AllOr::Some(&validity_boolean_buffer) ); - assert_eq!( - result - .to_struct() - .unmasked_field_by_name("a") - .unwrap() - .to_primitive() - .as_slice::(), - &[1, 2] - ); - assert_eq!( - result - .to_struct() - .unmasked_field_by_name("b") - .unwrap() - .to_primitive() - .as_slice::(), - &[3, 4] - ); + #[expect(deprecated)] + let result_struct = result.to_struct(); + #[expect(deprecated)] + let field_a = result_struct + .unmasked_field_by_name("a") + .unwrap() + .to_primitive(); + assert_eq!(field_a.as_slice::(), &[1, 2]); + #[expect(deprecated)] + let result_struct_b = result.to_struct(); + #[expect(deprecated)] + let field_b = result_struct_b + .unmasked_field_by_name("b") + .unwrap() + .to_primitive(); + assert_eq!(field_b.as_slice::(), &[3, 4]); }) } diff --git a/vortex-layout/src/layouts/repartition.rs b/vortex-layout/src/layouts/repartition.rs index 03c7e15d722..f9bd3de86ac 100644 --- a/vortex-layout/src/layouts/repartition.rs +++ b/vortex-layout/src/layouts/repartition.rs @@ -107,7 +107,9 @@ impl LayoutStrategy for RepartitionStrategy { dtype.clone(), stream.map(|chunk| { let (sequence_id, chunk) = chunk?; - VortexResult::Ok((sequence_id, chunk.to_canonical()?.into_array())) + #[expect(deprecated)] + let canonical = chunk.to_canonical()?.into_array(); + VortexResult::Ok((sequence_id, canonical)) }), ) .sendable() @@ -145,9 +147,11 @@ impl LayoutStrategy for RepartitionStrategy { let chunked = ChunkedArray::try_new(output_chunks, dtype_clone.clone())?; if !chunked.is_empty() { + #[expect(deprecated)] + let canonical = chunked.into_array().to_canonical()?.into_array(); yield ( sequence_pointer.advance(), - chunked.into_array().to_canonical()?.into_array(), + canonical, ) } } @@ -158,9 +162,11 @@ impl LayoutStrategy for RepartitionStrategy { dtype_clone.clone(), )?; if !to_flush.is_empty() { + #[expect(deprecated)] + let canonical = to_flush.into_array().to_canonical()?.into_array(); yield ( sequence_pointer.advance(), - to_flush.into_array().to_canonical()?.into_array(), + canonical, ) } } @@ -498,7 +504,8 @@ mod tests { // Transition SharedState from Source to Cached for ALL slices sharing this Arc. use vortex_array::arrays::shared::SharedArrayExt; - shared_handle.get_or_compute(|source| source.to_canonical())?; + #[expect(deprecated)] + let _canonical = shared_handle.get_or_compute(|source| source.to_canonical())?; // Before the fix this panicked with "attempt to subtract with overflow". let _s2 = buf.pop_front().unwrap(); diff --git a/vortex-layout/src/layouts/row_idx/mod.rs b/vortex-layout/src/layouts/row_idx/mod.rs index a4cabee8e45..f435d6bd992 100644 --- a/vortex-layout/src/layouts/row_idx/mod.rs +++ b/vortex-layout/src/layouts/row_idx/mod.rs @@ -293,7 +293,9 @@ fn row_idx_array_future( let expr = expr.clone(); async move { let array = idx_array(row_offset, &row_range).into_array(); - let array = array.filter(mask.await?)?.to_canonical()?.into_array(); + let filtered = array.filter(mask.await?)?; + #[expect(deprecated)] + let array = filtered.to_canonical()?.into_array(); array.apply(&expr) } .boxed() diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index a9ea39a6fe0..6c39a8f1086 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -11,6 +11,7 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::MaskFuture; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; @@ -359,6 +360,7 @@ impl LayoutReader for StructReader { // If root expression was a pack, then we apply the validity to each child field if is_pack_merge { + #[expect(deprecated)] let struct_array = array.to_struct(); let masked_fields: Vec = struct_array .iter_unmasked_fields() @@ -394,6 +396,7 @@ mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; @@ -688,14 +691,18 @@ mod tests { assert_eq!(result.len(), 2); let expected_a = PrimitiveArray::from_iter([7i32, 2]); + #[expect(deprecated)] + let result_struct_a = result.to_struct(); assert_arrays_eq!( - result.to_struct().unmasked_field_by_name("a").unwrap(), + result_struct_a.unmasked_field_by_name("a").unwrap(), expected_a ); let expected_b = PrimitiveArray::from_iter([4i32, 5]); + #[expect(deprecated)] + let result_struct_b = result.to_struct(); assert_arrays_eq!( - result.to_struct().unmasked_field_by_name("b").unwrap(), + result_struct_b.unmasked_field_by_name("b").unwrap(), expected_b ); } diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index 0c602cd78fa..ca7a9bb4c27 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -17,6 +17,7 @@ use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::struct_::StructArrayExt; @@ -236,6 +237,7 @@ impl LayoutStrategy for TableStrategy { let columns_vec_stream = stream.map(move |chunk| { let (sequence_id, chunk) = chunk?; let mut sequence_pointer = sequence_id.descend(); + #[expect(deprecated)] let struct_chunk = chunk.to_struct(); let mut columns: Vec<(SequenceId, ArrayRef)> = Vec::new(); if is_nullable { diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 23118e8ce5c..1fd1ab6b0cc 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -16,6 +16,7 @@ use itertools::Itertools; use parking_lot::RwLock; use vortex_array::ArrayRef; use vortex_array::MaskFuture; +#[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; @@ -136,6 +137,7 @@ impl ZonedReader { .vortex_expect("Failed construct zone map evaluation"); async move { + #[expect(deprecated)] let zones_array = zones_eval.await?.to_struct(); // SAFETY: This is only fine to call because we perform validation above Ok(unsafe { ZoneMap::new_unchecked(zones_array, present_stats) }) diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index cb49377007e..083dc5ca787 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -279,6 +279,7 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -340,20 +341,16 @@ mod tests { MIN_IS_TRUNCATED, ] ); + #[expect(deprecated)] + let field1_bool = stats_table.array.unmasked_field(1).to_bool(); assert_eq!( - stats_table - .array - .unmasked_field(1) - .to_bool() - .to_bit_buffer(), + field1_bool.to_bit_buffer(), BitBuffer::from(vec![false, true]) ); + #[expect(deprecated)] + let field3_bool = stats_table.array.unmasked_field(3).to_bool(); assert_eq!( - stats_table - .array - .unmasked_field(3) - .to_bool() - .to_bit_buffer(), + field3_bool.to_bit_buffer(), BitBuffer::from(vec![true, false]) ); } @@ -378,22 +375,12 @@ mod tests { Stat::Sum.name(), ] ); - assert_eq!( - stats_table - .array - .unmasked_field(1) - .to_bool() - .to_bit_buffer(), - BitBuffer::from(vec![false]) - ); - assert_eq!( - stats_table - .array - .unmasked_field(3) - .to_bool() - .to_bit_buffer(), - BitBuffer::from(vec![false]) - ); + #[expect(deprecated)] + let field1_bool = stats_table.array.unmasked_field(1).to_bool(); + assert_eq!(field1_bool.to_bit_buffer(), BitBuffer::from(vec![false])); + #[expect(deprecated)] + let field3_bool = stats_table.array.unmasked_field(3).to_bool(); + assert_eq!(field3_bool.to_bit_buffer(), BitBuffer::from(vec![false])); } #[rstest] diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index d53a38163fa..e1b652631ec 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -466,6 +466,7 @@ mod test { use parking_lot::Mutex; use vortex_array::IntoArray; use vortex_array::MaskFuture; + #[expect(deprecated)] use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; @@ -664,7 +665,9 @@ mod test { let mut values = Vec::new(); for chunk in &mut iter { - values.push(chunk?.to_primitive().into_buffer::()[0]); + #[expect(deprecated)] + let prim = chunk?.to_primitive(); + values.push(prim.into_buffer::()[0]); } assert_eq!(calls.load(Ordering::Relaxed), 1); diff --git a/vortex-python/src/arrays/compressed.rs b/vortex-python/src/arrays/compressed.rs index fd21c54fe69..ca14c9a52b5 100644 --- a/vortex-python/src/arrays/compressed.rs +++ b/vortex-python/src/arrays/compressed.rs @@ -3,6 +3,7 @@ use pyo3::prelude::*; use vortex::array::IntoArray; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::Dict; use vortex::encodings::alp::ALP; @@ -89,9 +90,9 @@ impl EncodingSubclass for PyZigZagArray { impl PyZigZagArray { #[staticmethod] pub fn encode(array: PyArrayRef) -> PyVortexResult { - Ok(PyVortex( - zigzag_encode(array.inner().clone().to_primitive().as_view())?.into_array(), - )) + #[expect(deprecated)] + let primitive = array.inner().clone().to_primitive(); + Ok(PyVortex(zigzag_encode(primitive.as_view())?.into_array())) } } diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index bdcc95d7726..af5f6afee6f 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -25,6 +25,7 @@ use pyo3_bytes::PyBytes; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::Chunked; @@ -527,10 +528,12 @@ impl PyArray { /// ``` fn filter(slf: Bound, mask: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); - let mask = (&*mask as &ArrayRef) - .to_bool() - .to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); - let inner = slf.filter(mask)?.to_canonical()?.into_array(); + #[expect(deprecated)] + let mask_bool = (&*mask as &ArrayRef).to_bool(); + let mask = mask_bool.to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); + #[expect(deprecated)] + let canonical = slf.filter(mask)?.to_canonical()?; + let inner = canonical.into_array(); Ok(PyArrayRef::from(inner)) } diff --git a/vortex-python/src/dataset.rs b/vortex-python/src/dataset.rs index dccb103037d..c9579043da6 100644 --- a/vortex-python/src/dataset.rs +++ b/vortex-python/src/dataset.rs @@ -11,6 +11,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyString; use vortex::array::ArrayRef; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::iter::ArrayIteratorExt; use vortex::dtype::FieldName; @@ -61,7 +62,9 @@ pub fn read_array_from_reader( } if let Some(indices) = indices { - let indices = indices.to_primitive().into_buffer(); + #[expect(deprecated)] + let primitive = indices.to_primitive(); + let indices = primitive.into_buffer(); scan = scan.with_row_indices(indices); } diff --git a/vortex-python/src/file.rs b/vortex-python/src/file.rs index 06cee7590c1..458736db0e5 100644 --- a/vortex-python/src/file.rs +++ b/vortex-python/src/file.rs @@ -9,6 +9,7 @@ use pyo3::prelude::*; use pyo3::types::PyList; use pyo3_object_store::PyObjectStore; use vortex::array::ArrayRef; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::builtins::ArrayBuiltins; use vortex::dtype::DType; @@ -213,10 +214,9 @@ impl PyVortexFile { } if let Some(indices) = indices { - let indices = indices - .cast(DType::Primitive(PType::U64, NonNullable))? - .to_primitive() - .into_buffer::(); + let casted = indices.cast(DType::Primitive(PType::U64, NonNullable))?; + #[expect(deprecated)] + let indices = casted.to_primitive().into_buffer::(); builder = builder.with_row_indices(indices); } diff --git a/vortex-tui/src/browse/ui/layouts.rs b/vortex-tui/src/browse/ui/layouts.rs index d90720c5223..845a5988c78 100644 --- a/vortex-tui/src/browse/ui/layouts.rs +++ b/vortex-tui/src/browse/ui/layouts.rs @@ -28,6 +28,7 @@ use ratatui::widgets::Widget; use ratatui::widgets::Wrap; use vortex::array::ArrayRef; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::struct_::StructArrayExt; @@ -142,6 +143,7 @@ fn render_array(app: &AppState, area: Rect, buf: &mut Buffer, is_stats_table: bo if is_stats_table { // Render the stats table horizontally + #[expect(deprecated)] let struct_array = array.to_struct(); // add 1 for the chunk column let field_count = struct_array.struct_fields().nfields() + 1; diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 7fbbd80620d..9a96c478e5c 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -16,6 +16,7 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; @@ -76,12 +77,14 @@ mod setup { let mut rng = StdRng::seed_from_u64(0); let uint_array = PrimitiveArray::from_iter((0..NUM_VALUES).map(|_| rng.random_range(42u32..256))); + #[expect(deprecated)] let int_array = uint_array .clone() .into_array() .cast(PType::I32.into()) .unwrap() .to_primitive(); + #[expect(deprecated)] let float_array = uint_array .clone() .into_array() @@ -113,7 +116,9 @@ mod setup { .unwrap(); // Manually construct ALP <- FoR <- BitPacked tree - let for_array = FoR::encode(alp_compressed.encoded().to_primitive()).unwrap(); + #[expect(deprecated)] + let alp_encoded_prim = alp_compressed.encoded().to_primitive(); + let for_array = FoR::encode(alp_encoded_prim).unwrap(); let inner = for_array.encoded(); let bp = BitPacked::encode(inner, 8).unwrap(); let for_with_bp = @@ -184,6 +189,7 @@ mod setup { let runend = RunEnd::encode(prim_array.into_array()).unwrap(); // Compress the ends with FoR <- BitPacked + #[expect(deprecated)] let ends_prim = runend.ends().to_primitive(); let ends_for = FoR::encode(ends_prim).unwrap(); let ends_inner = ends_for.encoded(); @@ -194,6 +200,7 @@ mod setup { .into_array(); // Compress the values with BitPacked + #[expect(deprecated)] let values_prim = runend.values().to_primitive(); let compressed_values = BitPacked::encode(&values_prim.into_array(), 8) .unwrap() @@ -268,6 +275,7 @@ mod setup { // Compress the VarBin offsets with BitPacked let codes = fsst.codes(); + #[expect(deprecated)] let offsets_prim = codes.offsets().to_primitive(); let offsets_bp = BitPacked::encode(&offsets_prim.into_array(), 20).unwrap(); @@ -321,6 +329,7 @@ mod setup { let parts = split_temporal(temporal_array.clone()).unwrap(); // Compress days with FoR <- BitPacked + #[expect(deprecated)] let days_prim = parts.days.to_primitive(); let days_for = FoR::encode(days_prim).unwrap(); let days_inner = days_for.encoded(); @@ -331,6 +340,7 @@ mod setup { .into_array(); // Compress seconds with FoR <- BitPacked + #[expect(deprecated)] let seconds_prim = parts.seconds.to_primitive(); let seconds_for = FoR::encode(seconds_prim).unwrap(); let seconds_inner = seconds_for.encoded(); @@ -343,6 +353,7 @@ mod setup { .into_array(); // Compress subseconds with FoR <- BitPacked + #[expect(deprecated)] let subseconds_prim = parts.subseconds.to_primitive(); let subseconds_for = FoR::encode(subseconds_prim).unwrap(); let subseconds_inner = subseconds_for.encoded(); diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 4ff5a8de9ef..4e2ef9b1beb 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -18,6 +18,7 @@ use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; +#[expect(deprecated)] use vortex::array::ToCanonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; @@ -77,12 +78,14 @@ fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) let mut rng = StdRng::seed_from_u64(0); let uint_array = PrimitiveArray::from_iter((0..NUM_VALUES).map(|_| rng.random_range(42u32..256))); + #[expect(deprecated)] let int_array = uint_array .clone() .into_array() .cast(PType::I32.into()) .unwrap() .to_primitive(); + #[expect(deprecated)] let float_array = uint_array .clone() .into_array() From 7e0af3c1f63049dc951608aa2852e560ce0923ff Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:08:40 -0400 Subject: [PATCH 090/250] Unify download management (#7490) ## Summary Unifies the download management for benchmarks. Also makes the downloads smarter with AIMD and nicer progress bars. ## Testing I just ran it in my terminal and it works well enough. Let me know if we want a video for this and I can figure that out. https://github.com/user-attachments/assets/85a7a0a6-3b88-4f35-bcbc-41dbdba37aff --------- Signed-off-by: Connor Tsui --- vortex-bench/src/clickbench/benchmark.rs | 28 +- vortex-bench/src/clickbench/data.rs | 20 +- vortex-bench/src/datasets/data_downloads.rs | 818 +++++++++++++++++--- vortex-bench/src/fineweb/mod.rs | 52 +- vortex-bench/src/public_bi.rs | 13 +- vortex-bench/src/realnest/gharchive.rs | 27 +- vortex-bench/src/utils/file.rs | 42 + vortex-bench/src/vector_dataset/download.rs | 31 +- 8 files changed, 789 insertions(+), 242 deletions(-) diff --git a/vortex-bench/src/clickbench/benchmark.rs b/vortex-bench/src/clickbench/benchmark.rs index 2e197bef8e1..9cfafb8832e 100644 --- a/vortex-bench/src/clickbench/benchmark.rs +++ b/vortex-bench/src/clickbench/benchmark.rs @@ -7,13 +7,13 @@ use std::path::Path; use anyhow::Result; use url::Url; -use vortex::error::VortexExpect; use crate::Benchmark; use crate::BenchmarkDataset; use crate::IdempotentPath; use crate::TableSpec; use crate::clickbench::*; +use crate::utils::file::resolve_data_url; /// ClickBench benchmark implementation pub struct ClickBenchBenchmark { @@ -37,31 +37,7 @@ impl ClickBenchBenchmark { } fn create_data_url(remote_data_dir: &Option, flavor: Flavor) -> Result { - match remote_data_dir { - None => { - let basepath = format!("clickbench_{flavor}").to_data_path(); - Ok(Url::parse(&format!( - "file:{}/", - basepath.to_str().vortex_expect("path should be utf8") - ))?) - } - Some(remote_data_dir) => { - if !remote_data_dir.ends_with("/") { - tracing::warn!( - "Supply a --use-remote-data-dir argument which ends in a slash e.g. s3://vortex-bench-dev-eu/parquet/" - ); - } - tracing::info!( - concat!( - "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\\n", - "If it does not, you should kill this command, locally generate the files (by running without\\n", - "--use-remote-data-dir) and upload data/clickbench/ to some remote location.", - ), - remote_data_dir, - ); - Ok(Url::parse(remote_data_dir)?) - } - } + resolve_data_url(remote_data_dir.as_deref(), &format!("clickbench_{flavor}")) } } diff --git a/vortex-bench/src/clickbench/data.rs b/vortex-bench/src/clickbench/data.rs index af11e468d45..ba2a2104192 100644 --- a/vortex-bench/src/clickbench/data.rs +++ b/vortex-bench/src/clickbench/data.rs @@ -14,7 +14,6 @@ use arrow_schema::TimeUnit; use clap::ValueEnum; use serde::Deserialize; use serde::Serialize; -use tokio::task::JoinSet; use tracing::info; use vortex::error::VortexExpect; @@ -22,6 +21,7 @@ use crate::Format; // Re-export for use by clickbench_benchmark pub use crate::conversions::convert_parquet_directory_to_vortex; use crate::datasets::data_downloads::download_data; +use crate::datasets::data_downloads::download_many; pub static HITS_SCHEMA: LazyLock = LazyLock::new(|| { use DataType::*; @@ -193,18 +193,14 @@ impl Flavor { Flavor::Partitioned => { // The clickbench-provided file is missing some higher-level type info, so we reprocess it // to add that info, see https://github.com/ClickHouse/ClickBench/issues/7. - - let mut tasks = (0_u32..100).map(|idx| { - let output_path = basepath.join(Format::Parquet.name()).join(format!("hits_{idx}.parquet")); - - info!("Downloading file {idx}"); + info!("Downloading 100 ClickBench parquet shards"); + let parquet_dir = basepath.join(Format::Parquet.name()); + let downloads = (0_u32..100).map(|idx| { + let output_path = parquet_dir.join(format!("hits_{idx}.parquet")); let url = format!("https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_{idx}.parquet"); - download_data(output_path, url) - }).collect::>(); - - while let Some(task) = tasks.join_next().await { - task??; - } + (output_path, url) + }); + download_many(downloads).await?; } } Ok(()) diff --git a/vortex-bench/src/datasets/data_downloads.rs b/vortex-bench/src/datasets/data_downloads.rs index a603436b458..ae1281f8366 100644 --- a/vortex-bench/src/datasets/data_downloads.rs +++ b/vortex-bench/src/datasets/data_downloads.rs @@ -1,153 +1,763 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::cmp::Ordering; use std::fs::File; -use std::io::Read; -use std::io::Write; +use std::io; +use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::LazyLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering as AtomicOrdering; use std::time::Duration; +use std::time::Instant; use anyhow::Context; use anyhow::Error; use anyhow::Result; use bzip2::read::BzDecoder; use futures::StreamExt; +use futures::stream; +use indicatif::MultiProgress; use indicatif::ProgressBar; use indicatif::ProgressStyle; -use parking_lot::RwLock; +use parking_lot::Mutex; use reqwest::Client; -use reqwest::Response; use tokio::fs::File as TokioFile; use tokio::io::AsyncWriteExt; +use tokio::sync::OwnedSemaphorePermit; +use tokio::sync::Semaphore; use tracing::info; use tracing::warn; use crate::utils::file::idempotent; use crate::utils::file::idempotent_async; -async fn retry_get>, R: Fn() -> F>( - make_req: R, - tmp_path: PathBuf, -) -> Result<()> { - const MAX_ATTEMPTS: u32 = 3; - let mut last_err: Option = None; - let progress = RwLock::new(None::); +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Public API +//////////////////////////////////////////////////////////////////////////////////////////////////// - let retry = async || -> Result<()> { - let mut file = TokioFile::create(tmp_path) - .await - .context("Failed to create file")?; - let response = make_req() +/// Anything that can be described as a `(target_path, url)` pair accepted by +/// [`download_many`]. +pub trait IntoDownload { + fn into_download(self) -> (PathBuf, String); +} + +impl IntoDownload for (P, S) +where + P: Into, + S: Into, +{ + fn into_download(self) -> (PathBuf, String) { + (self.0.into(), self.1.into()) + } +} + +/// Idempotently download a single URL to `fname`. +/// +/// Uses the shared HTTP client, a 3-attempt exponential backoff retry loop with jitter, +/// and an [`indicatif::ProgressBar`]. If `fname` already exists, the download is +/// skipped. +#[tracing::instrument(skip_all, fields(url = %data_url.as_ref(), path = %fname.display()))] +pub async fn download_data(fname: PathBuf, data_url: impl AsRef) -> Result { + download_one(fname, data_url.as_ref(), None).await +} + +/// Idempotently download many `(path, url)` pairs with adaptive parallelism. +/// +/// This is the preferred way to fetch multi-shard datasets (ClickBench partitioned, +/// vector dataset train shards, Public BI tables, etc.) because it: +/// +/// - starts at `INITIAL_IN_FLIGHT` concurrent downloads and ramps up to +/// `MAX_IN_FLIGHT` as clean completions come in (TCP-style slow-start), then +/// halves on retries to back off from upstream rate limits, +/// - reuses the shared HTTP client across every shard, +/// - renders a top-of-block `N/total` bar plus a fixed number of reusable slot bars via +/// a shared [`MultiProgress`]: the terminal block size stays constant for the entire +/// run, so nothing "jumps" as shards cycle, +/// - short-circuits on the first error (the remaining in-flight downloads are dropped +/// when the returned future is dropped), +/// - returns the resolved on-disk paths in completion order (not submission order). +#[tracing::instrument(skip_all, fields(count = tracing::field::Empty))] +pub async fn download_many(downloads: I) -> Result> +where + I: IntoIterator, + I::Item: IntoDownload, +{ + let downloads: Vec<(PathBuf, String)> = downloads + .into_iter() + .map(IntoDownload::into_download) + .collect(); + tracing::Span::current().record("count", downloads.len()); + + if downloads.is_empty() { + return Ok(Vec::new()); + } + + let num_slots = downloads.len().min(MAX_IN_FLIGHT); + let initial_in_flight = INITIAL_IN_FLIGHT.min(num_slots); + let batch = BatchProgress::new(downloads.len() as u64, num_slots, initial_in_flight); + + let results: Vec> = stream::iter(downloads) + .map(|(path, url)| { + let batch = batch.clone(); + async move { + let result = download_one(path, &url, Some(&batch)).await; + if result.is_ok() { + batch.advance(); + } + result + } + }) + .buffer_unordered(num_slots) + .collect() + .await; + + batch.finish(); + + results.into_iter().collect() +} + +/// Idempotently decompress a bzip2 file into `output_path`, streaming the decompressed bytes +/// straight to disk so memory stays bounded. +/// +/// This is used for the public BI dataset. +#[tracing::instrument(skip_all, fields(input = %input_path.display(), output = %output_path.display()))] +pub fn decompress_bz2(input_path: PathBuf, output_path: PathBuf) -> Result { + idempotent(&output_path, |path| { + info!( + "Decompressing bzip from {} to {}", + input_path.display(), + output_path.display() + ); + let input_file = File::open(&input_path) + .with_context(|| format!("Failed to open input file: {:?}", input_path))?; + let mut decoder = BzDecoder::new(input_file); + + let mut output_file = File::create(path) + .with_context(|| format!("Failed to create output file: {:?}", path))?; + io::copy(&mut decoder, &mut output_file).context("Failed to decompress bzip2 stream")?; + Ok(output_path.clone()) + }) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Shared HTTP client +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Shared HTTP client used by every dataset download. +/// +/// Reusing a single client gives us connection pooling, DNS caching, and consistent +/// timeouts across all callers. Each benchmark used to build its own +/// [`reqwest::Client`] on every download, which both wasted TLS handshakes and made it +/// hard to reason about total in-flight concurrency. +static HTTP_CLIENT: LazyLock = LazyLock::new(|| { + Client::builder() + .read_timeout(Duration::from_secs(60)) + .timeout(Duration::from_secs(60 * 15)) + .build() + .expect("failed to build shared benchmark HTTP client") +}); + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Progress-bar templates +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Template for a slot that currently holds no download. `{msg}` with an empty message +/// renders as a blank line while still reserving the terminal row. +const IDLE_TEMPLATE: &str = "{msg}"; + +/// Template for a slot that has acquired a download but has not yet received response +/// headers. Shows the filename plus an animated spinner so the user can see we are +/// still making forward progress during TLS + request + first-byte latency. +const CONNECTING_TEMPLATE: &str = "{prefix:>28!} {spinner} connecting..."; + +/// Template for an active download when the response advertised a `Content-Length`. +const KNOWN_SIZE_TEMPLATE: &str = + "{prefix:>28!} [{bar:30.cyan/blue}] {bytes:>9}/{total_bytes:>9} ({bytes_per_sec})"; + +/// Template for an active download when the response size is unknown. +const UNKNOWN_SIZE_TEMPLATE: &str = "{prefix:>28!} {spinner} {bytes} ({bytes_per_sec})"; + +/// Template for the top-of-block summary bar rendered by [`download_many`]. `{pos}/{len}` +/// tracks completed-of-total; `{msg}` is updated on every slot acquire / release to show +/// how many downloads are currently in flight. +const SHARDS_TEMPLATE: &str = + "[{elapsed_precise}] shards [{bar:30.green/white}] {pos}/{len} {msg}"; + +/// How often slot spinners redraw. Fast enough to feel alive; slow enough that stderr +/// writes sneaking past `MultiProgress` do not constantly fight for cursor position. +const SLOT_TICK: Duration = Duration::from_millis(80); + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Dynamic concurrency controller +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Number of in-flight downloads to start at. Matches TCP-style slow-start: start small so +/// we don't hammer the upstream on the very first connection, and double from there. +const INITIAL_IN_FLIGHT: usize = 4; + +/// Upper bound on the number of concurrent downloads the controller can ramp up to, and +/// the number of slot rows pre-allocated in the [`MultiProgress`] block. Chosen large +/// enough that the retry-based controller is the effective ceiling, not this constant. +/// The trade-off is that on large batches the MP block will exceed a typical local +/// terminal height — indicatif handles this by drawing the most recent rows plus the +/// top shards bar — but on CI there is no TTY so the visual overflow does not apply. +const MAX_IN_FLIGHT: usize = 256; + +/// Never let the controller drive concurrency below this floor on a flaky network. +/// A value of `1` means the fallback is serial downloads. +const MIN_IN_FLIGHT: usize = 1; + +/// Minimum time between successive halves, in milliseconds. Coalesces simultaneous +/// retries from one upstream hiccup into a single reaction, preventing over-halving. +const HALVE_COOLDOWN_MS: u64 = 1000; + +/// Decide the next in-flight limit after a clean (no-retry) download completes. +/// +/// Returns `Some(new_limit)` if the limit should change, or `None` if it is already at +/// the cap or the computed move would be a no-op. +fn decide_on_success(current: usize, in_slow_start: bool) -> Option { + if current >= MAX_IN_FLIGHT { + return None; + } + let new = if in_slow_start { + current.saturating_mul(2) + } else { + current.saturating_add(1) + } + .min(MAX_IN_FLIGHT); + (new != current).then_some(new) +} + +/// Decide the next in-flight limit after a failed download attempt. +/// +/// Returns `Some(new_limit)` if the limit should be halved now. Returns `None` if the +/// halve is debounced (another halve fired within [`HALVE_COOLDOWN_MS`]) or we are +/// already at the [`MIN_IN_FLIGHT`] floor. +fn decide_on_retry(current: usize, now_ms: u64, last_halve_ms: u64) -> Option { + if now_ms.saturating_sub(last_halve_ms) < HALVE_COOLDOWN_MS { + return None; + } + if current <= MIN_IN_FLIGHT { + return None; + } + let new = (current / 2).max(MIN_IN_FLIGHT); + (new != current).then_some(new) +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Batch download internals +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Shared rendering state for a batched download. +/// +/// Layout is built once at construction: the shards bar is registered first (top row), +/// then `num_slots` slot bars are registered below it. None of these are ever added to +/// or removed from the [`MultiProgress`] again. The block size on the terminal is +/// exactly `num_slots + 1` rows for the entire run of [`download_many`]. +/// +/// Per-download lifecycle reuses a single slot bar by swapping its style between an +/// idle placeholder and the active progress-bar / spinner variants. A +/// [`tokio::sync::Semaphore`] gates how many slots can be in use at any instant, which +/// lets a future controller adjust concurrency at runtime via +/// [`BatchProgress::set_max_in_flight`] without ever touching the MP layout. +#[derive(Clone)] +struct BatchProgress { + inner: Arc, +} + +struct BatchInner { + shards_bar: ProgressBar, + free: Mutex>, + in_flight: Arc, + /// Current concurrency limit — the source of truth read by the controller and + /// written via [`BatchProgress::set_max_in_flight`]. + current_in_flight: AtomicUsize, + num_slots: usize, + /// Controller state: are we still in slow-start (double on success) or have we + /// dropped into additive-increase (`+=1` on success) after the first retry? + in_slow_start: AtomicBool, + /// Millis since [`BatchInner::created_at`] of the most recent halve event, used to + /// debounce bursts of retries from a single upstream hiccup. + last_halve_at_ms: AtomicU64, + created_at: Instant, + // The MP is kept alive alongside the Arc so bars stay registered and rendered. + // Once the last BatchProgress clone drops, the MP drops and clears the block. + _mp: MultiProgress, +} + +impl BatchInner { + fn elapsed_ms(&self) -> u64 { + u64::try_from(self.created_at.elapsed().as_millis()).unwrap_or(u64::MAX) + } + + /// Refresh the shards bar message to reflect the current in-flight count. Called on + /// every slot acquire / release and from `set_max_in_flight`. + fn refresh_shards_message(&self) { + let active = self.num_slots - self.free.lock().len(); + let limit = self.current_in_flight.load(AtomicOrdering::Relaxed); + self.shards_bar + .set_message(format!("({active} active, limit {limit})")); + } +} + +impl BatchProgress { + fn new(total: u64, num_slots: usize, initial_in_flight: usize) -> Self { + let initial_in_flight = initial_in_flight.min(num_slots); + let mp = MultiProgress::new(); + + let shards_bar = mp.add(ProgressBar::new(total)); + shards_bar + .set_style(ProgressStyle::with_template(SHARDS_TEMPLATE).expect("valid template")); + + let idle_style = ProgressStyle::with_template(IDLE_TEMPLATE).expect("valid template"); + let mut slots = Vec::with_capacity(num_slots); + for _ in 0..num_slots { + let bar = mp.add(ProgressBar::new(0)); + bar.set_style(idle_style.clone()); + bar.set_message(""); + bar.enable_steady_tick(SLOT_TICK); + slots.push(bar); + } + + let inner = BatchInner { + shards_bar, + free: Mutex::new(slots), + in_flight: Arc::new(Semaphore::new(initial_in_flight)), + current_in_flight: AtomicUsize::new(initial_in_flight), + num_slots, + in_slow_start: AtomicBool::new(true), + last_halve_at_ms: AtomicU64::new(0), + created_at: Instant::now(), + _mp: mp, + }; + inner.refresh_shards_message(); + Self { + inner: Arc::new(inner), + } + } + + /// Wait for an in-flight permit, then claim an idle slot and switch it to the + /// [`CONNECTING_TEMPLATE`] style. The returned guard releases both the permit and + /// the slot on drop. + async fn acquire(&self, prefix: &str) -> SlotGuard { + let permit = Arc::clone(&self.inner.in_flight) + .acquire_owned() .await - .context("Failed to send HTTP request")? - .error_for_status() - .context("HTTP request returned error status")?; - - *progress.write() = response.content_length().map(|total| { - let progress = ProgressBar::new(total); - progress.set_style( - ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} ({bytes_per_sec})", - ) - .expect("valid template"), - ); - progress - }); - - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - AsyncWriteExt::write_all(&mut file, &chunk) - .await - .context("Failed to write to file")?; - if let Some(p) = progress.write().as_mut() { - p.inc(chunk.len() as u64) + .expect("batch semaphore is never closed while a download is in flight"); + let bar = self + .inner + .free + .lock() + .pop() + .expect("slot free list invariant broken: permits outnumber pre-allocated slots"); + bar.set_style(ProgressStyle::with_template(CONNECTING_TEMPLATE).expect("valid template")); + bar.set_prefix(prefix.to_owned()); + bar.set_message(""); + bar.set_length(0); + bar.reset(); + self.inner.refresh_shards_message(); + SlotGuard { + bar, + owner: Arc::clone(&self.inner), + _permit: permit, + } + } + + fn advance(&self) { + self.inner.shards_bar.inc(1); + } + + fn finish(&self) { + self.inner.shards_bar.finish_and_clear(); + } + + /// Called when a download completed on its first attempt (no retries). Drives the + /// slow-start / additive-increase side of AIMD. + fn report_clean_success(&self) { + let current = self.inner.current_in_flight.load(AtomicOrdering::Relaxed); + let in_slow_start = self.inner.in_slow_start.load(AtomicOrdering::Relaxed); + if let Some(new) = decide_on_success(current, in_slow_start) { + self.set_max_in_flight(new); + } + } + + /// Called when a download attempt failed. Drives the halving side of AIMD, with an + /// internal cooldown so a burst of simultaneous retries from one upstream hiccup + /// halves the limit at most once. + fn report_retry(&self) { + let now_ms = self.inner.elapsed_ms(); + let current = self.inner.current_in_flight.load(AtomicOrdering::Relaxed); + let last_halve_ms = self.inner.last_halve_at_ms.load(AtomicOrdering::Relaxed); + if let Some(new) = decide_on_retry(current, now_ms, last_halve_ms) { + self.inner + .in_slow_start + .store(false, AtomicOrdering::Relaxed); + self.inner + .last_halve_at_ms + .store(now_ms, AtomicOrdering::Relaxed); + self.set_max_in_flight(new); + } + } + + /// Adjust how many downloads may run concurrently. Clamped to the pre-allocated + /// slot count. Raising the limit returns immediately; lowering spawns a background + /// task that acquires and forgets the delta so the limit takes effect as active + /// downloads complete naturally, never interrupting an in-flight transfer. + fn set_max_in_flight(&self, target: usize) { + let target = target.min(self.inner.num_slots); + let prev = self + .inner + .current_in_flight + .swap(target, AtomicOrdering::Relaxed); + match target.cmp(&prev) { + Ordering::Greater => { + self.inner.in_flight.add_permits(target - prev); + } + Ordering::Less => { + let delta = u32::try_from(prev - target).expect("delta fits in u32"); + let sem = Arc::clone(&self.inner.in_flight); + tokio::spawn(async move { + if let Ok(permit) = sem.acquire_many_owned(delta).await { + permit.forget(); + } + }); } + Ordering::Equal => {} } + self.inner.refresh_shards_message(); + } +} - AsyncWriteExt::flush(&mut file).await?; - Ok(()) - }; +/// RAII handle for a borrowed slot bar. Drives the bar through its active lifecycle and +/// resets it back to the idle placeholder on drop. +struct SlotGuard { + bar: ProgressBar, + owner: Arc, + _permit: OwnedSemaphorePermit, +} - for attempt in 0..MAX_ATTEMPTS { - let outcome = retry.clone()().await; +impl SlotGuard { + fn activate_known(&self, total: u64) { + self.bar + .set_style(ProgressStyle::with_template(KNOWN_SIZE_TEMPLATE).expect("valid template")); + self.bar.set_length(total); + self.bar.reset(); + } + + fn activate_unknown(&self) { + self.bar.set_style( + ProgressStyle::with_template(UNKNOWN_SIZE_TEMPLATE).expect("valid template"), + ); + self.bar.set_length(0); + self.bar.reset(); + } + + fn inc(&self, n: u64) { + self.bar.inc(n); + } + + fn reset_for_retry(&self) { + self.bar + .set_style(ProgressStyle::with_template(CONNECTING_TEMPLATE).expect("valid template")); + self.bar.set_length(0); + self.bar.reset(); + } +} + +impl Drop for SlotGuard { + fn drop(&mut self) { + let bar = self.bar.clone(); + bar.set_style(ProgressStyle::with_template(IDLE_TEMPLATE).expect("valid template")); + bar.set_prefix(""); + bar.set_message(""); + bar.set_length(0); + bar.reset(); + self.owner.free.lock().push(bar); + self.owner.refresh_shards_message(); + } +} - match outcome { - Ok(_) => { - if let Some(p) = progress.write().take() { - p.finish_and_clear() +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Core download implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Core download implementation shared by [`download_data`] and [`download_many`]. +/// +/// When `batch` is `Some`, the download reuses one of the pre-allocated slot bars from +/// the batch's [`MultiProgress`]. When `batch` is `None` the download renders its own +/// standalone bar. +async fn download_one(fname: PathBuf, url: &str, batch: Option<&BatchProgress>) -> Result { + let display_name = fname + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_owned(); + idempotent_async(&fname, async move |tmp_path| { + retry_get(&HTTP_CLIENT, url, &tmp_path, &display_name, batch).await + }) + .await +} + +/// Perform an HTTP GET into `tmp_path`, retrying up to three times with exponential +/// backoff and a small jitter to avoid lockstep retries across concurrent shards. A +/// partial temp file from an exhausted retry loop is removed before returning the final +/// error. +/// +/// When `batch` is `Some`, a pre-allocated slot bar is reused across retries. When +/// `batch` is `None`, a standalone [`ProgressBar`] is created and cleared at the end. +async fn retry_get( + client: &Client, + url: &str, + tmp_path: &Path, + display_name: &str, + batch: Option<&BatchProgress>, +) -> Result<()> { + const MAX_ATTEMPTS: u32 = 3; + let progress = DownloadProgress::new(batch, display_name).await; + let mut last_err: Option = None; + + for attempt in 0..MAX_ATTEMPTS { + if attempt > 0 { + progress.reset_for_retry(); + } + match single_attempt(client, url, tmp_path, &progress).await { + Ok(()) => { + if attempt == 0 + && let Some(b) = batch + { + b.report_clean_success(); } + progress.finalize(); return Ok(()); } Err(e) => { - if let Some(p) = progress.write().take() { - p.abandon() + if let Some(b) = batch { + b.report_retry(); } - last_err = Some(e) + last_err = Some(e); } } - let backoff = Duration::from_secs(1u64 << attempt); - warn!( - "download attempt {} failed; retrying in {:?}", - attempt + 1, - backoff - ); - tokio::time::sleep(backoff).await; + if attempt + 1 < MAX_ATTEMPTS { + sleep_with_jitter(attempt).await; + } } + + progress.finalize(); + cleanup_partial_temp(tmp_path); Err(last_err.unwrap_or_else(|| anyhow::anyhow!("retry_get exhausted with no recorded error"))) } -pub async fn download_data(fname: PathBuf, data_url: impl AsRef) -> Result { - let client = Client::builder() - .read_timeout(Duration::from_secs(60)) - .timeout(Duration::from_secs(60 * 15)) - .build() - .context("Failed to build HTTP client")?; +/// Perform one download attempt end to end: create the temp file, issue the GET, stream +/// bytes to disk while advancing the progress bar. Returns on the first error so the +/// retry loop in [`retry_get`] can decide whether to try again. +async fn single_attempt( + client: &Client, + url: &str, + tmp_path: &Path, + progress: &DownloadProgress, +) -> Result<()> { + let mut file = TokioFile::create(tmp_path) + .await + .context("Failed to create file")?; + let response = client + .get(url) + .send() + .await + .context("Failed to send HTTP request")? + .error_for_status() + .context("HTTP request returned error status")?; - idempotent_async(&fname, async |path| { - let url = data_url.as_ref(); - info!( - "Downloading {} from {}", - fname.to_str().context("Failed to convert path to string")?, - url + progress.activate(response.content_length()); + + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + AsyncWriteExt::write_all(&mut file, &chunk) + .await + .context("Failed to write to file")?; + progress.inc(chunk.len() as u64); + } + + AsyncWriteExt::flush(&mut file).await?; + Ok(()) +} + +/// Sleep `2^attempt` seconds plus 0-500 ms of jitter before the next retry. +async fn sleep_with_jitter(attempt: u32) { + let jitter = Duration::from_millis(rand::random::() % 500); + let backoff = Duration::from_secs(1u64 << attempt) + jitter; + warn!( + "download attempt {} failed; retrying in {:?}", + attempt + 1, + backoff + ); + tokio::time::sleep(backoff).await; +} + +/// Best-effort removal of a partial temp file left behind when every retry attempt +/// failed. The UUID-named temp lives under `target/`; leaking it would be mostly +/// harmless but adds up over many CI runs. +fn cleanup_partial_temp(tmp_path: &Path) { + if let Err(err) = std::fs::remove_file(tmp_path) { + warn!( + "failed to remove leftover temp download {}: {}", + tmp_path.display(), + err ); - retry_get( - async || { - let res = client.get(url).send().await?; - Ok(res) - }, - path, - ) - .await - }) - .await + } } -pub fn decompress_bz2(input_path: PathBuf, output_path: PathBuf) -> Result { - idempotent(&output_path, |path| { - info!( - "Decompressing bzip from {} to {}", - input_path - .to_str() - .context("Failed to convert input path to string")?, - output_path - .to_str() - .context("Failed to convert output path to string")? +/// Unified progress handle for a single download. Hides the split between pooled slot +/// bars (batched path) and one-off standalone bars (single-download path) so +/// [`retry_get`] does not have to branch on every update. +enum DownloadProgress { + Slot(SlotGuard), + Standalone(ProgressBar), +} + +impl DownloadProgress { + async fn new(batch: Option<&BatchProgress>, display_name: &str) -> Self { + match batch { + Some(b) => Self::Slot(b.acquire(display_name).await), + None => Self::Standalone(new_standalone_bar(display_name)), + } + } + + fn reset_for_retry(&self) { + match self { + Self::Slot(s) => s.reset_for_retry(), + Self::Standalone(bar) => { + bar.set_style( + ProgressStyle::with_template(CONNECTING_TEMPLATE).expect("valid template"), + ); + bar.set_length(0); + bar.reset(); + } + } + } + + fn activate(&self, content_length: Option) { + match (self, content_length) { + (Self::Slot(s), Some(total)) => s.activate_known(total), + (Self::Slot(s), None) => s.activate_unknown(), + (Self::Standalone(bar), Some(total)) => { + bar.set_style( + ProgressStyle::with_template(KNOWN_SIZE_TEMPLATE).expect("valid template"), + ); + bar.set_length(total); + bar.reset(); + } + (Self::Standalone(bar), None) => { + bar.set_style( + ProgressStyle::with_template(UNKNOWN_SIZE_TEMPLATE).expect("valid template"), + ); + bar.set_length(0); + bar.reset(); + } + } + } + + fn inc(&self, n: u64) { + match self { + Self::Slot(s) => s.inc(n), + Self::Standalone(bar) => bar.inc(n), + } + } + + /// Tear down any visible state. Standalone bars are explicitly cleared here; + /// slot bars clean themselves up when their [`SlotGuard`] drops. + fn finalize(&self) { + if let Self::Standalone(bar) = self { + bar.finish_and_clear(); + } + } +} + +fn new_standalone_bar(display_name: &str) -> ProgressBar { + let bar = ProgressBar::new(0); + bar.set_style(ProgressStyle::with_template(CONNECTING_TEMPLATE).expect("valid template")); + bar.set_prefix(display_name.to_owned()); + bar.enable_steady_tick(SLOT_TICK); + bar +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod tests { + use super::*; + + const COOLDOWN_MS: u64 = HALVE_COOLDOWN_MS; + + #[test] + fn ramp_up_doubles_in_slow_start() { + // Start at INITIAL (4). Each clean success in slow-start doubles until MAX. + let mut cur = INITIAL_IN_FLIGHT; + let expected = [8, 16, 32, 64, 128, 256]; + for want in expected { + let next = decide_on_success(cur, true).expect("should ramp"); + assert_eq!(next, want); + cur = next; + } + // At MAX, further successes are no-ops. + assert_eq!(cur, MAX_IN_FLIGHT); + assert_eq!(decide_on_success(cur, true), None); + assert_eq!(decide_on_success(cur, false), None); + } + + #[test] + fn additive_increase_after_slow_start_exits() { + // Once out of slow-start, successes add 1 instead of doubling. + assert_eq!(decide_on_success(16, false), Some(17)); + assert_eq!(decide_on_success(17, false), Some(18)); + } + + #[test] + fn retry_halves() { + // At 64, a single retry (past the cooldown) halves to 32. + assert_eq!(decide_on_retry(64, COOLDOWN_MS + 1, 0), Some(32)); + assert_eq!(decide_on_retry(32, COOLDOWN_MS + 1, 0), Some(16)); + assert_eq!(decide_on_retry(2, COOLDOWN_MS + 1, 0), Some(1)); + } + + #[test] + fn halve_is_debounced() { + // Three retries at t=100, t=200, t=500 (all within the 1 s cooldown after the + // first halve at t=100) only produce one halve. + let last_halve = 100; + assert_eq!(decide_on_retry(64, 200, last_halve), None); + assert_eq!(decide_on_retry(64, 500, last_halve), None); + // A retry past the cooldown halves again. + assert_eq!( + decide_on_retry(64, last_halve + COOLDOWN_MS + 1, last_halve), + Some(32) ); - let input_file = File::open(&input_path) - .with_context(|| format!("Failed to open input file: {:?}", input_path))?; - let mut decoder = BzDecoder::new(input_file); + } - let mut buffer = Vec::new(); - decoder - .read_to_end(&mut buffer) - .context("Failed to decompress bzip2 data")?; + #[test] + fn halve_respects_min_floor() { + // At MIN (1), retries are no-ops — we never go below 1. + assert_eq!(decide_on_retry(MIN_IN_FLIGHT, COOLDOWN_MS + 1, 0), None); + // At 2, halving to 1 is the last step. + assert_eq!(decide_on_retry(2, COOLDOWN_MS + 1, 0), Some(1)); + } - let mut output_file = File::create(path) - .with_context(|| format!("Failed to create output file: {:?}", path))?; - output_file - .write_all(&buffer) - .context("Failed to write decompressed data")?; - Ok(output_path.clone()) - }) + #[test] + fn ramp_up_respects_max_cap() { + // Even from a large `current`, we never exceed MAX. + assert_eq!( + decide_on_success(MAX_IN_FLIGHT - 1, true), + Some(MAX_IN_FLIGHT) + ); + assert_eq!(decide_on_success(MAX_IN_FLIGHT, true), None); + // Additive at the cap is also a no-op. + assert_eq!(decide_on_success(MAX_IN_FLIGHT, false), None); + } } diff --git a/vortex-bench/src/fineweb/mod.rs b/vortex-bench/src/fineweb/mod.rs index 743e7737313..5acb57f64e9 100644 --- a/vortex-bench/src/fineweb/mod.rs +++ b/vortex-bench/src/fineweb/mod.rs @@ -3,15 +3,14 @@ use std::path::PathBuf; -use tokio::io::AsyncWriteExt; use tracing::info; use url::Url; use crate::Benchmark; use crate::BenchmarkDataset; -use crate::IdempotentPath; use crate::TableSpec; -use crate::idempotent_async; +use crate::datasets::data_downloads::download_data; +use crate::utils::file::resolve_data_url; /// URL to the sample file const SAMPLE_URL: &str = "https://huggingface.co/datasets/HuggingFaceFW/fineweb/resolve/v1.4.0/sample/10BT/001_00000.parquet"; @@ -56,30 +55,7 @@ impl FinewebBenchmark { } fn create_data_url(remote_data_dir: &Option) -> anyhow::Result { - match remote_data_dir { - None => { - let data_dir = "fineweb".to_data_path(); - Url::from_directory_path(&data_dir).map_err(|_| { - anyhow::anyhow!("Failed to create URL from directory path: {:?}", &data_dir) - }) - } - Some(remote_data_dir) => { - if !remote_data_dir.ends_with("/") { - tracing::warn!( - "Supply a --use-remote-data-dir argument which ends in a slash e.g. s3://vortex-bench-dev-eu/develop/12345/fineweb/" - ); - } - tracing::info!( - concat!( - "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\n", - "If it does not, you should kill this command, locally generate the files (by running without\n", - "--use-remote-data-dir) and upload data/fineweb/ to some remote location.", - ), - remote_data_dir, - ); - Ok(Url::parse(remote_data_dir)?) - } - } + resolve_data_url(remote_data_dir.as_deref(), "fineweb") } } @@ -104,27 +80,7 @@ impl Benchmark for FinewebBenchmark { return Ok(()); } - let parquet = idempotent_async(&self.parquet_path()?, |parquet_path| async move { - info!("Downloading FineWeb Parquet source from HuggingFace"); - - let response = reqwest::get(SAMPLE_URL) - .await? - .error_for_status() - .map_err(|err| { - anyhow::anyhow!("error fetching fineweb sample from HuggingFace: {err}") - })?; - - let bytes = response.bytes().await?; - let mut w = tokio::fs::File::create(parquet_path).await?; - - w.write_all(&bytes).await?; - - w.flush().await?; - - Ok(()) - }) - .await?; - + let parquet = download_data(self.parquet_path()?, SAMPLE_URL).await?; info!("fineweb base data generated in {}", parquet.display()); Ok(()) diff --git a/vortex-bench/src/public_bi.rs b/vortex-bench/src/public_bi.rs index 3602cd2cd33..fe937876275 100644 --- a/vortex-bench/src/public_bi.rs +++ b/vortex-bench/src/public_bi.rs @@ -43,7 +43,7 @@ use crate::TableSpec; use crate::conversions::parquet_to_vortex_chunks; use crate::datasets::Dataset; use crate::datasets::data_downloads::decompress_bz2; -use crate::datasets::data_downloads::download_data; +use crate::datasets::data_downloads::download_many; use crate::idempotent_async; use crate::workspace_root; @@ -289,16 +289,13 @@ pub struct PBIData { impl PBIData { async fn download_bzips(&self) -> anyhow::Result<()> { - let download_futures = self.tables.iter().map(|table| { - download_data( + let downloads = self.tables.iter().map(|table| { + ( self.get_file_path(&table.name, FileType::CsvBzip2), - table.data_url.as_str(), + table.data_url.as_str().to_owned(), ) }); - let results = join_all(download_futures).await; - for result in results { - result?; - } + download_many(downloads).await?; Ok(()) } diff --git a/vortex-bench/src/realnest/gharchive.rs b/vortex-bench/src/realnest/gharchive.rs index 154cf945185..86b981a9fbe 100644 --- a/vortex-bench/src/realnest/gharchive.rs +++ b/vortex-bench/src/realnest/gharchive.rs @@ -14,10 +14,10 @@ use url::Url; use crate::Benchmark; use crate::BenchmarkDataset; -use crate::IdempotentPath; use crate::TableSpec; use crate::idempotent; use crate::idempotent_async; +use crate::utils::file::resolve_data_url; /// Template URL for raw JSON dataset fn raw_json_url(hour: usize) -> String { @@ -48,30 +48,7 @@ impl GithubArchiveBenchmark { } fn create_data_url(remote_data_dir: &Option) -> anyhow::Result { - match remote_data_dir { - None => { - let data_dir = "gharchive".to_data_path(); - Url::from_directory_path(&data_dir).map_err(|_| { - anyhow::anyhow!("Failed to create URL from directory path: {:?}", &data_dir) - }) - } - Some(remote_data_dir) => { - if !remote_data_dir.ends_with("/") { - tracing::warn!( - "Supply a --use-remote-data-dir argument which ends in a slash e.g. s3://vortex-bench-dev-eu/develop/12345/gharchive/" - ); - } - tracing::info!( - concat!( - "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\n", - "If it does not, you should kill this command, locally generate the files (by running without\n", - "--use-remote-data-dir) and upload data/gharchive/ to some remote location.", - ), - remote_data_dir, - ); - Ok(Url::parse(remote_data_dir)?) - } - } + resolve_data_url(remote_data_dir.as_deref(), "gharchive") } } diff --git a/vortex-bench/src/utils/file.rs b/vortex-bench/src/utils/file.rs index b6d202bb29a..c8916d68398 100644 --- a/vortex-bench/src/utils/file.rs +++ b/vortex-bench/src/utils/file.rs @@ -110,6 +110,48 @@ impl IdempotentPath for &Path { } } +/// Resolve the `--use-remote-data-dir` CLI option to a `Url` for a named dataset. +/// +/// When `remote_data_dir` is `None`, returns a `file://` URL pointing at the dataset's local cache +/// directory (`//`). +/// +/// When `remote_data_dir` is `Some(...)`, parses it as a remote URL (typically `s3://` or `gs://`). +/// The user must have pre-uploaded the expected data layout; a warning is logged if the URL does +/// not end in `/`, and an informational message describes the expected layout. +/// +/// This helper replaces the boilerplate `create_data_url()` that used to be duplicated across every +/// benchmark that supports remote data directories (ClickBench, Fineweb, GhArchive, ...). +pub fn resolve_data_url(remote_data_dir: Option<&str>, local_subdir: &str) -> Result { + match remote_data_dir { + None => { + let data_dir = data_dir().join(local_subdir); + Url::from_directory_path(&data_dir).map_err(|_| { + anyhow::anyhow!("Failed to create URL from directory path: {:?}", &data_dir) + }) + } + Some(remote_data_dir) => { + if !remote_data_dir.ends_with('/') { + tracing::warn!( + "Supply a --use-remote-data-dir argument which ends in a slash \ + e.g. s3://vortex-bench-dev-eu/develop/12345/{}/", + local_subdir, + ); + } + tracing::info!( + concat!( + "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\n", + "If it does not, you should kill this command, locally generate the files ", + "(by running without\n", + "--use-remote-data-dir) and upload data/{}/ to some remote location.", + ), + remote_data_dir, + local_subdir, + ); + Ok(Url::parse(remote_data_dir)?) + } + } +} + /// Convert a URL scheme to a storage type string /// /// Maps URL schemes (s3, gcs, file) to storage type identifiers diff --git a/vortex-bench/src/vector_dataset/download.rs b/vortex-bench/src/vector_dataset/download.rs index 228c51db771..a7b7d8c6e89 100644 --- a/vortex-bench/src/vector_dataset/download.rs +++ b/vortex-bench/src/vector_dataset/download.rs @@ -17,11 +17,9 @@ use std::path::PathBuf; use anyhow::Context; use anyhow::Result; -use tokio::task::JoinSet; -use tracing::info; use crate::datasets::data_downloads::download_data; -use crate::utils::file::idempotent_async; +use crate::datasets::data_downloads::download_many; use crate::vector_dataset::catalog::VectorDataset; use crate::vector_dataset::layout::LayoutSpec; use crate::vector_dataset::layout::TrainLayout; @@ -95,28 +93,23 @@ pub struct DatasetPaths { /// This has idempotent semantics, so files already present on disk are skipped, and re-runs only /// pay for new files. /// -/// Train shards download in parallel using a shared HTTP client; the small `test.parquet` and -/// `neighbors.parquet` files use the simple [`download_data`] helper. +/// Train shards download via [`download_many`] with adaptive parallelism; the small +/// `test.parquet` and `neighbors.parquet` files use the simple [`download_data`] helper. +/// All HTTP requests share a single pooled client. pub async fn download(ds: VectorDataset, layout: TrainLayout) -> Result { let spec = ds.validate_layout(layout)?; let urls = train_urls(ds, spec); let train_targets = paths::train_files(ds, layout, spec.num_files()); debug_assert_eq!(urls.len(), train_targets.len()); - let mut tasks: JoinSet> = JoinSet::new(); - for (url, target) in urls.into_iter().zip(train_targets.iter().cloned()) { - tasks.spawn(async move { - idempotent_async(target, |tmp| async move { - info!("downloading {}", url); - download_data(tmp, &url).await - }) - .await?; - Ok(()) - }); - } - while let Some(joined) = tasks.join_next().await { - joined.context("train download task panicked")??; - } + let train_downloads: Vec<(PathBuf, String)> = train_targets + .iter() + .cloned() + .zip(urls.into_iter()) + .collect(); + download_many(train_downloads) + .await + .with_context(|| format!("download train shards for {}", ds.name()))?; let test = download_data(paths::test_path(ds, layout), &test_url(ds)) .await From 34509f2ecd21c894215a30d2e28b430fe5e28f21 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:28:38 -0400 Subject: [PATCH 091/250] Optimize L2Norm for ConstantArray (#7495) ## Summary Optimizes L2Norm for ConstantArray, and makes sure we dont hit that tracing info case. ## Testing Some basic tests. Signed-off-by: Connor Tsui --- vortex-tensor/src/scalar_fns/l2_norm.rs | 115 +++++++++++++++++++++++- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index cd77da65f3c..59a2e5a45b1 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -10,6 +10,8 @@ use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::arrays::Constant; +use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; @@ -26,6 +28,7 @@ use vortex_array::dtype::Nullability; use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::match_each_float_ptype; +use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; @@ -131,6 +134,8 @@ impl ScalarFnVTable for L2Norm { let tensor_flat_size = tensor_match.list_size(); let element_ptype = tensor_match.element_ptype(); + let norm_dtype = DType::Primitive(element_ptype, ext.nullability()); + // L2Norm(L2Denorm(normalized, norms)) is defined to read back the authoritative stored // norms. Exact callers of lossy encodings like TurboQuant opt into that storage semantics // instead of forcing a decode-and-recompute path here. @@ -139,14 +144,37 @@ impl ScalarFnVTable for L2Norm { .nth_child(1) .vortex_expect("L2Denom must have at 2 children"); - vortex_ensure_eq!( - norms.dtype(), - &DType::Primitive(element_ptype, input_ref.dtype().nullability()) - ); + vortex_ensure_eq!(norms.dtype(), &norm_dtype); return Ok(norms); } + // Optimize for the constant array case. + if let Some(array) = input_ref.as_opt::() { + let scalar = array.scalar().as_extension().to_storage_scalar(); + + let Some(elements) = scalar.as_list().elements() else { + return Ok(ConstantArray::new(Scalar::null(norm_dtype), row_count).into_array()); + }; + + let norm_scalar = match_each_float_ptype!(element_ptype, |T| { + let values: Vec = elements + .iter() + .map(|s| { + s.as_primitive() + .as_::() + .vortex_expect("element was somehow not the correct float") + }) + .collect(); + let norm = l2_norm_row::(&values); + + Scalar::try_new(norm_dtype, Some(norm.into())) + })?; + + let norms = ConstantArray::new(norm_scalar, row_count).into_array(); + return Ok(norms); + } + let input: ExtensionArray = input_ref.execute(ctx)?; let validity = input.as_ref().validity()?; @@ -244,10 +272,18 @@ mod tests { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; + use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::extension::ExtDType; + use vortex_array::extension::EmptyMetadata; + use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_error::VortexResult; @@ -256,6 +292,7 @@ mod tests { use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; + use crate::vector::Vector; /// Evaluates L2 norm on a tensor/vector array and returns the result as `Vec`. fn eval_l2_norm(input: ArrayRef, len: usize) -> VortexResult> { @@ -326,6 +363,76 @@ mod tests { Ok(()) } + /// Builds a [`ConstantArray`] whose scalar is a [`Vector`] extension scalar wrapping a + /// fixed-size list of `elements`, broadcast to `len` rows. + fn constant_vector_extension_array(elements: &[f64], len: usize) -> ArrayRef { + let element_dtype = DType::Primitive(PType::F64, Nullability::NonNullable); + let children: Vec = elements + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = + Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + let ext_scalar = Scalar::extension::(EmptyMetadata, storage_scalar); + ConstantArray::new(ext_scalar, len).into_array() + } + + /// A constant input whose scalar is a non-null tensor should short-circuit to a + /// [`ConstantArray`] output whose scalar is the precomputed norm. Uses [`execute_until`] so + /// execution stops at the [`Constant`] encoding instead of canonicalizing into a + /// [`PrimitiveArray`]. + #[test] + fn constant_non_null_input_yields_constant_output() -> VortexResult<()> { + let input = constant_vector_extension_array(&[3.0, 4.0], 4); + + let scalar_fn = L2Norm::new().erased(); + let result = ScalarFnArray::try_new(scalar_fn, vec![input], 4)?.into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let output = result.execute_until::(&mut ctx)?; + + let constant = output + .as_opt::() + .expect("L2Norm over a constant input must produce a constant output"); + assert_eq!(constant.len(), 4); + let norm = constant + .scalar() + .as_primitive() + .as_::() + .expect("norm scalar must be a non-null primitive"); + assert_close(&[norm], &[5.0]); + Ok(()) + } + + /// A constant input whose scalar is null should short-circuit to a null [`ConstantArray`] of + /// the correct primitive dtype and length. + #[test] + fn constant_null_input_yields_null_constant_output() -> VortexResult<()> { + let storage_dtype = DType::FixedSizeList( + DType::Primitive(PType::F64, Nullability::NonNullable).into(), + 2, + Nullability::Nullable, + ); + let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage_dtype)?.erased(); + let null_scalar = Scalar::null(DType::Extension(ext_dtype)); + let input = ConstantArray::new(null_scalar, 3).into_array(); + + let scalar_fn = L2Norm::new().erased(); + let result = ScalarFnArray::try_new(scalar_fn, vec![input], 3)?.into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let output = result.execute_until::(&mut ctx)?; + + let constant = output + .as_opt::() + .expect("null constant input must produce a constant output"); + assert_eq!(constant.len(), 3); + assert!(constant.scalar().is_null()); + assert_eq!( + constant.dtype(), + &DType::Primitive(PType::F64, Nullability::Nullable) + ); + Ok(()) + } + #[rstest] #[case::fixed_shape_tensor(tensor_array(&[3], &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] #[case::vector(vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] From ce52b71852b32a8ed2aa0d309cccda00339498bf Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:30:22 -0400 Subject: [PATCH 092/250] Actual kebab case (#7496) ## Summary Clap's rename all for kebab case macro doesn't work for numbers. ## Testing N/A Signed-off-by: Connor Tsui --- vortex-bench/src/vector_dataset/catalog.rs | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/vortex-bench/src/vector_dataset/catalog.rs b/vortex-bench/src/vector_dataset/catalog.rs index 5549add0c14..f34eaf812ec 100644 --- a/vortex-bench/src/vector_dataset/catalog.rs +++ b/vortex-bench/src/vector_dataset/catalog.rs @@ -48,6 +48,8 @@ pub const ALL_VECTOR_DATASETS: &[VectorDataset] = &[ VectorDataset::LaionLarge100m, ]; +// NB: We can't do `#[clap(rename_all = "kebab-case")]` here because it won't put a dash in front of +// any numbers. /// The publicly hosted vector benchmark datasets. /// /// Variants are named ``, kebab-cased on the CLI (e.g. `cohere-large-10m`). @@ -55,47 +57,62 @@ pub const ALL_VECTOR_DATASETS: &[VectorDataset] = &[ /// The static metadata for each variant (dimensionality, row count, hosted layouts, etc.) is /// exposed via the inherent methods below; the full table is reachable via [`ALL_VECTOR_DATASETS`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ValueEnum)] -#[clap(rename_all = "kebab-case")] pub enum VectorDataset { /// Cohere wiki-22-12, 100K × 768 f32, cosine. Single + SingleShuffled. + #[clap(name = "cohere-small-100k")] CohereSmall100k, /// Cohere wiki-22-12, 1M × 768 f32, cosine. Single + SingleShuffled. + #[clap(name = "cohere-medium-1m")] CohereMedium1m, /// Cohere wiki-22-12, 10M × 768 f32, cosine. Partitioned + PartitionedShuffled (10 shards). + #[clap(name = "cohere-large-10m")] CohereLarge10m, /// OpenAI embeddings on C4, 50K × 1536 f64, cosine. Single + SingleShuffled. + #[clap(name = "openai-small-50k")] OpenaiSmall50k, /// OpenAI embeddings on C4, 500K × 1536 f64, cosine. Single + SingleShuffled. + #[clap(name = "openai-medium-500k")] OpenaiMedium500k, /// OpenAI embeddings on C4, 5M × 1536 f64, cosine. Partitioned + PartitionedShuffled (10 /// shards). + #[clap(name = "openai-large-5m")] OpenaiLarge5m, /// Bioasq biomedical, 1M × 1024 f32, cosine. SingleShuffled only. + #[clap(name = "bioasq-medium-1m")] BioasqMedium1m, /// Bioasq biomedical, 10M × 1024 f32, cosine. PartitionedShuffled only (10 shards). + #[clap(name = "bioasq-large-10m")] BioasqLarge10m, /// GloVe word vectors, 100K × 200 f32, cosine. Single only. No neighbors / labels. + #[clap(name = "glove-small-100k")] GloveSmall100k, /// GloVe word vectors, 1M × 200 f32, cosine. Single only. No neighbors / labels. + #[clap(name = "glove-medium-1m")] GloveMedium1m, /// GIST image features, 100K × 960 f32, L2. Single only. No neighbors / labels. + #[clap(name = "gist-small-100k")] GistSmall100k, /// GIST image features, 1M × 960 f32, L2. Single only. No neighbors / labels. + #[clap(name = "gist-medium-1m")] GistMedium1m, /// SIFT image features, 500K × 128 f32, L2. Single only. No neighbors / labels. + #[clap(name = "sift-small-500k")] SiftSmall500k, /// SIFT image features, 5M × 128 f32, L2. Single only. No neighbors / labels. + #[clap(name = "sift-medium-5m")] SiftMedium5m, /// SIFT image features, 50M × 128 f32, L2. Partitioned only (50 shards). No labels. + #[clap(name = "sift-large-50m")] SiftLarge50m, /// LAION image embeddings, 100M × 768 f32, L2. Partitioned only (100 shards). /// Has `neighbors.parquet` and `scalar_labels.parquet`. + #[clap(name = "laion-large-100m")] LaionLarge100m, } @@ -305,12 +322,6 @@ impl VectorDataset { } } } - - /// Pick the default layout for this dataset — the first entry in [`Self::layouts`]. - /// Stable across runs since the catalog table is statically ordered. - pub fn default_layout(&self) -> LayoutSpec { - self.layouts()[0] - } } #[cfg(test)] From 869b2d11acc23adeaaeebc7c299f9083a60874c0 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 04:50:38 -0400 Subject: [PATCH 093/250] Add extension constant pushdown rule and fix `InnerProduct` rule (#7507) ## Summary Most of our optimizer rules interact with `ConstantArray`, as often there is an optimized implementation when a child is known to be constant. We can have both of these arrays that mean the same thing, but one is not easily matched on: ``` ConstantArray( scalar: Scalar::Extension(...) ) ExtensionArray( storage: ConstantArray( Scalar::* ) ) ``` This change adds a rule to convert the second representation into the first. It additionally fixes a bug in the `InnerProduct` reduction (right now it is in `execute` but it should really be a reduction rule, that will come in the future) where it checks for the second case instead of the first. ## Testing Basic tests. Signed-off-by: Connor Tsui --- .../src/arrays/extension/compute/rules.rs | 61 ++++++++++++++ vortex-tensor/src/scalar_fns/inner_product.rs | 82 +++++++++++++------ 2 files changed, 118 insertions(+), 25 deletions(-) diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 6a58e4838be..7408488a0f1 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -6,18 +6,23 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; +use crate::arrays::Constant; +use crate::arrays::ConstantArray; use crate::arrays::Extension; use crate::arrays::ExtensionArray; use crate::arrays::Filter; use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; +use crate::matcher::AnyArray; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ParentRuleSet; +use crate::scalar::Scalar; use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ + ParentRuleSet::lift(&ExtensionConstantParentRule), ParentRuleSet::lift(&ExtensionFilterPushDownRule), ParentRuleSet::lift(&CastReduceAdaptor(Extension)), ParentRuleSet::lift(&FilterReduceAdaptor(Extension)), @@ -25,6 +30,36 @@ pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&SliceReduceAdaptor(Extension)), ]); +/// Normalize `Extension(Constant(storage))` children to `Constant(Extension(storage))`. +#[derive(Debug)] +struct ExtensionConstantParentRule; + +impl ArrayParentReduceRule for ExtensionConstantParentRule { + type Parent = AnyArray; + + fn reduce_parent( + &self, + child: ArrayView<'_, Extension>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + let Some(const_array) = child.storage_array().as_opt::() else { + return Ok(None); + }; + + let storage_scalar = const_array.scalar().clone(); + let ext_scalar = Scalar::extension_ref(child.ext_dtype().clone(), storage_scalar); + + let constant_with_extension_scalar = + ConstantArray::new(ext_scalar, child.len()).into_array(); + + parent + .clone() + .with_slot(child_idx, constant_with_extension_scalar) + .map(Some) + } +} + /// Push filter operations into the storage array of an extension array. #[derive(Debug)] struct ExtensionFilterPushDownRule; @@ -58,6 +93,7 @@ mod tests { use crate::IntoArray; #[expect(deprecated)] use crate::ToCanonical as _; + use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Extension; use crate::arrays::ExtensionArray; @@ -177,6 +213,31 @@ mod tests { assert_eq!(canonical.len(), 3); } + #[test] + fn test_extension_constant_child_normalizes_under_scalar_fn() { + let ext_dtype = test_ext_dtype(); + + let constant_storage = ConstantArray::new(Scalar::from(10i64), 3).into_array(); + let constant_ext = ExtensionArray::new(ext_dtype.clone(), constant_storage).into_array(); + + let storage = buffer![15i64, 25, 35].into_array(); + let ext_array = ExtensionArray::new(ext_dtype, storage).into_array(); + + let scalar_fn_array = Binary + .try_new_array(3, Operator::Lt, [constant_ext, ext_array]) + .unwrap(); + + let optimized = scalar_fn_array.optimize().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); + let children = scalar_fn.children(); + let constant = children[0] + .as_opt::() + .expect("constant extension child should be normalized"); + + assert!(constant.scalar().as_extension_opt().is_some()); + assert_eq!(constant.len(), 3); + } + #[test] fn test_scalar_fn_no_pushdown_different_ext_types() { #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 5928335ccf8..dd9c2a7381f 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -448,18 +448,10 @@ impl InnerProduct { return Ok(None); } - // The other side must be a constant-backed tensor-like extension whose scalar is - // non-null. - let Some(const_ext) = const_ref.as_opt::() else { + // The other side must be a constant tensor. + let Some(const_storage) = constant_tensor_storage(const_ref) else { return Ok(None); }; - let const_storage = const_ext.storage_array(); - let Some(const_backing) = const_storage.as_opt::() else { - return Ok(None); - }; - if const_backing.scalar().is_null() { - return Ok(None); - } let dim = sorf_view.options.dimension as usize; let num_rounds = sorf_view.options.num_rounds as usize; @@ -467,7 +459,7 @@ impl InnerProduct { let padded_dim = dim.next_power_of_two(); // Extract the single stored row of the constant via the stride-0 short-circuit. - let flat = extract_flat_elements(const_storage, dim, ctx)?; + let flat = extract_flat_elements(&const_storage, dim, ctx)?; if flat.ptype() != PType::F32 { // TODO(connor): as above, f16/f64 are not supported by this rewrite yet. The // standard path handles them correctly. @@ -482,9 +474,9 @@ impl InnerProduct { let mut rotated_query = vec![0.0f32; padded_dim]; rotation.rotate(&padded_query, &mut rotated_query); - // Build the rewritten constant as a `Vector` extension wrapping a - // `ConstantArray` of length `len`. We reuse the original storage FSL nullability so - // the new extension dtype stays consistent with whatever the original tree expected. + // Build the rewritten constant as a `Vector` extension scalar. We reuse + // the original storage FSL nullability so the new extension dtype stays consistent with + // whatever the original tree expected. let storage_fsl_nullability = const_storage.dtype().nullability(); let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); let children: Vec = rotated_query @@ -493,7 +485,6 @@ impl InnerProduct { .collect(); let fsl_scalar = Scalar::fixed_size_list(element_dtype.clone(), children, storage_fsl_nullability); - let new_storage = ConstantArray::new(fsl_scalar, len).into_array(); // Build a fresh `Vector` extension dtype. We cannot reuse the // original extension dtype because that one has `dim`, not `padded_dim`. @@ -504,7 +495,8 @@ impl InnerProduct { storage_fsl_nullability, ); let new_ext_dtype = ExtDType::::try_new(EmptyMetadata, new_fsl_dtype)?.erased(); - let new_constant = ExtensionArray::new(new_ext_dtype, new_storage).into_array(); + let new_constant = + ConstantArray::new(Scalar::extension_ref(new_ext_dtype, fsl_scalar), len).into_array(); // Extract the SorfTransform child (the already-padded Vector). let sorf_child = sorf_view @@ -572,16 +564,9 @@ impl InnerProduct { }; // Navigate the constant side and require its scalar be non-null. - let Some(const_ext) = const_candidate.as_opt::() else { + let Some(const_storage) = constant_tensor_storage(const_candidate) else { return Ok(None); }; - let const_storage = const_ext.storage_array(); - let Some(const_backing) = const_storage.as_opt::() else { - return Ok(None); - }; - if const_backing.scalar().is_null() { - return Ok(None); - } // Canonicalize codes and values. Codes may be e.g. BitPacked; executing is cheaper // than falling through to the standard path (which would also canonicalize). @@ -602,7 +587,7 @@ impl InnerProduct { let padded_dim = usize::try_from(fsl.list_size()).vortex_expect("fsl list_size fits usize"); - let flat = extract_flat_elements(const_storage, padded_dim, ctx)?; + let flat = extract_flat_elements(&const_storage, padded_dim, ctx)?; if flat.ptype() != PType::F32 { // TODO(connor): case 2 is f32-only. For f16/f64 we fall through to the standard // path, which computes the inner product with the correct element type. @@ -637,6 +622,16 @@ impl InnerProduct { } } +/// Return the storage constant for a canonical tensor-like constant query. +fn constant_tensor_storage(array: &ArrayRef) -> Option { + let constant = array.as_opt::()?; + if constant.scalar().is_null() { + return None; + } + let ext_scalar = constant.scalar().as_extension_opt()?; + Some(ConstantArray::new(ext_scalar.to_storage_scalar(), array.len()).into_array()) +} + /// Computes the inner product (dot product) of two equal-length float slices. /// /// Returns `sum(a_i * b_i)`. @@ -959,6 +954,7 @@ mod tests { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; @@ -978,9 +974,11 @@ mod tests { use vortex_session::VortexSession; use crate::scalar_fns::inner_product::InnerProduct; + use crate::scalar_fns::inner_product::constant_tensor_storage; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; + use crate::utils::extract_flat_elements; use crate::vector::Vector; static SESSION: LazyLock = @@ -1011,6 +1009,19 @@ mod tests { Ok(ExtensionArray::new(ext_dtype, storage).into_array()) } + /// Expression-literal shape: a ConstantArray whose scalar itself is a Vector extension. + fn literal_vector_f32(elements: &[f32], len: usize) -> ArrayRef { + let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); + let children: Vec = elements + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = + Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + let vector_scalar = Scalar::extension::(EmptyMetadata, storage_scalar); + ConstantArray::new(vector_scalar, len).into_array() + } + /// Build an `ExtensionArray>` whose storage is /// `FSL(DictArray(codes: u8, values: f32))`. This mirrors the shape that /// TurboQuant produces as the SorfTransform child. @@ -1115,6 +1126,27 @@ mod tests { // ---- Case 1: SorfTransform + Constant pull-through ---- + #[test] + fn constant_tensor_storage_accepts_extension_scalar_literal() -> VortexResult<()> { + let literal = literal_vector_f32(&[1.0, 2.0, 3.0], 5); + let storage = + constant_tensor_storage(&literal).expect("literal vector should be recognized"); + + assert_eq!(storage.len(), 5); + let const_storage = storage + .as_opt::() + .expect("storage should remain constant-backed"); + assert!(matches!( + const_storage.scalar().dtype(), + DType::FixedSizeList(_, 3, Nullability::NonNullable) + )); + + let mut ctx = SESSION.create_execution_ctx(); + let flat = extract_flat_elements(&storage, 3, &mut ctx)?; + assert_eq!(flat.row::(0), &[1.0, 2.0, 3.0]); + Ok(()) + } + /// Case 1: SorfTransform on LHS, constant query on RHS, with `dim < padded_dim` /// so the zero-padding branch is exercised. #[test] From cd798b3ae58a535774f3ec678f36f0208a601ee9 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 04:54:33 -0400 Subject: [PATCH 094/250] L2 denorm scheme (#7506) ## Summary We want to merge https://github.com/vortex-data/vortex/pull/7499 soon, but it is not a fair comparison because we do not normalize the uncompressed vectors flavor, but we do normalize the TurboQuant-encoded vectors. This just makes the comparison fair. I've marked this as semi-unstable (there is no `with_turboquant` equivalent on the compressor builder, but the ID is clearly unstable) just so we can get the benchmark in #7499 over the line in a reasonable state. ## Testing It already works for the TurboQuant scheme, this is just a reduction of that so we don't need to test this. Signed-off-by: Connor Tsui --- .../vector-search-bench/src/compression.rs | 11 +++- vortex-tensor/public-api.lock | 18 +++++++ vortex-tensor/src/encodings/l2_denorm.rs | 52 +++++++++++++++++++ vortex-tensor/src/encodings/mod.rs | 1 + 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 vortex-tensor/src/encodings/l2_denorm.rs diff --git a/benchmarks/vector-search-bench/src/compression.rs b/benchmarks/vector-search-bench/src/compression.rs index 6e2733cc1cd..65bf080db85 100644 --- a/benchmarks/vector-search-bench/src/compression.rs +++ b/benchmarks/vector-search-bench/src/compression.rs @@ -22,6 +22,7 @@ use vortex::session::VortexSession; use vortex::utils::aliases::hash_set::HashSet; use vortex_bench::Format; use vortex_btrblocks::BtrBlocksCompressorBuilder; +use vortex_tensor::encodings::l2_denorm::L2DenormScheme; use vortex_tensor::scalar_fns::l2_denorm::L2Denorm; use vortex_tensor::scalar_fns::sorf_transform::SorfTransform; @@ -76,10 +77,18 @@ impl VectorFlavor { pub fn create_write_options(&self, session: &VortexSession) -> VortexWriteOptions { let strategy = match self { VectorFlavor::Uncompressed => { - let compressor = BtrBlocksCompressorBuilder::empty().build(); + // Even though this is uncompressed, we still want to denormalize the data first so + // that the results are fair. + let compressor = BtrBlocksCompressorBuilder::empty() + .with_new_scheme(&L2DenormScheme) + .build(); + + let mut allowed: HashSet = ALLOWED_ENCODINGS.clone(); + allowed.insert(L2Denorm.id()); WriteStrategyBuilder::default() .with_compressor(compressor) + .with_allow_encodings(allowed) .build() } VectorFlavor::TurboQuant => { diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 99dc8f0edbd..96b95e1e91e 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -2,6 +2,24 @@ pub mod vortex_tensor pub mod vortex_tensor::encodings +pub mod vortex_tensor::encodings::l2_denorm + +pub struct vortex_tensor::encodings::l2_denorm::L2DenormScheme + +impl core::fmt::Debug for vortex_tensor::encodings::l2_denorm::L2DenormScheme + +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl vortex_compressor::scheme::Scheme for vortex_tensor::encodings::l2_denorm::L2DenormScheme + +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult + +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate + +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool + +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::scheme_name(&self) -> &'static str + pub mod vortex_tensor::encodings::turboquant pub struct vortex_tensor::encodings::turboquant::TurboQuantConfig diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs new file mode 100644 index 00000000000..172191abf6e --- /dev/null +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::IntoArray; +use vortex_compressor::CascadingCompressor; +use vortex_compressor::ctx::CompressorContext; +use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::EstimateVerdict; +use vortex_compressor::scheme::Scheme; +use vortex_compressor::stats::ArrayAndStats; +use vortex_error::VortexResult; + +use crate::matcher::AnyTensor; +use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; + +#[derive(Debug)] +pub struct L2DenormScheme; + +impl Scheme for L2DenormScheme { + // TODO(connor): FIX THIS!!! + fn scheme_name(&self) -> &'static str { + "vortex.tensor.UNSTABLE.l2_denorm" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches!( + canonical, + Canonical::Extension(ext) if ext.ext_dtype().is::() + ) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) + } + + fn compress( + &self, + compressor: &CascadingCompressor, + data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + let l2_denorm = + normalize_as_l2_denorm(data.array().clone(), &mut compressor.execution_ctx())?; + Ok(l2_denorm.into_array()) + } +} diff --git a/vortex-tensor/src/encodings/mod.rs b/vortex-tensor/src/encodings/mod.rs index 084baf97e57..22e57763171 100644 --- a/vortex-tensor/src/encodings/mod.rs +++ b/vortex-tensor/src/encodings/mod.rs @@ -6,4 +6,5 @@ // TODO(connor): // pub mod spherical; // Spherical transform on unit-normalized vectors. +pub mod l2_denorm; pub mod turboquant; From 6a2d4adb5f50c8c19f8753616097df3e1a72a321 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 17 Apr 2026 10:44:34 +0100 Subject: [PATCH 095/250] High level C Scan API (#7212) Expose data source, partition, and scan primitives through C with rust-side and C-side tests. Signed-off-by: Mikhail Kot --- Cargo.lock | 3 + vortex-ffi/CMakeLists.txt | 1 - vortex-ffi/Cargo.toml | 3 + vortex-ffi/README.md | 2 +- vortex-ffi/build.rs | 24 +- vortex-ffi/cbindgen.toml | 44 +- vortex-ffi/cinclude/vortex.h | 308 ++++++++++++- vortex-ffi/src/array.rs | 16 +- vortex-ffi/src/data_source.rs | 236 ++++++++++ vortex-ffi/src/dtype.rs | 22 + vortex-ffi/src/error.rs | 23 +- vortex-ffi/src/expression.rs | 6 +- vortex-ffi/src/lib.rs | 101 +++++ vortex-ffi/src/scan.rs | 728 ++++++++++++++++++++++++++++++ vortex-ffi/src/session.rs | 32 ++ vortex-ffi/src/struct_array.rs | 1 + vortex-ffi/test/CMakeLists.txt | 14 +- vortex-ffi/test/common.h | 27 +- vortex-ffi/test/scan.cpp | 782 +++++++++++++++++++++++++++++++++ 19 files changed, 2339 insertions(+), 34 deletions(-) create mode 100644 vortex-ffi/src/data_source.rs create mode 100644 vortex-ffi/src/scan.rs create mode 100644 vortex-ffi/test/scan.cpp diff --git a/Cargo.lock b/Cargo.lock index deacfaf7977..ad18bbb2cef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10564,6 +10564,8 @@ dependencies = [ name = "vortex-ffi" version = "0.1.0" dependencies = [ + "arrow-array 58.0.0", + "arrow-schema 58.0.0", "async-fs", "cbindgen", "futures", @@ -10572,6 +10574,7 @@ dependencies = [ "object_store 0.13.2", "paste", "prost 0.14.3", + "rand 0.8.5", "tempfile", "tracing", "tracing-subscriber", diff --git a/vortex-ffi/CMakeLists.txt b/vortex-ffi/CMakeLists.txt index 7bac7537211..dc6a231fc55 100644 --- a/vortex-ffi/CMakeLists.txt +++ b/vortex-ffi/CMakeLists.txt @@ -100,7 +100,6 @@ if (BUILD_TESTS) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wextra -Wpedantic") - enable_testing() add_subdirectory(test) endif() diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index 27f2c7b6140..c8daac6200d 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -20,6 +20,8 @@ categories = { workspace = true } all-features = true [dependencies] +arrow-array = { workspace = true, features = ["ffi"] } +arrow-schema = { workspace = true } async-fs = { workspace = true } futures = { workspace = true } itertools = { workspace = true } @@ -33,6 +35,7 @@ url = { workspace = true, features = [] } vortex = { workspace = true, features = ["object_store"] } [dev-dependencies] +rand = "0.8" tempfile = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/vortex-ffi/README.md b/vortex-ffi/README.md index 35911ccccc2..472bc89f71c 100644 --- a/vortex-ffi/README.md +++ b/vortex-ffi/README.md @@ -111,6 +111,6 @@ cmake -Bbuild -DWITH_ASAN=1 -DTARGET_TRIPLE= 3. Run the tests (ctest doesn't output failures in detail): -``` +```sh ./build/test/vortex_ffi_test 2>& 1 | rustfilt -i- ``` diff --git a/vortex-ffi/build.rs b/vortex-ffi/build.rs index 909ca578057..16e72db792a 100644 --- a/vortex-ffi/build.rs +++ b/vortex-ffi/build.rs @@ -8,31 +8,41 @@ use std::process::Command; use std::process::exit; fn main() { + println!("cargo:rustc-check-cfg=cfg(vortex_asan)"); println!("cargo:rerun-if-changed=src"); println!("cargo:rerun-if-changed=cbindgen.toml"); println!("cargo:rerun-if-changed=Cargo.toml"); println!("cargo:rerun-if-changed=build.rs"); - for env in ["MIRI", "MIRIFLAGS", "RUSTFLAGS"] { + for env in ["MIRI", "MIRIFLAGS", "CARGO_ENCODED_RUSTFLAGS"] { println!("cargo:rerun-if-env-changed={env}"); } - if env::var("MIRI").is_ok() || env::var("MIRIFLAGS").is_ok() { - println!("cargo:info=Skipping header generation under miri (cbindgen incompatible)"); + let is_asan = env::var("CARGO_ENCODED_RUSTFLAGS") + .unwrap_or_default() + .contains("address"); + if is_asan { + println!("cargo:info=building with asan"); + println!("cargo:rustc-cfg=vortex_asan"); + println!("cargo:info=Skipping header generation due to sanitizers"); return; } - if env::var("RUSTFLAGS") + if env::var("CARGO_ENCODED_RUSTFLAGS") .unwrap_or_default() .contains("sanitizer") { println!("cargo:info=Skipping header generation due to sanitizers"); + } + + if env::var("MIRI").is_ok() || env::var("MIRIFLAGS").is_ok() { + println!("cargo:info=Skipping header generation under miri (cbindgen incompatible)"); return; } // cbindgen macro expansion is only available on nightly - let is_nightly = Command::new("rustc") - .arg("-V") - .output() + let rustc = Command::new("rustc").arg("-V").output(); + let is_nightly = rustc + .as_ref() .map(|output| String::from_utf8_lossy(&output.stdout).contains("nightly")) .unwrap_or(false); if !is_nightly { diff --git a/vortex-ffi/cbindgen.toml b/vortex-ffi/cbindgen.toml index 12e484f093e..5153eb0f9fc 100644 --- a/vortex-ffi/cbindgen.toml +++ b/vortex-ffi/cbindgen.toml @@ -1,6 +1,6 @@ language = "C" braces = "SameLine" -pragma_once = true +pragma_once = false cpp_compat = true usize_is_size_t = true style = "type" @@ -8,10 +8,52 @@ style = "type" header = """ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once +#include // // THIS FILE IS AUTO-GENERATED, DO NOT MAKE EDITS DIRECTLY // + +// https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions +// We don't want to bundle nanoarrow or similar just for these two definitions. +// If you use your own Arrow library, define this macro and +// typedef FFI_ArrowSchema ArrowSchema; +// typedef FFI_ArrowArrayStream ArrowArrayStream; +#ifndef USE_OWN_ARROW +struct ArrowSchema { + const char* format; + const char* name; + const char* metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema** children; + struct ArrowSchema* dictionary; + void (*release)(struct ArrowSchema*); + void* private_data; +}; +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void** buffers; + struct ArrowArray** children; + struct ArrowArray* dictionary; + void (*release)(struct ArrowArray*); + void* private_data; +}; +struct ArrowArrayStream { + int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); + int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); + const char* (*get_last_error)(struct ArrowArrayStream*); + void (*release)(struct ArrowArrayStream*); + void* private_data; +}; +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#endif """ [export.rename] diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index f1f25491b98..bb4495b34de 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -1,11 +1,51 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once +#include // // THIS FILE IS AUTO-GENERATED, DO NOT MAKE EDITS DIRECTLY // -#pragma once +// https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions +// We don't want to bundle nanoarrow or similar just for these two definitions. +// If you use your own Arrow library, define this macro and +// typedef FFI_ArrowSchema ArrowSchema; +// typedef FFI_ArrowArrayStream ArrowArrayStream; +#ifndef USE_OWN_ARROW +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; +struct ArrowArrayStream { + int (*get_schema)(struct ArrowArrayStream *, struct ArrowSchema *out); + int (*get_next)(struct ArrowArrayStream *, struct ArrowArray *out); + const char *(*get_last_error)(struct ArrowArrayStream *); + void (*release)(struct ArrowArrayStream *); + void *private_data; +}; +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#endif #include #include @@ -134,6 +174,12 @@ typedef enum { VX_VALIDITY_ARRAY = 3, } vx_validity_type; +typedef enum { + VX_CARD_UNKNOWN = 0, + VX_CARD_ESTIMATE = 1, + VX_CARD_MAXIMUM = 2, +} vx_cardinality; + /** * Equalities, inequalities, and boolean operations over possibly null values. * For most operations, if either side is null, the result is null. @@ -224,6 +270,33 @@ typedef enum { LOG_LEVEL_TRACE = 5, } vx_log_level; +typedef enum { + VX_SELECTION_INCLUDE_ALL = 0, + /** + * Include rows at the indices in vx_scan_selection.idx. + */ + VX_SELECTION_INCLUDE_RANGE = 1, + /** + * Exclude rows at the indices in vx_scan_selection.idx. + */ + VX_SELECTION_EXCLUDE_RANGE = 2, +} vx_scan_selection_include; + +typedef enum { + /** + * No estimate is available. + */ + VX_ESTIMATE_UNKNOWN = 0, + /** + * The value in vx_estimate.estimate is exact. + */ + VX_ESTIMATE_EXACT = 1, + /** + * The value in vx_estimate.estimate is an upper bound. + */ + VX_ESTIMATE_INEXACT = 2, +} vx_estimate_type; + /** * Physical type enum, represents the in-memory physical layout but might represent a different logical type. */ @@ -369,6 +442,14 @@ typedef struct vx_array_sink vx_array_sink; */ typedef struct vx_binary vx_binary; +/** + * A reference to one or more (possibly remote) paths. + * Creating vx_data_source opens the first matched path to read the schema. + * All other I/O is deferred until a scan is requested. Multiple scans may + * be requested from a single data source. + */ +typedef struct vx_data_source vx_data_source; + /** * A Vortex data type. * @@ -403,6 +484,14 @@ typedef struct vx_expression vx_expression; */ typedef struct vx_file vx_file; +/** + * A partition is an independent unit of work. Call vx_partition_next repeatedly to + * retrieve arrays, then free the partition with vx_partition_free. + */ +typedef struct vx_partition vx_partition; + +typedef struct vx_scan vx_scan; + /** * A handle to a Vortex session. */ @@ -435,6 +524,27 @@ typedef struct { const vx_array *array; } vx_validity; +/** + * Options for creating a data source. + */ +typedef struct { + /** + * Required: paths to files, tables, or layout trees. + * May be a glob pattern like "*.vortex". + * If you want to include multiple paths, concat them with a comma: + * "file1.vortex,../file2.vortex". + */ + const char *paths; +} vx_data_source_options; + +typedef struct { + vx_cardinality cardinality; + /** + * Set only when "cardinality" is not VX_CARD_UNKNOWN + */ + uint64_t rows; +} vx_data_source_row_count; + /** * Options supplied for opening a file. */ @@ -497,6 +607,73 @@ typedef struct { unsigned long row_offset; } vx_file_scan_options; +/** + * Scan row selection. + * "idx" is copied while calling vx_data_source_scan and can be freed after. + */ +typedef struct { + /** + * Used only when "include" is not VX_SELECTION_INCLUDE_ALL. + * If set, must point to an array of len "idx_len" row_indices. + */ + const uint64_t *idx; + /** + * Used only when "include" is not VX_SELECTION_INCLUDE_ALL + */ + size_t idx_len; + vx_scan_selection_include include; +} vx_scan_selection; + +/** + * Scan options. All fields are optional. To return everything, + * zero-initialize this struct. + */ +typedef struct { + /** + * What columns to return. NULL means all columns. + */ + const vx_expression *projection; + /** + * Predicate expression. NULL means no filter. + */ + const vx_expression *filter; + /** + * Row range [begin, end). Setting row_range_begin and row_range_end to 0 + * means no limit. + */ + uint64_t row_range_begin; + uint64_t row_range_end; + /** + * Row-index filter applied after row_range. + */ + vx_scan_selection selection; + /** + * Maximum number of rows to return. 0 means no limit. + */ + uint64_t limit; + /** + * Upper limit for parallelism. 0 means no limit. + * Scan will return at most "max_threads" partitions. + */ + uint64_t max_threads; + /** + * If true, return in storage order. + */ + bool ordered; +} vx_scan_options; + +/** + * Used for estimating number of partitions in a data source or number of rows + * in a partition. + */ +typedef struct { + vx_estimate_type type; + /** + * Set only when "type" is not VX_ESTIMATE_UNKNOWN. + */ + uint64_t estimate; +} vx_estimate; + #ifdef __cplusplus extern "C" { #endif // __cplusplus @@ -711,6 +888,41 @@ size_t vx_binary_len(const vx_binary *ptr); */ const char *vx_binary_ptr(const vx_binary *ptr); +/** + * Clone a borrowed [`vx_data_source`], returning an owned [`vx_data_source`]. + * + * + * Must be released with [`vx_data_source_free`]. + */ +const vx_data_source *vx_data_source_clone(const vx_data_source *ptr); + +/** + * Free an owned [`vx_data_source`] object. + */ +void vx_data_source_free(const vx_data_source *ptr); + +/** + * Create a data source. + * The first matched file is opened eagerly. to read the schema. All other I/O + * is deferred until a scan is requested. The returned pointer is owned by the + * caller and must be freed with vx_data_source_free. + * + * On error, returns NULL and sets "err". + */ +const vx_data_source * +vx_data_source_new(const vx_session *session, const vx_data_source_options *options, vx_error **err); + +/** + * Return the schema of the data source as a non-owned dtype. + * The returned pointer is valid as long as "ds" is alive. Do not free it. + */ +const vx_dtype *vx_data_source_dtype(const vx_data_source *ds); + +/** + * Write data source's row count estimate into "row_count". + */ +void vx_data_source_get_row_count(const vx_data_source *ds, vx_data_source_row_count *row_count); + /** * Clone a borrowed [`vx_dtype`], returning an owned [`vx_dtype`]. * @@ -854,6 +1066,13 @@ uint8_t vx_dtype_time_unit(const DType *dtype); */ const vx_string *vx_dtype_time_zone(const DType *dtype); +/** + * Convert a dtype to ArrowSchema. + * You can use the dtype after conversion + * On success, returns 0. On error, sets err and returns 1. + */ +int vx_dtype_to_arrow_schema(const vx_dtype *dtype, FFI_ArrowSchema *schema, vx_error **err); + /** * Free an owned [`vx_error`] object. */ @@ -896,6 +1115,8 @@ vx_expression *vx_expression_root(void); * expression. Child expression must have a DTYPE_STRUCT dtype. Errors in * vx_array_apply if the child expression doesn't have a specified field. * + * Returns a DTYPE_STRUCT array with selected fields. + * * Example: * * vx_expression* root = vx_expression_root(); @@ -965,7 +1186,9 @@ vx_expression *vx_expression_is_null(const vx_expression *child); * Errors in vx_array_apply if the root array doesn't have a specified field. * * Accesses the specified field from the result of the child expression. - * Equivalent to select(&item, 1, child). + * + * Example: if child is Struct { name=u8, age=u16 } and we do + * vx_expression_get_item("name", child), output type will be DTYPE_U8 */ vx_expression *vx_expression_get_item(const char *item, const vx_expression *child); @@ -1034,6 +1257,86 @@ vx_array_iterator *vx_file_scan(const vx_session *session, */ void vx_set_log_level(vx_log_level level); +/** + * Free an owned [`vx_scan`] object. + */ +void vx_scan_free(vx_scan *ptr); + +/** + * Free an owned [`vx_partition`] object. + */ +void vx_partition_free(vx_partition *ptr); + +/** + * Scan a data source. + * + * Return an owned scan that must be freed with vx_scan_free. A scan may be + * consumed only once. + * + * "options" and "estimate" may be NULL. + * + * If "options" is NULL, all rows and columns are returned. + * If "estimate" is not NULL, the estimated partition count is written to + * *estimate before returning. + * + * Returns NULL and writes an error to "*err" on failure. + */ +vx_scan *vx_data_source_scan(const vx_data_source *data_source, + const vx_scan_options *options, + vx_estimate *estimate, + vx_error **err); + +/** + * Return borrowed vx_scan's dtype. + * This function will fail if called after vx_scan_next_partition. + * Called must not free the returned pointer as its lifetime is bound to the + * lifetime of the scan. + * On error returns NULL and sets "err". + */ +const vx_dtype *vx_scan_dtype(const vx_scan *scan, vx_error **err); + +/** + * Return an owned partition from a scan. + * The returned partition must be freed with vx_partition_free. + * + * On success returns a partition. + * On exhaustion (no more partitions in scan) returns NULL but doesn't set + * "err". + * On error returns NULL and sets "err". + * + * This function is thread-unsafe. Callers running a multi-threaded pipeline + * should synchronise on calls to this function and dispatch each produced + * partition to a dedicated worker thread. + */ +vx_partition *vx_scan_next_partition(vx_scan *scan, vx_error **err); + +/** + * Get partition's estimated row count. + * Must be called before the first call to vx_partition_next. + * + * On success, returns 0. + * On error, return 1 and sets "error". + */ +int vx_partition_row_count(const vx_partition *partition, vx_estimate *count, vx_error **err); + +int vx_partition_scan_arrow(const vx_session *session, + vx_partition *partition, + FFI_ArrowArrayStream *stream, + vx_error **err); + +/** + * Return an owned owned array from a partition. + * The returned array must be freed with vx_array_free. + * + * On success returns an array. + * On exhaustion (no more arrays in partition) returns NULL but doesn't set + * "err". + * On error return NULL and sets "err". + * + * This function is not thread-safe: call from one thread per partition. + */ +const vx_array *vx_partition_next(vx_partition *partition, vx_error **err); + /** * Free an owned [`vx_session`] object. */ @@ -1115,6 +1418,7 @@ void vx_struct_column_builder_free(vx_struct_column_builder *ptr); /** * Create a new column-wise struct array builder with given validity and a * capacity hint. validity can't be NULL. + * Capacity hint is for the number of columns. * If you don't know capacity, pass 0. * if validity is NULL, returns NULL. */ diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index d2a7923e5ad..2b02bb6fcbc 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -196,6 +196,9 @@ pub unsafe extern "C-unwind" fn vx_array_dtype(array: *const vx_array) -> *const vx_dtype::new_ref(vx_array::as_ref(array).dtype()) } +// Return an owned field for array at index. +// Returns NULL and sets error_out if index is out of bounds or array doesn't +// have dtype DTYPE_STRUCT. #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_array_get_field( array: *const vx_array, @@ -450,20 +453,9 @@ mod tests { use crate::dtype::vx_dtype_get_variant; use crate::dtype::vx_dtype_variant; use crate::error::vx_error_free; - use crate::error::vx_error_get_message; use crate::expression::vx_expression_free; use crate::string::vx_string_free; - - fn assert_no_error(error: *mut vx_error) { - if !error.is_null() { - let message; - unsafe { - message = vx_string::as_str(vx_error_get_message(error)).to_owned(); - vx_error_free(error); - } - panic!("{message}"); - } - } + use crate::tests::assert_no_error; #[test] // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477 diff --git a/vortex-ffi/src/data_source.rs b/vortex-ffi/src/data_source.rs new file mode 100644 index 00000000000..2055e697209 --- /dev/null +++ b/vortex-ffi/src/data_source.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#![allow(non_camel_case_types)] +#![deny(missing_docs)] + +use std::ffi::c_char; +use std::ptr; +use std::sync::Arc; + +use vortex::error::VortexResult; +use vortex::error::vortex_ensure; +use vortex::expr::stats::Precision::Exact; +use vortex::expr::stats::Precision::Inexact; +use vortex::file::multi::MultiFileDataSource; +use vortex::io::runtime::BlockingRuntime; +use vortex::scan::DataSource; +use vortex::scan::DataSourceRef; + +use crate::RUNTIME; +use crate::dtype::vx_dtype; +use crate::error::try_or; +use crate::error::vx_error; +use crate::session::vx_session; +use crate::to_string; + +crate::arc_dyn_wrapper!( + /// A reference to one or more (possibly remote) paths. + /// Creating vx_data_source opens the first matched path to read the schema. + /// All other I/O is deferred until a scan is requested. Multiple scans may + /// be requested from a single data source. + dyn DataSource, + vx_data_source); + +/// Options for creating a data source. +#[repr(C)] +#[cfg_attr(test, derive(Default))] +pub struct vx_data_source_options { + /// Required: paths to files, tables, or layout trees. + /// May be a glob pattern like "*.vortex". + /// If you want to include multiple paths, concat them with a comma: + /// "file1.vortex,../file2.vortex". + pub paths: *const c_char, +} + +#[cfg(vortex_asan)] +unsafe extern "C" { + pub fn __lsan_disable(); + pub fn __lsan_enable(); +} + +unsafe fn data_source_new( + session: *const vx_session, + opts: *const vx_data_source_options, +) -> VortexResult<*const vx_data_source> { + vortex_ensure!(!session.is_null()); + vortex_ensure!(!opts.is_null()); + + let session = vx_session::as_ref(session); + + let opts = unsafe { &*opts }; + vortex_ensure!(!opts.paths.is_null()); + + let glob = unsafe { to_string(opts.paths) }; + let mut data_source = MultiFileDataSource::new(session.clone()); + for glob in glob.split(',') { + data_source = data_source.with_glob(glob, None); + } + + let data_source = RUNTIME.block_on(async { + // TODO(myrrc): see https://github.com/vortex-data/vortex/issues/7324 + #[cfg(vortex_asan)] + unsafe { + __lsan_disable(); + } + let data_source = data_source.build().await; + #[cfg(vortex_asan)] + unsafe { + __lsan_enable(); + } + data_source + })?; + Ok(vx_data_source::new(Arc::new(data_source) as DataSourceRef)) +} + +/// Create a data source. +/// The first matched file is opened eagerly. to read the schema. All other I/O +/// is deferred until a scan is requested. The returned pointer is owned by the +/// caller and must be freed with vx_data_source_free. +/// +/// On error, returns NULL and sets "err". +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_new( + session: *const vx_session, + options: *const vx_data_source_options, + err: *mut *mut vx_error, +) -> *const vx_data_source { + try_or(err, ptr::null(), || unsafe { + data_source_new(session, options) + }) +} + +/// Return the schema of the data source as a non-owned dtype. +/// The returned pointer is valid as long as "ds" is alive. Do not free it. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_dtype(ds: *const vx_data_source) -> *const vx_dtype { + vx_dtype::new_ref(vx_data_source::as_ref(ds).dtype()) +} + +#[repr(C)] +#[cfg_attr(test, derive(PartialEq, Debug))] +enum vx_cardinality { + VX_CARD_UNKNOWN = 0, + VX_CARD_ESTIMATE = 1, + VX_CARD_MAXIMUM = 2, +} + +#[repr(C)] +pub struct vx_data_source_row_count { + cardinality: vx_cardinality, + /// Set only when "cardinality" is not VX_CARD_UNKNOWN + rows: u64, +} + +/// Write data source's row count estimate into "row_count". +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_get_row_count( + ds: *const vx_data_source, + row_count: *mut vx_data_source_row_count, +) { + let rc = unsafe { &mut *row_count }; + match vx_data_source::as_ref(ds).row_count() { + Some(Exact(rows)) => { + rc.cardinality = vx_cardinality::VX_CARD_MAXIMUM; + rc.rows = rows; + } + Some(Inexact(rows)) => { + rc.cardinality = vx_cardinality::VX_CARD_ESTIMATE; + rc.rows = rows; + } + None => { + rc.cardinality = vx_cardinality::VX_CARD_UNKNOWN; + } + } +} + +// Object store error: Generic LocalFileSystem error: Unable to convert +// URL "file:///C:%255CWindows%255CSystemTemp%255C.tmpRXzX38" to filesystem path +// https://github.com/servo/rust-url/issues/1077 +#[cfg(not(windows))] +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::ptr; + + use crate::data_source::vx_cardinality; + use crate::data_source::vx_data_source_dtype; + use crate::data_source::vx_data_source_free; + use crate::data_source::vx_data_source_get_row_count; + use crate::data_source::vx_data_source_new; + use crate::data_source::vx_data_source_options; + use crate::data_source::vx_data_source_row_count; + use crate::dtype::vx_dtype; + use crate::session::vx_session_free; + use crate::session::vx_session_new; + use crate::tests::SAMPLE_ROWS; + use crate::tests::assert_error; + use crate::tests::assert_no_error; + use crate::tests::write_sample; + + #[test] + #[cfg_attr(miri, ignore)] + fn test_create_invalid() { + unsafe { + let session = vx_session_new(); + let mut error = ptr::null_mut(); + + let ds = vx_data_source_new(ptr::null_mut(), ptr::null(), &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + let ds = vx_data_source_new(session, ptr::null(), &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + let mut opts = vx_data_source_options::default(); + let ds = vx_data_source_new(session, &raw const opts, &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + opts.paths = c"test.vortex".as_ptr(); + let ds = vx_data_source_new(session, &raw const opts, &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + opts.paths = c"*.vortex".as_ptr(); + let ds = vx_data_source_new(session, &raw const opts, &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + vx_session_free(session); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_row_count() { + unsafe { + let session = vx_session_new(); + let (sample, struct_array) = write_sample(session); + + let path = CString::new(sample.path().to_str().unwrap()).unwrap(); + let opts = vx_data_source_options { + paths: path.as_ptr(), + }; + + let mut error = ptr::null_mut(); + let ds = vx_data_source_new(session, &raw const opts, &raw mut error); + assert_no_error(error); + assert!(!ds.is_null()); + + let dtype = vx_dtype::as_ref(vx_data_source_dtype(ds)); + assert_eq!(dtype, struct_array.dtype()); + + let mut row_count = vx_data_source_row_count { + cardinality: vx_cardinality::VX_CARD_UNKNOWN, + rows: 0, + }; + vx_data_source_get_row_count(ds, &raw mut row_count); + assert_eq!(row_count.cardinality, vx_cardinality::VX_CARD_MAXIMUM); + assert_eq!(row_count.rows, SAMPLE_ROWS as u64); + + vx_data_source_free(ds); + vx_session_free(session); + } + } +} diff --git a/vortex-ffi/src/dtype.rs b/vortex-ffi/src/dtype.rs index 57bc3560d8e..d3fe6f1a5c4 100644 --- a/vortex-ffi/src/dtype.rs +++ b/vortex-ffi/src/dtype.rs @@ -1,9 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::ffi::c_int; use std::ptr; use std::sync::Arc; +use arrow_array::ffi::FFI_ArrowSchema; use vortex::dtype::DType; use vortex::dtype::DecimalDType; use vortex::error::VortexExpect; @@ -14,6 +16,8 @@ use vortex::extension::datetime::Time; use vortex::extension::datetime::Timestamp; use crate::arc_wrapper; +use crate::error::try_or; +use crate::error::vx_error; use crate::ptype::vx_ptype; use crate::string::vx_string; use crate::struct_fields::vx_struct_fields; @@ -323,6 +327,24 @@ pub unsafe extern "C-unwind" fn vx_dtype_time_zone(dtype: *const DType) -> *cons } } +/// Convert a dtype to ArrowSchema. +/// You can use the dtype after conversion +/// On success, returns 0. On error, sets err and returns 1. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_dtype_to_arrow_schema( + dtype: *const vx_dtype, + schema: *mut FFI_ArrowSchema, + err: *mut *mut vx_error, +) -> c_int { + try_or(err, 1, || { + let dtype = vx_dtype::as_ref(dtype); + let arrow_schema = dtype.to_arrow_schema()?; + let arrow_schema = FFI_ArrowSchema::try_from(&arrow_schema)?; + unsafe { ptr::write(schema, arrow_schema) }; + Ok(0) + }) +} + #[cfg(test)] #[expect(clippy::cast_possible_truncation)] mod tests { diff --git a/vortex-ffi/src/error.rs b/vortex-ffi/src/error.rs index 79115bad43b..4bc5a1ed4be 100644 --- a/vortex-ffi/src/error.rs +++ b/vortex-ffi/src/error.rs @@ -28,6 +28,7 @@ pub(crate) fn write_error(error: *mut *mut vx_error, message: &str) { unsafe { error.write(err) }; } +#[inline] pub fn try_or_default( error_out: *mut *mut vx_error, function: impl FnOnce() -> VortexResult, @@ -38,15 +39,29 @@ pub fn try_or_default( value } Err(err) => { - let err = vx_error::new(VortexError { - message: err.to_string().into(), - }); - unsafe { error_out.write(err) }; + write_error(error_out, &err.to_string()); T::default() } } } +pub fn try_or( + error_out: *mut *mut vx_error, + error_value: T, + function: impl FnOnce() -> VortexResult, +) -> T { + match function() { + Ok(value) => { + unsafe { error_out.write(ptr::null_mut()) }; + value + } + Err(err) => { + write_error(error_out, &err.to_string()); + error_value + } + } +} + /// Returns the error message from the given Vortex error. /// /// The returned pointer is valid as long as the error is valid. diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 37f9b602cae..442f9325959 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -66,6 +66,8 @@ pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression { /// expression. Child expression must have a DTYPE_STRUCT dtype. Errors in /// vx_array_apply if the child expression doesn't have a specified field. /// +/// Returns a DTYPE_STRUCT array with selected fields. +/// /// Example: /// /// vx_expression* root = vx_expression_root(); @@ -242,7 +244,9 @@ pub unsafe extern "C" fn vx_expression_is_null(child: *const vx_expression) -> * /// Errors in vx_array_apply if the root array doesn't have a specified field. /// /// Accesses the specified field from the result of the child expression. -/// Equivalent to select(&item, 1, child). +/// +/// Example: if child is Struct { name=u8, age=u16 } and we do +/// vx_expression_get_item("name", child), output type will be DTYPE_U8 #[unsafe(no_mangle)] pub unsafe extern "C" fn vx_expression_get_item( item: *const c_char, diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs index 1d11fc168dc..fb675c558bc 100644 --- a/vortex-ffi/src/lib.rs +++ b/vortex-ffi/src/lib.rs @@ -8,6 +8,7 @@ mod array; mod array_iterator; mod binary; +mod data_source; mod dtype; mod error; mod expression; @@ -15,6 +16,7 @@ mod file; mod log; mod macros; mod ptype; +mod scan; mod session; mod sink; mod string; @@ -76,3 +78,102 @@ pub(crate) unsafe fn to_field_names( }) .collect() } + +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::ptr; + use std::sync::Arc; + + use rand::Rng; + use tempfile::NamedTempFile; + use vortex_array::IntoArray; + use vortex_array::arrays::PrimitiveArray; + use vortex_array::arrays::StructArray; + use vortex_array::arrays::VarBinViewArray; + use vortex_array::validity::Validity; + + use crate::array::vx_array; + use crate::array::vx_array_free; + use crate::dtype::vx_dtype; + use crate::dtype::vx_dtype_free; + use crate::error::vx_error; + use crate::error::vx_error_free; + use crate::error::vx_error_get_message; + use crate::session::vx_session; + use crate::sink::vx_array_sink_close; + use crate::sink::vx_array_sink_open_file; + use crate::sink::vx_array_sink_push; + use crate::string::vx_string; + + /// Panic if error is NULL. Free the error if it's not + pub(crate) fn assert_error(error: *mut vx_error) { + assert!(!error.is_null(), "Expected error"); + unsafe { vx_error_free(error) }; + } + + /// Panic if error is not NULL. + pub(crate) fn assert_no_error(error: *mut vx_error) { + if !error.is_null() { + let message; + unsafe { + message = vx_string::as_str(vx_error_get_message(error)).to_owned(); + vx_error_free(error); + } + panic!("{message}"); + } + } + + fn random_str(length: usize) -> String { + const CHARSET: &[u8] = b"0123456789"; + let mut rng = rand::thread_rng(); + + (0..length) + .map(|_| { + let idx = rng.gen_range(0..CHARSET.len()); + CHARSET[idx] as char + }) + .collect() + } + + pub const SAMPLE_ROWS: usize = 200; + + /// Write 200 rows of Struct { age=i32, height=i32, name=String } into a + /// temporary file + pub(crate) unsafe fn write_sample(session: *const vx_session) -> (NamedTempFile, StructArray) { + let age = (0..SAMPLE_ROWS as u64).map(Some); + let age = PrimitiveArray::from_option_iter(age); + + let height = (0..SAMPLE_ROWS as u64).map(|x| Some(200 * x)); + let height = PrimitiveArray::from_option_iter(height); + + let name = (0..SAMPLE_ROWS).map(random_str); + let name = VarBinViewArray::from_iter_str(name); + + let struct_array = StructArray::try_new( + ["age", "height", "name"].into(), + vec![age.into_array(), height.into_array(), name.into_array()], + SAMPLE_ROWS, + Validity::NonNullable, + ) + .unwrap(); + + let file = NamedTempFile::new().unwrap(); + let path = CString::new(file.path().to_str().unwrap()).unwrap(); + let dtype = struct_array.dtype(); + + unsafe { + let vx_dtype_ptr = vx_dtype::new(Arc::new(dtype.clone())); + let mut error = ptr::null_mut(); + let sink = + vx_array_sink_open_file(session, path.as_ptr(), vx_dtype_ptr, &raw mut error); + let array = vx_array::new(Arc::new(struct_array.clone().into_array())); + vx_array_sink_push(sink, array, &raw mut error); + vx_array_sink_close(sink, &raw mut error); + vx_array_free(array); + vx_dtype_free(vx_dtype_ptr); + } + + (file, struct_array) + } +} diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs new file mode 100644 index 00000000000..fdae46aca4f --- /dev/null +++ b/vortex-ffi/src/scan.rs @@ -0,0 +1,728 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#![allow(non_camel_case_types)] +#![deny(missing_docs)] + +use core::slice; +use std::ffi::c_int; +use std::ops::Range; +use std::ptr; +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::ffi_stream::FFI_ArrowArrayStream; +use arrow_schema::ArrowError; +use arrow_schema::DataType; +use futures::StreamExt; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowArrayExecutor; +use vortex::array::expr::stats::Precision; +use vortex::array::stream::SendableArrayStream; +use vortex::buffer::Buffer; +use vortex::error::VortexResult; +use vortex::error::vortex_bail; +use vortex::error::vortex_ensure; +use vortex::expr::root; +use vortex::io::runtime::BlockingRuntime; +use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; +use vortex::scan::DataSourceScan; +use vortex::scan::Partition; +use vortex::scan::PartitionStream; +use vortex::scan::ScanRequest; +use vortex::scan::selection::Selection; + +use crate::RUNTIME; +use crate::array::vx_array; +use crate::data_source::vx_data_source; +use crate::dtype::vx_dtype; +use crate::error::try_or; +use crate::error::try_or_default; +use crate::error::vx_error; +use crate::expression::vx_expression; +use crate::session::vx_session; + +pub enum VxScan { + Pending(Box), + Started(PartitionStream), + Finished, +} +crate::box_wrapper!(VxScan, vx_scan); + +pub enum VxPartitionScan { + Pending(Box), + Started(SendableArrayStream), + Finished, +} +crate::box_wrapper!( + /// A partition is an independent unit of work. Call vx_partition_next repeatedly to + /// retrieve arrays, then free the partition with vx_partition_free. + VxPartitionScan, + vx_partition); + +// We parse Selection from vx_scan_selection[_include], so we don't need +// to instantiate VX_SELECTION_* items directly. +#[repr(C)] +#[allow(dead_code)] +#[cfg_attr(test, derive(Default))] +pub enum vx_scan_selection_include { + #[cfg_attr(test, default)] + VX_SELECTION_INCLUDE_ALL = 0, + /// Include rows at the indices in vx_scan_selection.idx. + VX_SELECTION_INCLUDE_RANGE = 1, + /// Exclude rows at the indices in vx_scan_selection.idx. + VX_SELECTION_EXCLUDE_RANGE = 2, +} + +/// Scan row selection. +/// "idx" is copied while calling vx_data_source_scan and can be freed after. +#[repr(C)] +#[cfg_attr(test, derive(Default))] +pub struct vx_scan_selection { + /// Used only when "include" is not VX_SELECTION_INCLUDE_ALL. + /// If set, must point to an array of len "idx_len" row_indices. + pub idx: *const u64, + /// Used only when "include" is not VX_SELECTION_INCLUDE_ALL + pub idx_len: usize, + pub include: vx_scan_selection_include, +} + +/// Scan options. All fields are optional. To return everything, +/// zero-initialize this struct. +#[repr(C)] +#[cfg_attr(test, derive(Default))] +pub struct vx_scan_options { + /// What columns to return. NULL means all columns. + pub projection: *const vx_expression, + /// Predicate expression. NULL means no filter. + pub filter: *const vx_expression, + /// Row range [begin, end). Setting row_range_begin and row_range_end to 0 + /// means no limit. + pub row_range_begin: u64, + pub row_range_end: u64, + /// Row-index filter applied after row_range. + pub selection: vx_scan_selection, + /// Maximum number of rows to return. 0 means no limit. + pub limit: u64, + /// Upper limit for parallelism. 0 means no limit. + /// Scan will return at most "max_threads" partitions. + pub max_threads: u64, + /// If true, return in storage order. + pub ordered: bool, +} + +#[repr(C)] +pub enum vx_estimate_type { + /// No estimate is available. + VX_ESTIMATE_UNKNOWN = 0, + /// The value in vx_estimate.estimate is exact. + VX_ESTIMATE_EXACT = 1, + /// The value in vx_estimate.estimate is an upper bound. + VX_ESTIMATE_INEXACT = 2, +} + +/// Used for estimating number of partitions in a data source or number of rows +/// in a partition. +#[repr(C)] +pub struct vx_estimate { + r#type: vx_estimate_type, + /// Set only when "type" is not VX_ESTIMATE_UNKNOWN. + estimate: u64, +} + +fn scan_request(opts: *const vx_scan_options) -> VortexResult { + if opts.is_null() { + return Ok(ScanRequest::default()); + } + let opts = unsafe { &*opts }; + + let projection = if opts.projection.is_null() { + root() + } else { + vx_expression::as_ref(opts.projection).clone() + }; + + let filter = if opts.filter.is_null() { + None + } else { + Some(vx_expression::as_ref(opts.filter).clone()) + }; + + let selection = &opts.selection; + let selection = match selection.include { + vx_scan_selection_include::VX_SELECTION_INCLUDE_ALL => Selection::All, + vx_scan_selection_include::VX_SELECTION_INCLUDE_RANGE => { + vortex_ensure!(!selection.idx.is_null()); + let buf = unsafe { slice::from_raw_parts(selection.idx, selection.idx_len) }; + let buf = Buffer::copy_from(buf); + Selection::IncludeByIndex(buf) + } + vx_scan_selection_include::VX_SELECTION_EXCLUDE_RANGE => { + vortex_ensure!(!selection.idx.is_null()); + let buf = unsafe { slice::from_raw_parts(selection.idx, selection.idx_len) }; + let buf = Buffer::copy_from(buf); + Selection::ExcludeByIndex(buf) + } + }; + + let ordered = opts.ordered; + + let start = opts.row_range_begin; + let end = opts.row_range_end; + let row_range = (start > 0 || end > 0).then_some(Range { start, end }); + + let limit = (opts.limit != 0).then_some(opts.limit); + + Ok(ScanRequest { + projection, + filter, + row_range, + selection, + ordered, + limit, + }) +} + +fn write_estimate>(estimate: Option>, out: &mut vx_estimate) { + match estimate { + Some(Precision::Exact(value)) => { + out.r#type = vx_estimate_type::VX_ESTIMATE_EXACT; + out.estimate = value.into(); + } + Some(Precision::Inexact(value)) => { + out.r#type = vx_estimate_type::VX_ESTIMATE_INEXACT; + out.estimate = value.into(); + } + None => { + out.r#type = vx_estimate_type::VX_ESTIMATE_UNKNOWN; + } + } +} + +/// Scan a data source. +/// +/// Return an owned scan that must be freed with vx_scan_free. A scan may be +/// consumed only once. +/// +/// "options" and "estimate" may be NULL. +/// +/// If "options" is NULL, all rows and columns are returned. +/// If "estimate" is not NULL, the estimated partition count is written to +/// *estimate before returning. +/// +/// Returns NULL and writes an error to "*err" on failure. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_scan( + data_source: *const vx_data_source, + options: *const vx_scan_options, + estimate: *mut vx_estimate, + err: *mut *mut vx_error, +) -> *mut vx_scan { + try_or(err, ptr::null_mut(), || { + let request = scan_request(options)?; + RUNTIME.block_on(async { + let scan = vx_data_source::as_ref(data_source).scan(request).await?; + if !estimate.is_null() { + write_estimate( + scan.partition_count().map(|x| match x { + Precision::Exact(v) => Precision::Exact(v as u64), + Precision::Inexact(v) => Precision::Inexact(v as u64), + }), + unsafe { &mut *estimate }, + ); + } + Ok(vx_scan::new(VxScan::Pending(scan))) + }) + }) +} + +/// Return borrowed vx_scan's dtype. +/// This function will fail if called after vx_scan_next_partition. +/// Called must not free the returned pointer as its lifetime is bound to the +/// lifetime of the scan. +/// On error returns NULL and sets "err". +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_scan_dtype( + scan: *const vx_scan, + err: *mut *mut vx_error, +) -> *const vx_dtype { + try_or(err, ptr::null(), || { + let scan = vx_scan::as_ref(scan); + let VxScan::Pending(ref scan) = *scan else { + vortex_bail!("dtype unavailable: scan already started"); + }; + Ok(vx_dtype::new_ref(scan.dtype())) + }) +} + +/// Return an owned partition from a scan. +/// The returned partition must be freed with vx_partition_free. +/// +/// On success returns a partition. +/// On exhaustion (no more partitions in scan) returns NULL but doesn't set +/// "err". +/// On error returns NULL and sets "err". +/// +/// This function is thread-unsafe. Callers running a multi-threaded pipeline +/// should synchronise on calls to this function and dispatch each produced +/// partition to a dedicated worker thread. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_scan_next_partition( + scan: *mut vx_scan, + err: *mut *mut vx_error, +) -> *mut vx_partition { + let scan = vx_scan::as_mut(scan); + let scan = &mut *scan; + unsafe { + let ptr = scan as *mut VxScan; + + let on_finish = || -> VortexResult<*mut vx_partition> { + ptr::write(ptr, VxScan::Finished); + Ok(ptr::null_mut()) + }; + + let on_stream = |mut stream: PartitionStream| -> VortexResult<*mut vx_partition> { + match RUNTIME.block_on(stream.next()) { + Some(partition) => { + let partition = VxPartitionScan::Pending(partition?); + let partition = vx_partition::new(partition); + ptr::write(ptr, VxScan::Started(stream)); + Ok(partition) + } + None => on_finish(), + } + }; + + let owned = ptr::read(ptr); + try_or_default(err, || match owned { + VxScan::Pending(scan) => on_stream(scan.partitions()), + VxScan::Started(stream) => on_stream(stream), + VxScan::Finished => on_finish(), + }) + } +} + +/// Get partition's estimated row count. +/// Must be called before the first call to vx_partition_next. +/// +/// On success, returns 0. +/// On error, return 1 and sets "error". +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_partition_row_count( + partition: *const vx_partition, + count: *mut vx_estimate, + err: *mut *mut vx_error, +) -> c_int { + try_or(err, 1, || { + let partition = vx_partition::as_ref(partition); + let VxPartitionScan::Pending(partition) = partition else { + vortex_bail!("row count unavailable: partition already started"); + }; + write_estimate(partition.row_count(), unsafe { &mut *count }); + Ok(0) + }) +} + +// Scan partition to ArrowArrayStream. +// Consumes partition fully: subsequent calls to vx_partition_scan_arrow or +// vx_partition_next are undefined behaviour. +// This call blocks current thread until underlying stream is fully consumed. +// +// Caller must not free partition after calling this function. +// +// On success, sets "stream" and returns 0. +// On error, sets "err" and returns 1, freeing the partition. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_partition_scan_arrow( + session: *const vx_session, + partition: *mut vx_partition, + stream: *mut FFI_ArrowArrayStream, + err: *mut *mut vx_error, +) -> c_int { + try_or(err, 1, || { + let partition = match *vx_partition::into_box(partition) { + VxPartitionScan::Pending(partition) => partition, + _ => vortex_bail!( + "Can't consume partition into ArrowArrayStream: partition already being consumed" + ), + }; + let array_stream = partition.execute()?; + let dtype = array_stream.dtype(); + + let schema = dtype.to_arrow_schema()?; + let schema = Arc::new(schema); + let data_type = DataType::Struct(schema.fields().clone()); + + let session = vx_session::as_ref(session); + + let on_chunk = move |chunk: VortexResult| -> VortexResult { + let chunk: ArrayRef = chunk?; + let mut ctx: ExecutionCtx = session.create_execution_ctx(); + let arrow = chunk.execute_arrow(Some(&data_type), &mut ctx)?; + Ok(RecordBatch::from(arrow.as_struct().clone())) + }; + + let iter = RUNTIME + .block_on_stream(array_stream) + .map(on_chunk) + .map(|result| result.map_err(|e| ArrowError::ExternalError(Box::new(e)))); + + let reader = RecordBatchIteratorAdapter::new(iter, schema); + let arrow_stream = FFI_ArrowArrayStream::new(Box::new(reader)); + unsafe { + ptr::write(stream, arrow_stream); + }; + Ok(0) + }) +} + +/// Return an owned owned array from a partition. +/// The returned array must be freed with vx_array_free. +/// +/// On success returns an array. +/// On exhaustion (no more arrays in partition) returns NULL but doesn't set +/// "err". +/// On error return NULL and sets "err". +/// +/// This function is not thread-safe: call from one thread per partition. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_partition_next( + partition: *mut vx_partition, + err: *mut *mut vx_error, +) -> *const vx_array { + let partition = vx_partition::as_mut(partition); + unsafe { + let ptr = partition as *mut VxPartitionScan; + + let on_finish = || -> VortexResult<*const vx_array> { + ptr::write(ptr, VxPartitionScan::Finished); + Ok(ptr::null_mut()) + }; + + let on_stream = |mut stream: SendableArrayStream| -> VortexResult<*const vx_array> { + match RUNTIME.block_on(stream.next()) { + Some(array) => { + let array = vx_array::new(Arc::new(array?)); + ptr::write(ptr, VxPartitionScan::Started(stream)); + Ok(array) + } + None => on_finish(), + } + }; + + let owned = ptr::read(ptr); + try_or_default(err, || match owned { + VxPartitionScan::Pending(partition) => on_stream(partition.execute()?), + VxPartitionScan::Started(stream) => on_stream(stream), + VxPartitionScan::Finished => on_finish(), + }) + } +} + +// Object store error: Generic LocalFileSystem error: Unable to convert +// URL "file:///C:%255CWindows%255CSystemTemp%255C.tmpRXzX38" to filesystem path +// https://github.com/servo/rust-url/issues/1077 +#[cfg(not(windows))] +#[cfg(test)] +mod tests { + use std::ffi::CString; + use std::ptr; + + use vortex::VortexSessionDefault; + use vortex::array::arrays::StructArray; + use vortex::expr::lit; + use vortex::session::VortexSession; + use vortex_array::ExecutionCtx; + use vortex_array::arrays::struct_::StructArrayExt; + use vortex_array::assert_arrays_eq; + + use crate::array::vx_array; + use crate::array::vx_array_free; + use crate::data_source::vx_data_source_free; + use crate::data_source::vx_data_source_new; + use crate::data_source::vx_data_source_options; + use crate::expression::vx_binary_operator; + use crate::expression::vx_expression; + use crate::expression::vx_expression_binary; + use crate::expression::vx_expression_free; + use crate::expression::vx_expression_get_item; + use crate::expression::vx_expression_root; + use crate::scan::vx_data_source_scan; + use crate::scan::vx_estimate; + use crate::scan::vx_partition_free; + use crate::scan::vx_partition_next; + use crate::scan::vx_partition_row_count; + use crate::scan::vx_scan_free; + use crate::scan::vx_scan_next_partition; + use crate::scan::vx_scan_options; + use crate::scan::vx_scan_selection; + use crate::scan::vx_scan_selection_include; + use crate::session::vx_session_free; + use crate::session::vx_session_new; + use crate::tests::SAMPLE_ROWS; + use crate::tests::assert_no_error; + use crate::tests::write_sample; + + /// Perform a scan with options over a sample file, return owned read array and + /// original generated array for the sample file. + fn scan(options: *const vx_scan_options) -> (*const vx_array, StructArray) { + unsafe { + let session = vx_session_new(); + let (sample, struct_array) = write_sample(session); + let path = CString::new(sample.path().to_str().unwrap()).unwrap(); + let ds_options = vx_data_source_options { + paths: path.as_ptr(), + }; + + let mut error = ptr::null_mut(); + let ds = vx_data_source_new(session, &raw const ds_options, &raw mut error); + assert_no_error(error); + assert!(!ds.is_null()); + + let mut error = ptr::null_mut(); + let scan = vx_data_source_scan(ds, options, ptr::null_mut(), &raw mut error); + assert_no_error(error); + assert!(!scan.is_null()); + + let partition = vx_scan_next_partition(scan, &raw mut error); + assert_no_error(error); + assert!(!partition.is_null()); + + let array = vx_partition_next(partition, &raw mut error); + assert_no_error(error); + assert!(!array.is_null()); + + assert!(vx_partition_next(partition, &raw mut error).is_null()); + assert_no_error(error); + assert!(vx_partition_next(partition, &raw mut error).is_null()); + assert_no_error(error); + + vx_partition_free(partition); + vx_scan_free(scan); + vx_data_source_free(ds); + vx_session_free(session); + + (array, struct_array) + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_no_options() { + let (array, struct_array) = scan(ptr::null()); + assert_arrays_eq!(vx_array::as_ref(array), struct_array); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_project_all() { + let opts = vx_scan_options::default(); + let (array, struct_array) = scan(&raw const opts); + assert_arrays_eq!(vx_array::as_ref(array), struct_array); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_project_single_field() { + unsafe { + let root = vx_expression_root(); + let mut opts = vx_scan_options::default(); + + for (field, c_field) in [("age", c"age"), ("height", c"height"), ("name", c"name")] { + let field_expr = vx_expression_get_item(c_field.as_ptr(), root); + assert!(!field_expr.is_null()); + opts.projection = field_expr; + let (array, struct_array) = scan(&raw const opts); + assert_arrays_eq!( + vx_array::as_ref(array), + struct_array.unmasked_field_by_name(field).unwrap() + ); + vx_array_free(array); + vx_expression_free(field_expr); + } + vx_expression_free(root); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_project_sum() { + let session = VortexSession::default(); + let mut ctx = ExecutionCtx::new(session); + unsafe { + let root = vx_expression_root(); + let mut opts = vx_scan_options::default(); + + let expr_age = vx_expression_get_item(c"age".as_ptr(), root); + let expr_height = vx_expression_get_item(c"height".as_ptr(), root); + let expr_sum = + vx_expression_binary(vx_binary_operator::VX_OPERATOR_ADD, expr_age, expr_height); + + opts.projection = expr_sum; + let (array, _) = scan(&raw const opts); + { + let array = vx_array::as_ref(array); + let stats = array.statistics(); + assert!(stats.compute_is_sorted(&mut ctx).unwrap()); + assert_eq!(stats.compute_min(&mut ctx), Some(0)); + assert_eq!( + stats.compute_max(&mut ctx), + Some(200 * (SAMPLE_ROWS - 1) + 199) + ); + } + vx_array_free(array); + + vx_expression_free(expr_age); + vx_expression_free(expr_height); + vx_expression_free(expr_sum); + vx_expression_free(root); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_filter() { + unsafe { + let root = vx_expression_root(); + let age_expr = vx_expression_get_item(c"age".as_ptr(), root); + let lit_100 = vx_expression::new(lit(100u64)); + let filter = + vx_expression_binary(vx_binary_operator::VX_OPERATOR_GTE, age_expr, lit_100); + + let opts = vx_scan_options { + filter, + ..Default::default() + }; + let (array, _) = scan(&raw const opts); + assert_eq!(vx_array::as_ref(array).len(), 100); + + vx_array_free(array); + vx_expression_free(filter); + vx_expression_free(age_expr); + vx_expression_free(lit_100); + vx_expression_free(root); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_filter_project() { + unsafe { + let root = vx_expression_root(); + let age_expr = vx_expression_get_item(c"age".as_ptr(), root); + let lit_100 = vx_expression::new(lit(100u64)); + let filter = + vx_expression_binary(vx_binary_operator::VX_OPERATOR_GTE, age_expr, lit_100); + let projection = vx_expression_get_item(c"age".as_ptr(), root); + + let opts = vx_scan_options { + projection, + filter, + ..Default::default() + }; + let (array, _) = scan(&raw const opts); + assert_eq!(vx_array::as_ref(array).len(), 100); + + vx_array_free(array); + vx_expression_free(filter); + vx_expression_free(age_expr); + vx_expression_free(lit_100); + vx_expression_free(projection); + vx_expression_free(root); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_row_range() { + let opts = vx_scan_options { + row_range_begin: 50, + row_range_end: 100, + ..Default::default() + }; + let (array, _) = scan(&raw const opts); + assert_eq!(vx_array::as_ref(array).len(), 50); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_selection() { + let indices = [0u64, 50, 100, 150, 199]; + let opts = vx_scan_options { + selection: vx_scan_selection { + idx: indices.as_ptr(), + idx_len: indices.len(), + include: vx_scan_selection_include::VX_SELECTION_INCLUDE_RANGE, + }, + ..Default::default() + }; + let (array, _) = scan(&raw const opts); + assert_eq!(vx_array::as_ref(array).len(), indices.len()); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_limit() { + let opts = vx_scan_options { + limit: 50, + ..Default::default() + }; + let (array, _) = scan(&raw const opts); + assert_eq!(vx_array::as_ref(array).len(), 50); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_ordered() { + let opts = vx_scan_options { + ordered: true, + ..Default::default() + }; + let (array, struct_array) = scan(&raw const opts); + assert_arrays_eq!(vx_array::as_ref(array), struct_array); + unsafe { vx_array_free(array) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_row_count() { + unsafe { + let session = vx_session_new(); + let (sample, _) = write_sample(session); + let path = CString::new(sample.path().to_str().unwrap()).unwrap(); + let ds_options = vx_data_source_options { + paths: path.as_ptr(), + }; + + let mut error = ptr::null_mut(); + let ds = vx_data_source_new(session, &raw const ds_options, &raw mut error); + assert_no_error(error); + + let mut error = ptr::null_mut(); + let scan_ptr = vx_data_source_scan(ds, ptr::null(), ptr::null_mut(), &raw mut error); + assert_no_error(error); + + let mut error = ptr::null_mut(); + let partition = vx_scan_next_partition(scan_ptr, &raw mut error); + assert_no_error(error); + assert!(!partition.is_null()); + + let mut count: vx_estimate = std::mem::zeroed(); + let result = vx_partition_row_count(partition, &raw mut count, &raw mut error); + assert_no_error(error); + assert_eq!(result, 0); + + vx_partition_free(partition); + vx_scan_free(scan_ptr); + vx_data_source_free(ds); + vx_session_free(session); + } + } +} diff --git a/vortex-ffi/src/session.rs b/vortex-ffi/src/session.rs index 668598852eb..5e8fc4b4909 100644 --- a/vortex-ffi/src/session.rs +++ b/vortex-ffi/src/session.rs @@ -31,3 +31,35 @@ pub unsafe extern "C-unwind" fn vx_session_clone(session: *const vx_session) -> let session = vx_session::as_ref(session); vx_session::new(session.clone()) } + +#[cfg(test)] +mod tests { + use crate::session::vx_session_clone; + use crate::session::vx_session_free; + use crate::session::vx_session_new; + + #[test] + #[cfg_attr(miri, ignore)] + fn test_basic() { + unsafe { + let session = vx_session_new(); + assert!(!session.is_null()); + vx_session_free(session); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_clone() { + unsafe { + let session = vx_session_new(); + assert!(!session.is_null()); + + let copy = vx_session_clone(session); + assert!(!copy.is_null()); + vx_session_free(session); + + vx_session_free(copy); + } + } +} diff --git a/vortex-ffi/src/struct_array.rs b/vortex-ffi/src/struct_array.rs index 8141ce70e91..19506524b4e 100644 --- a/vortex-ffi/src/struct_array.rs +++ b/vortex-ffi/src/struct_array.rs @@ -28,6 +28,7 @@ crate::box_wrapper!(StructBuilder, vx_struct_column_builder); /// Create a new column-wise struct array builder with given validity and a /// capacity hint. validity can't be NULL. +/// Capacity hint is for the number of columns. /// If you don't know capacity, pass 0. /// if validity is NULL, returns NULL. #[unsafe(no_mangle)] diff --git a/vortex-ffi/test/CMakeLists.txt b/vortex-ffi/test/CMakeLists.txt index 029a63efffc..6ec4c130800 100644 --- a/vortex-ffi/test/CMakeLists.txt +++ b/vortex-ffi/test/CMakeLists.txt @@ -10,9 +10,21 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(Catch) include(Catch) +# https://github.com/catchorg/Catch2/issues/1833 +target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) + +FetchContent_Declare( + Nanoarrow + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0 +) +FetchContent_MakeAvailable(Nanoarrow) file(GLOB TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") message(NOTICE "Test files ${TEST_FILES}") add_executable(vortex_ffi_test ${TEST_FILES}) -target_link_libraries(vortex_ffi_test PRIVATE vortex_ffi_shared Catch2::Catch2WithMain) +target_link_libraries(vortex_ffi_test PRIVATE + vortex_ffi_shared + Catch2::Catch2WithMain + nanoarrow_shared) catch_discover_tests(vortex_ffi_test) diff --git a/vortex-ffi/test/common.h b/vortex-ffi/test/common.h index dbbcf56d318..29087e7a48d 100644 --- a/vortex-ffi/test/common.h +++ b/vortex-ffi/test/common.h @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors #pragma once - #include #include #include "vortex.h" @@ -19,8 +18,28 @@ inline std::string_view to_string_view(vx_error *err) { return to_string_view(vx_error_get_message(err)); } -inline void require_no_error(vx_error *err) { - if (err) { - FAIL(to_string(err)); +inline void require_no_error(vx_error *error, bool assert = true) { + if (!error) { + return; + } + auto message = to_string(error); + vx_error_free(error); + if (assert) { + FAIL(message); + } else { + throw std::runtime_error(message); } } + +template +struct Defer { + Defer(F &&f) : f(std::move(f)) { + } + ~Defer() { + f(); + } + F f; +}; +#define CONCAT(x, y) x##y +#define CONCAT2(x, y) CONCAT(x, y) +#define defer Defer CONCAT2(defer_, __LINE__) = [&] diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp new file mode 100644 index 00000000000..93a476ce614 --- /dev/null +++ b/vortex-ffi/test/scan.cpp @@ -0,0 +1,782 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using FFI_ArrowArrayStream = ArrowArrayStream; +using FFI_ArrowSchema = ArrowSchema; +#define USE_OWN_ARROW 1 +#include + +#include "common.h" + +namespace fs = std::filesystem; +using namespace std::string_literals; +using namespace std::string_view_literals; +using Catch::Matchers::ContainsSubstring; +using nanoarrow::UniqueArray; +using nanoarrow::UniqueArrayStream; +using nanoarrow::UniqueArrayView; +using nanoarrow::UniqueSchema; + +struct TempPath : fs::path { + TempPath() = default; + explicit TempPath(fs::path p) : fs::path(std::move(p)) { + } + + TempPath(const TempPath &) = delete; + TempPath &operator=(const TempPath &) = delete; + + TempPath(TempPath &&other) noexcept : fs::path(std::move(other)) { + } + TempPath &operator=(TempPath &&other) noexcept { + if (this != &other) { + fs::remove(*this); + fs::path::operator=(std::move(other)); + } + return *this; + } + + ~TempPath() { + if (!empty()) { + fs::remove(*this); + } + } +}; + +// StructArray { age=u8, height=u16? } +[[nodiscard]] const vx_dtype *sample_dtype() { + vx_struct_fields_builder *builder = vx_struct_fields_builder_new(); + + constexpr auto age = "age"sv; + const vx_string *age_name = vx_string_new(age.data(), age.size()); + const vx_dtype *age_type = vx_dtype_new_primitive(PTYPE_U8, false); + vx_struct_fields_builder_add_field(builder, age_name, age_type); + + constexpr auto height = "height"sv; + const vx_string *height_name = vx_string_new(height.data(), height.size()); + const vx_dtype *height_type = vx_dtype_new_primitive(PTYPE_U16, true); + vx_struct_fields_builder_add_field(builder, height_name, height_type); + + vx_struct_fields *fields = vx_struct_fields_builder_finalize(builder); + return vx_dtype_new_struct(fields, false); +} + +constexpr size_t SAMPLE_ROWS = 100; +std::vector sample_age() { + std::vector buf; + for (uint8_t age = 0; age < SAMPLE_ROWS; ++age) { + buf.push_back(age); + } + return buf; +} + +std::vector sample_height() { + std::vector buf; + for (uint16_t height = 0; height < SAMPLE_ROWS; ++height) { + buf.push_back((height + 1) % 200); + } + return buf; +} + +[[nodiscard]] const vx_array *sample_array() { + vx_validity validity = {}; + validity.type = VX_VALIDITY_NON_NULLABLE; + + vx_struct_column_builder *builder = vx_struct_column_builder_new(&validity, SAMPLE_ROWS); + vx_error *error = nullptr; + + std::vector age_buffer = sample_age(); + const vx_array *age_array = + vx_array_new_primitive(PTYPE_U8, age_buffer.data(), age_buffer.size(), &validity, &error); + defer { + vx_array_free(age_array); + }; + require_no_error(error); + + vx_struct_column_builder_add_field(builder, "age", age_array, &error); + require_no_error(error); + + std::vector height_buffer = sample_height(); + validity.type = VX_VALIDITY_ALL_VALID; + const vx_array *height_array = + vx_array_new_primitive(PTYPE_U16, height_buffer.data(), height_buffer.size(), &validity, &error); + defer { + vx_array_free(height_array); + }; + require_no_error(error); + + vx_struct_column_builder_add_field(builder, "height", height_array, &error); + require_no_error(error); + + const vx_array *array = vx_struct_column_builder_finalize(builder, &error); + require_no_error(error); + return array; +} + +UniqueSchema sample_schema() { + UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 2) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_UINT8) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "age") == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[1], NANOARROW_TYPE_UINT16) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[1], "height") == NANOARROW_OK); + return schema; +} + +UniqueArrayStream sample_array_stream() { + UniqueSchema schema = sample_schema(); + UniqueArray arr; + + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + + auto ages = sample_age(); + auto heights = sample_height(); + for (size_t i = 0; i < ages.size(); ++i) { + REQUIRE(ArrowArrayAppendInt(arr->children[0], ages[i]) == NANOARROW_OK); + REQUIRE(ArrowArrayAppendInt(arr->children[1], heights[i]) == NANOARROW_OK); + REQUIRE(ArrowArrayFinishElement(arr.get()) == NANOARROW_OK); + } + + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + UniqueArrayStream stream; + REQUIRE(ArrowBasicArrayStreamInit(stream.get(), schema.get(), 1) == NANOARROW_OK); + + ArrowBasicArrayStreamSetArray(stream.get(), 0, arr.get()); + return stream; +} + +[[nodiscard]] TempPath write_sample(vx_session *session) { + const fs::path path = std::filesystem::temp_directory_path() / + fs::path("test-" + std::to_string(std::random_device {}()) + ".vortex"); + + const vx_dtype *dtype = sample_dtype(); + defer { + vx_dtype_free(dtype); + }; + + vx_error *error = nullptr; + vx_array_sink *sink = vx_array_sink_open_file(session, path.c_str(), dtype, &error); + REQUIRE(sink != nullptr); + require_no_error(error); + + const vx_array *array = sample_array(); + defer { + vx_array_free(array); + }; + vx_array_sink_push(sink, array, &error); + require_no_error(error); + + vx_array_sink_close(sink, &error); + require_no_error(error); + + INFO("Written vortex file "s + path.generic_string()); + return TempPath {path}; +} + +TEST_CASE("Creating datasources", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + vx_error *error = nullptr; + + const vx_data_source *ds = vx_data_source_new(session, nullptr, &error); + REQUIRE(ds == nullptr); + REQUIRE(error != nullptr); + vx_error_free(error); + + vx_data_source_options opts = {}; + ds = vx_data_source_new(session, &opts, &error); + REQUIRE(ds == nullptr); + REQUIRE(error != nullptr); + REQUIRE_THAT(to_string(error), ContainsSubstring("opts.paths")); + vx_error_free(error); + + // First file is opened eagerly + opts.paths = "nonexistent"; + ds = vx_data_source_new(session, &opts, &error); + REQUIRE(ds == nullptr); + REQUIRE(error != nullptr); + REQUIRE_THAT(to_string(error), ContainsSubstring("No such file or directory")); + vx_error_free(error); + + opts.paths = "/tmp2/*.vortex"; + ds = vx_data_source_new(session, &opts, &error); + REQUIRE(ds == nullptr); + REQUIRE(error != nullptr); + vx_error_free(error); + + TempPath file = write_sample(session); + opts.paths = file.c_str(); + ds = vx_data_source_new(session, &opts, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + vx_data_source_free(ds); +} + +TEST_CASE("Write file", "[sink]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); +} + +TEST_CASE("Write file and read dtypes", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options opts = {}; + opts.paths = path.c_str(); + + const vx_data_source *ds = vx_data_source_new(session, &opts, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_data_source_row_count row_count = {}; + vx_data_source_get_row_count(ds, &row_count); + + CHECK(row_count.cardinality == VX_CARD_MAXIMUM); + CHECK(row_count.rows == SAMPLE_ROWS); + + const vx_dtype *data_source_dtype = vx_data_source_dtype(ds); + REQUIRE(vx_dtype_get_variant(data_source_dtype) == DTYPE_STRUCT); + + const vx_struct_fields *fields = vx_dtype_struct_dtype(data_source_dtype); + const size_t len = vx_struct_fields_nfields(fields); + REQUIRE(len == 2); + + const vx_dtype *age_dtype = vx_struct_fields_field_dtype(fields, 0); + defer { + vx_dtype_free(age_dtype); + }; + const vx_string *age_name = vx_struct_fields_field_name(fields, 0); + REQUIRE(vx_dtype_get_variant(age_dtype) == DTYPE_PRIMITIVE); + REQUIRE(vx_dtype_primitive_ptype(age_dtype) == PTYPE_U8); + REQUIRE_FALSE(vx_dtype_is_nullable(age_dtype)); + REQUIRE(to_string_view(age_name) == "age"); + + const vx_dtype *height_dtype = vx_struct_fields_field_dtype(fields, 1); + defer { + vx_dtype_free(height_dtype); + }; + const vx_string *height_name = vx_struct_fields_field_name(fields, 1); + REQUIRE(vx_dtype_get_variant(height_dtype) == DTYPE_PRIMITIVE); + REQUIRE(vx_dtype_primitive_ptype(height_dtype) == PTYPE_U16); + REQUIRE(vx_dtype_is_nullable(height_dtype)); + REQUIRE(to_string_view(height_name) == "height"); +} + +void verify_age_field(const vx_array *age_field) { + REQUIRE(vx_array_has_dtype(age_field, DTYPE_PRIMITIVE)); + REQUIRE(vx_dtype_primitive_ptype(vx_array_dtype(age_field)) == PTYPE_U8); + REQUIRE(vx_array_len(age_field) == SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(vx_array_get_u8(age_field, i) == i); + } +} + +void verify_height_field(const vx_array *height_field) { + REQUIRE(vx_array_has_dtype(height_field, DTYPE_PRIMITIVE)); + REQUIRE(vx_dtype_primitive_ptype(vx_array_dtype(height_field)) == PTYPE_U16); + REQUIRE(vx_array_len(height_field) == SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(vx_array_get_u16(height_field, i) > 0); + } +} + +void verify_sample_array(const vx_array *array) { + REQUIRE(vx_array_len(array) == SAMPLE_ROWS); + REQUIRE(vx_array_has_dtype(array, DTYPE_STRUCT)); + + const vx_struct_fields *fields = vx_dtype_struct_dtype(vx_array_dtype(array)); + size_t len = vx_struct_fields_nfields(fields); + REQUIRE(len == 2); + + const vx_dtype *age_dtype = vx_struct_fields_field_dtype(fields, 0); + REQUIRE(age_dtype != nullptr); + REQUIRE(vx_dtype_get_variant(age_dtype) == DTYPE_PRIMITIVE); + REQUIRE(vx_dtype_primitive_ptype(age_dtype) == PTYPE_U8); + vx_dtype_free(age_dtype); + const vx_string *age_name = vx_struct_fields_field_name(fields, 0); + REQUIRE(to_string_view(age_name) == "age"); + + const vx_dtype *height_dtype = vx_struct_fields_field_dtype(fields, 1); + REQUIRE(vx_dtype_get_variant(height_dtype) == DTYPE_PRIMITIVE); + REQUIRE(vx_dtype_primitive_ptype(height_dtype) == PTYPE_U16); + vx_dtype_free(height_dtype); + const vx_string *height_name = vx_struct_fields_field_name(fields, 1); + REQUIRE(to_string_view(height_name) == "height"); + + vx_error *error = nullptr; + vx_validity validity = {}; + vx_array_get_validity(array, &validity, &error); + require_no_error(error); + REQUIRE(validity.type == VX_VALIDITY_NON_NULLABLE); + + const vx_array *age_field = vx_array_get_field(array, 0, &error); + require_no_error(error); + verify_age_field(age_field); + vx_array_free(age_field); + + const vx_array *height_field = vx_array_get_field(array, 1, &error); + require_no_error(error); + verify_height_field(height_field); + vx_array_free(height_field); + + REQUIRE(vx_array_get_field(array, 2, &error) == nullptr); + REQUIRE(error != nullptr); + vx_error_free(error); +} + +TEST_CASE("Requesting scans", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + + vx_error *error = nullptr; + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + { + vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); + require_no_error(error); + REQUIRE(scan != nullptr); + vx_scan_free(scan); + } + + vx_scan_options options = {}; + options.max_threads = 1; + + { + vx_scan *scan = vx_data_source_scan(ds, &options, nullptr, &error); + require_no_error(error); + REQUIRE(scan != nullptr); + vx_scan_free(scan); + } +} + +TEST_CASE("Basic scan", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_estimate estimate = {}; + vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); + require_no_error(error); + defer { + vx_scan_free(scan); + }; + REQUIRE(scan != nullptr); + REQUIRE(estimate.type == VX_ESTIMATE_EXACT); + REQUIRE(estimate.estimate == 1); + + vx_partition *partition = vx_scan_next_partition(scan, &error); + require_no_error(error); + defer { + vx_partition_free(partition); + }; + + estimate = {}; + vx_partition_row_count(partition, &estimate, &error); + require_no_error(error); + REQUIRE(estimate.type == VX_ESTIMATE_EXACT); + REQUIRE(estimate.estimate == SAMPLE_ROWS); + + REQUIRE(vx_scan_next_partition(scan, &error) == nullptr); + require_no_error(error); + + const vx_array *array = vx_partition_next(partition, &error); + require_no_error(error); + REQUIRE(array != nullptr); + defer { + vx_array_free(array); + }; + + REQUIRE(vx_partition_next(partition, &error) == nullptr); + require_no_error(error); + + verify_sample_array(array); +} + +TEST_CASE("Multithreaded scan", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + + constexpr size_t NUM_FILES = 10; + std::vector paths(NUM_FILES); + std::string paths_str; + for (size_t i = 0; i < NUM_FILES; ++i) { + paths[i] = write_sample(session); + if (i == 0) { + paths_str = paths[i].c_str(); + } else { + paths_str += ","s + paths[i].c_str(); + } + } + + vx_data_source_options ds_options = {}; + ds_options.paths = paths_str.c_str(); + + vx_error *error = nullptr; + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_estimate estimate = {}; + vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); + require_no_error(error); + defer { + vx_scan_free(scan); + }; + REQUIRE(scan != nullptr); + REQUIRE(estimate.type == VX_ESTIMATE_INEXACT); + REQUIRE(estimate.estimate == NUM_FILES); + + // Catch doesn't support multithreaded assertions, so we throw here + std::mutex mutex; + std::vector threads(NUM_FILES); + threads.reserve(NUM_FILES); + std::vector arrays(NUM_FILES); + for (size_t i = 0; i < NUM_FILES; ++i) { + threads[i] = std::thread([&, i] { + vx_error *error_tl = nullptr; + vx_partition *partition = nullptr; + { + std::lock_guard _(mutex); + partition = vx_scan_next_partition(scan, &error_tl); + require_no_error(error_tl, false); + if (!partition) { + throw std::runtime_error("partition = nullptr"); + } + } + + defer { + vx_partition_free(partition); + }; + + estimate = {}; + vx_partition_row_count(partition, &estimate, &error_tl); + require_no_error(error_tl, false); + if (estimate.type != VX_ESTIMATE_EXACT) { + throw std::runtime_error("estimate type mismatch"); + } + if (estimate.estimate != SAMPLE_ROWS) { + throw std::runtime_error("estimate mismatch"); + } + + const vx_array *array = vx_partition_next(partition, &error_tl); + require_no_error(error_tl, false); + if (!array) { + throw std::runtime_error("array = nullptr"); + } + + arrays[i] = array; + }); + } + for (auto &thread : threads) { + thread.join(); + } + + for (const vx_array *array : arrays) { + REQUIRE(array != nullptr); + defer { + vx_array_free(array); + }; + verify_sample_array(array); + } +} + +const vx_array *scan_with_options(vx_scan_options &options) { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_scan *scan = vx_data_source_scan(ds, &options, nullptr, &error); + require_no_error(error); + REQUIRE(scan != nullptr); + defer { + vx_scan_free(scan); + }; + + vx_partition *partition = vx_scan_next_partition(scan, &error); + require_no_error(error); + REQUIRE(partition != nullptr); + defer { + vx_partition_free(partition); + }; + + const vx_array *array = vx_partition_next(partition, &error); + require_no_error(error); + REQUIRE(array != nullptr); + + return array; +} + +TEST_CASE("Project all fields", "[projection]") { + vx_scan_options opts = {}; + const vx_array *array = scan_with_options(opts); + defer { + vx_array_free(array); + }; + verify_sample_array(array); +} + +TEST_CASE("Project root", "[projection]") { + vx_expression *root = vx_expression_root(); + defer { + vx_expression_free(root); + }; + vx_scan_options opts = {}; + opts.projection = root; + const vx_array *array = scan_with_options(opts); + defer { + vx_array_free(array); + }; + verify_sample_array(array); +} + +TEST_CASE("Project single field", "[projection]") { + vx_expression *root = vx_expression_root(); + defer { + vx_expression_free(root); + }; + vx_scan_options opts = {}; + + vx_expression *age_field = vx_expression_get_item("age", root); + REQUIRE(age_field != nullptr); + defer { + vx_expression_free(age_field); + }; + + { + opts.projection = age_field; + const vx_array *array = scan_with_options(opts); + defer { + vx_array_free(array); + }; + verify_age_field(array); + } + + vx_expression *height_field = vx_expression_get_item("height", root); + REQUIRE(height_field != nullptr); + defer { + vx_expression_free(height_field); + }; + + { + opts.projection = height_field; + const vx_array *array = scan_with_options(opts); + defer { + vx_array_free(array); + }; + verify_height_field(array); + } +} + +void compare_schemas(const UniqueSchema &left, const UniqueSchema &right) { + REQUIRE(std::string_view {left->format} == std::string_view {right->format}); + REQUIRE(left->n_children == right->n_children); + for (int64_t i = 0; i < left->n_children; i++) { + std::string_view name_left = left->children[i]->name; + std::string_view name_right = right->children[i]->name; + REQUIRE(name_left == name_right); + compare_schemas(left->children[i], right->children[i]); + } +} + +void compare_schema_with_sample(const UniqueSchema &left) { + compare_schemas(left, sample_schema()); +} + +void compare_stream_with_sample(UniqueArrayStream &left) { + UniqueArrayStream right = sample_array_stream(); + UniqueSchema schema_right = sample_schema(); + + ArrowError error; + UniqueSchema schema_left; + ArrowErrorCode res = ArrowArrayStreamGetSchema(left.get(), schema_left.get(), &error); + REQUIRE(res == NANOARROW_OK); + + while (true) { + UniqueArray chunk_left, chunk_right; + int next_left = left->get_next(left.get(), chunk_left.get()); + int next_right = right->get_next(right.get(), chunk_right.get()); + REQUIRE(next_left == next_right); + if (next_left != NANOARROW_OK || next_right != NANOARROW_OK) { + return; + } + + bool done_left = chunk_left->release == nullptr; + bool done_right = chunk_right->release == nullptr; + REQUIRE(done_left == done_right); + if (done_left || done_right) { + return; + } + + REQUIRE(chunk_left->length == chunk_right->length); + + UniqueArrayView view_left, view_right; + res = ArrowArrayViewInitFromSchema(view_left.get(), schema_left.get(), nullptr); + REQUIRE(res == NANOARROW_OK); + res = ArrowArrayViewInitFromSchema(view_right.get(), schema_right.get(), nullptr); + REQUIRE(res == NANOARROW_OK); + + res = ArrowArrayViewSetArray(view_left.get(), chunk_left.get(), nullptr); + REQUIRE(res == NANOARROW_OK); + res = ArrowArrayViewSetArray(view_right.get(), chunk_right.get(), nullptr); + REQUIRE(res == NANOARROW_OK); + + for (int64_t i = 0; i < chunk_left->length; i++) { + auto name_left = ArrowArrayViewGetIntUnsafe(view_left->children[0], i); + auto name_right = ArrowArrayViewGetIntUnsafe(view_right->children[0], i); + REQUIRE(name_left == name_right); + + auto age_left = ArrowArrayViewGetIntUnsafe(view_left->children[1], i); + auto age_right = ArrowArrayViewGetIntUnsafe(view_right->children[1], i); + REQUIRE(age_left == age_right); + } + } +} + +TEST_CASE("Scan Arrow schema", "[scan]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); + require_no_error(error); + REQUIRE(scan != nullptr); + defer { + vx_scan_free(scan); + }; + + ArrowSchema schema; + const vx_dtype *dtype = vx_scan_dtype(scan, &error); + require_no_error(error); + REQUIRE(dtype != nullptr); + defer { + vx_dtype_free(dtype); + }; + + int res = vx_dtype_to_arrow_schema(dtype, &schema, &error); + REQUIRE(res == 0); + require_no_error(error); + + UniqueSchema unique_schema; + ArrowSchemaMove(&schema, unique_schema.get()); + compare_schema_with_sample(unique_schema); +} + +TEST_CASE("Scan to Arrow", "[scan]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); + require_no_error(error); + REQUIRE(scan != nullptr); + defer { + vx_scan_free(scan); + }; + + vx_partition *partition = vx_scan_next_partition(scan, &error); + require_no_error(error); + REQUIRE(partition != nullptr); + + UniqueArrayStream unique_stream; + { + ArrowArrayStream stream = {}; + int res = vx_partition_scan_arrow(session, partition, &stream, &error); + REQUIRE(res == 0); + require_no_error(error); + ArrowArrayStreamMove(&stream, unique_stream.get()); + } + compare_stream_with_sample(unique_stream); +} From 6ac69e7b0a6d653dd94472fbd39c55d354b7020f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:36:18 +0000 Subject: [PATCH 096/250] Update Rust crate rand to 0.10 [SECURITY] (#7508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [rand](https://rust-random.github.io/book) ([source](https://redirect.github.com/rust-random/rand)) | dev-dependencies | minor | `0.8` → `0.10` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. ### GitHub Vulnerability Alerts #### [GHSA-cq8v-f236-94qc](https://redirect.github.com/rust-random/rand/pull/1763) It has been reported (by @​lopopolo) that the `rand` library is [unsound](https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#soundness-of-code--of-a-library) (i.e. that safe code using the public API can cause Undefined Behaviour) when all the following conditions are met: - The `log` and `thread_rng` features are enabled - A [custom logger](https://docs.rs/log/latest/log/#implementing-a-logger) is defined - The custom logger accesses `rand::rng()` (previously `rand::thread_rng()`) and calls any `TryRng` (previously `RngCore`) methods on `ThreadRng` - The `ThreadRng` (attempts to) reseed while called from the custom logger (this happens every 64 kB of generated data) - Trace-level logging is enabled or warn-level logging is enabled and the random source (the `getrandom` crate) is unable to provide a new seed `TryRng` (previously `RngCore`) methods for `ThreadRng` use `unsafe` code to cast `*mut BlockRng` to `&mut BlockRng`. When all the above conditions are met this results in an aliased mutable reference, violating the Stacked Borrows rules. Miri is able to detect this violation in sample code. Since construction of [aliased mutable references is Undefined Behaviour](https://doc.rust-lang.org/stable/nomicon/references.html), the behaviour of optimized builds is hard to predict. Affected versions of `rand` are `>= 0.7, < 0.9.3` and `0.10.0`. ##### Severity Low --- ### Release Notes

rust-random/rand (rand) ### [`v0.10.1`](https://redirect.github.com/rust-random/rand/blob/HEAD/CHANGELOG.md#0101--2026-02-11) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.10.0...0.10.1) This release includes a fix for a soundness bug; see [#​1763]. ##### Changes - Document panic behavior of `make_rng` and add `#[track_caller]` ([#​1761]) - Deprecate feature `log` ([#​1763]) [#​1761]: https://redirect.github.com/rust-random/rand/pull/1761 [#​1763]: https://redirect.github.com/rust-random/rand/pull/1763 ### [`v0.10.0`](https://redirect.github.com/rust-random/rand/blob/HEAD/CHANGELOG.md#0100---2026-02-08) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.9.4...0.10.0) ##### Changes - The dependency on `rand_chacha` has been replaced with a dependency on `chacha20`. This changes the implementation behind `StdRng`, but the output remains the same. There may be some API breakage when using the ChaCha-types directly as these are now the ones in `chacha20` instead of `rand_chacha` ([#​1642]). - Rename fns `IndexedRandom::choose_multiple` -> `sample`, `choose_multiple_array` -> `sample_array`, `choose_multiple_weighted` -> `sample_weighted`, struct `SliceChooseIter` -> `IndexedSamples` and fns `IteratorRandom::choose_multiple` -> `sample`, `choose_multiple_fill` -> `sample_fill` ([#​1632]) - Use Edition 2024 and MSRV 1.85 ([#​1653]) - Let `Fill` be implemented for element types, not sliceable types ([#​1652]) - Fix `OsError::raw_os_error` on UEFI targets by returning `Option` ([#​1665]) - Replace fn `TryRngCore::read_adapter(..) -> RngReadAdapter` with simpler struct `RngReader` ([#​1669]) - Remove fns `SeedableRng::from_os_rng`, `try_from_os_rng` ([#​1674]) - Remove `Clone` support for `StdRng`, `ReseedingRng` ([#​1677]) - Use `postcard` instead of `bincode` to test the serde feature ([#​1693]) - Avoid excessive allocation in `IteratorRandom::sample` when `amount` is much larger than iterator size ([#​1695]) - Rename `os_rng` -> `sys_rng`, `OsRng` -> `SysRng`, `OsError` -> `SysError` ([#​1697]) - Rename `Rng` -> `RngExt` as upstream `rand_core` has renamed `RngCore` -> `Rng` ([#​1717]) ##### Additions - Add fns `IndexedRandom::choose_iter`, `choose_weighted_iter` ([#​1632]) - Pub export `Xoshiro128PlusPlus`, `Xoshiro256PlusPlus` prngs ([#​1649]) - Pub export `ChaCha8Rng`, `ChaCha12Rng`, `ChaCha20Rng` behind `chacha` feature ([#​1659]) - Fn `rand::make_rng() -> R where R: SeedableRng` ([#​1734]) ##### Removals - Removed `ReseedingRng` ([#​1722]) - Removed unused feature "nightly" ([#​1732]) - Removed feature `small_rng` ([#​1732]) [#​1632]: https://redirect.github.com/rust-random/rand/pull/1632 [#​1642]: https://redirect.github.com/rust-random/rand/pull/1642 [#​1649]: https://redirect.github.com/rust-random/rand/pull/1649 [#​1652]: https://redirect.github.com/rust-random/rand/pull/1652 [#​1653]: https://redirect.github.com/rust-random/rand/pull/1653 [#​1659]: https://redirect.github.com/rust-random/rand/pull/1659 [#​1665]: https://redirect.github.com/rust-random/rand/pull/1665 [#​1669]: https://redirect.github.com/rust-random/rand/pull/1669 [#​1674]: https://redirect.github.com/rust-random/rand/pull/1674 [#​1677]: https://redirect.github.com/rust-random/rand/pull/1677 [#​1693]: https://redirect.github.com/rust-random/rand/pull/1693 [#​1695]: https://redirect.github.com/rust-random/rand/pull/1695 [#​1697]: https://redirect.github.com/rust-random/rand/pull/1697 [#​1717]: https://redirect.github.com/rust-random/rand/pull/1717 [#​1722]: https://redirect.github.com/rust-random/rand/pull/1722 [#​1732]: https://redirect.github.com/rust-random/rand/pull/1732 [#​1734]: https://redirect.github.com/rust-random/rand/pull/1734 ### [`v0.9.4`](https://redirect.github.com/rust-random/rand/releases/tag/0.9.4) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.9.3...0.9.4) ##### Fixes - Fix doc build ([#​1766]) [#​1766]: https://redirect.github.com/rust-random/rand/pull/1766 **Full Changelog**: ### [`v0.9.3`](https://redirect.github.com/rust-random/rand/compare/0.9.2...0.9.3) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.9.2...0.9.3) ### [`v0.9.2`](https://redirect.github.com/rust-random/rand/blob/HEAD/CHANGELOG.md#092---2025-07-20) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.9.1...0.9.2) ##### Deprecated - Deprecate `rand::rngs::mock` module and `StepRng` generator ([#​1634](https://redirect.github.com/rust-random/rand/issues/1634)) ##### Additions - Enable `WeightedIndex` (de)serialization ([#​1646](https://redirect.github.com/rust-random/rand/issues/1646)) ### [`v0.9.1`](https://redirect.github.com/rust-random/rand/blob/HEAD/CHANGELOG.md#091---2025-04-17) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.9.0...0.9.1) ##### Security and unsafe - Revise "not a crypto library" policy again ([#​1565](https://redirect.github.com/rust-random/rand/issues/1565)) - Remove `zerocopy` dependency from `rand` ([#​1579](https://redirect.github.com/rust-random/rand/issues/1579)) ##### Fixes - Fix feature `simd_support` for recent nightly rust ([#​1586](https://redirect.github.com/rust-random/rand/issues/1586)) ##### Changes - Allow `fn rand::seq::index::sample_weighted` and `fn IndexedRandom::choose_multiple_weighted` to return fewer than `amount` results ([#​1623](https://redirect.github.com/rust-random/rand/issues/1623)), reverting an undocumented change ([#​1382](https://redirect.github.com/rust-random/rand/issues/1382)) to the previous release. ##### Additions - Add `rand::distr::Alphabetic` distribution. ([#​1587](https://redirect.github.com/rust-random/rand/issues/1587)) - Re-export `rand_core` ([#​1604](https://redirect.github.com/rust-random/rand/issues/1604)) ### [`v0.9.0`](https://redirect.github.com/rust-random/rand/blob/HEAD/CHANGELOG.md#090---2025-01-27) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.8.6...0.9.0) ##### Security and unsafe - Policy: "rand is not a crypto library" ([#​1514](https://redirect.github.com/rust-random/rand/issues/1514)) - Remove fork-protection from `ReseedingRng` and `ThreadRng`. Instead, it is recommended to call `ThreadRng::reseed` on fork. ([#​1379](https://redirect.github.com/rust-random/rand/issues/1379)) - Use `zerocopy` to replace some `unsafe` code ([#​1349](https://redirect.github.com/rust-random/rand/issues/1349), [#​1393](https://redirect.github.com/rust-random/rand/issues/1393), [#​1446](https://redirect.github.com/rust-random/rand/issues/1446), [#​1502](https://redirect.github.com/rust-random/rand/issues/1502)) ##### Dependencies - Bump the MSRV to 1.63.0 ([#​1207](https://redirect.github.com/rust-random/rand/issues/1207), [#​1246](https://redirect.github.com/rust-random/rand/issues/1246), [#​1269](https://redirect.github.com/rust-random/rand/issues/1269), [#​1341](https://redirect.github.com/rust-random/rand/issues/1341), [#​1416](https://redirect.github.com/rust-random/rand/issues/1416), [#​1536](https://redirect.github.com/rust-random/rand/issues/1536)); note that 1.60.0 may work for dependents when using `--ignore-rust-version` - Update to `rand_core` v0.9.0 ([#​1558](https://redirect.github.com/rust-random/rand/issues/1558)) ##### Features - Support `std` feature without `getrandom` or `rand_chacha` ([#​1354](https://redirect.github.com/rust-random/rand/issues/1354)) - Enable feature `small_rng` by default ([#​1455](https://redirect.github.com/rust-random/rand/issues/1455)) - Remove implicit feature `rand_chacha`; use `std_rng` instead. ([#​1473](https://redirect.github.com/rust-random/rand/issues/1473)) - Rename feature `serde1` to `serde` ([#​1477](https://redirect.github.com/rust-random/rand/issues/1477)) - Rename feature `getrandom` to `os_rng` ([#​1537](https://redirect.github.com/rust-random/rand/issues/1537)) - Add feature `thread_rng` ([#​1547](https://redirect.github.com/rust-random/rand/issues/1547)) ##### API changes: rand\_core traits - Add fn `RngCore::read_adapter` implementing `std::io::Read` ([#​1267](https://redirect.github.com/rust-random/rand/issues/1267)) - Add trait `CryptoBlockRng: BlockRngCore`; make `trait CryptoRng: RngCore` ([#​1273](https://redirect.github.com/rust-random/rand/issues/1273)) - Add traits `TryRngCore`, `TryCryptoRng` ([#​1424](https://redirect.github.com/rust-random/rand/issues/1424), [#​1499](https://redirect.github.com/rust-random/rand/issues/1499)) - Rename `fn SeedableRng::from_rng` -> `try_from_rng` and add infallible variant `fn from_rng` ([#​1424](https://redirect.github.com/rust-random/rand/issues/1424)) - Rename `fn SeedableRng::from_entropy` -> `from_os_rng` and add fallible variant `fn try_from_os_rng` ([#​1424](https://redirect.github.com/rust-random/rand/issues/1424)) - Add bounds `Clone` and `AsRef` to associated type `SeedableRng::Seed` ([#​1491](https://redirect.github.com/rust-random/rand/issues/1491)) ##### API changes: Rng trait and top-level fns - Rename fn `rand::thread_rng()` to `rand::rng()` and remove from the prelude ([#​1506](https://redirect.github.com/rust-random/rand/issues/1506)) - Remove fn `rand::random()` from the prelude ([#​1506](https://redirect.github.com/rust-random/rand/issues/1506)) - Add top-level fns `random_iter`, `random_range`, `random_bool`, `random_ratio`, `fill` ([#​1488](https://redirect.github.com/rust-random/rand/issues/1488)) - Re-introduce fn `Rng::gen_iter` as `random_iter` ([#​1305](https://redirect.github.com/rust-random/rand/issues/1305), [#​1500](https://redirect.github.com/rust-random/rand/issues/1500)) - Rename fn `Rng::gen` to `random` to avoid conflict with the new `gen` keyword in Rust 2024 ([#​1438](https://redirect.github.com/rust-random/rand/issues/1438)) - Rename fns `Rng::gen_range` to `random_range`, `gen_bool` to `random_bool`, `gen_ratio` to `random_ratio` ([#​1505](https://redirect.github.com/rust-random/rand/issues/1505)) - Annotate panicking methods with `#[track_caller]` ([#​1442](https://redirect.github.com/rust-random/rand/issues/1442), [#​1447](https://redirect.github.com/rust-random/rand/issues/1447)) ##### API changes: RNGs - Fix `::Seed` size to 256 bits ([#​1455](https://redirect.github.com/rust-random/rand/issues/1455)) - Remove first parameter (`rng`) of `ReseedingRng::new` ([#​1533](https://redirect.github.com/rust-random/rand/issues/1533)) ##### API changes: Sequences - Split trait `SliceRandom` into `IndexedRandom`, `IndexedMutRandom`, `SliceRandom` ([#​1382](https://redirect.github.com/rust-random/rand/issues/1382)) - Add `IndexedRandom::choose_multiple_array`, `index::sample_array` ([#​1453](https://redirect.github.com/rust-random/rand/issues/1453), [#​1469](https://redirect.github.com/rust-random/rand/issues/1469)) ##### API changes: Distributions: renames - Rename module `rand::distributions` to `rand::distr` ([#​1470](https://redirect.github.com/rust-random/rand/issues/1470)) - Rename distribution `Standard` to `StandardUniform` ([#​1526](https://redirect.github.com/rust-random/rand/issues/1526)) - Move `distr::Slice` -> `distr::slice::Choose`, `distr::EmptySlice` -> `distr::slice::Empty` ([#​1548](https://redirect.github.com/rust-random/rand/issues/1548)) - Rename trait `distr::DistString` -> `distr::SampleString` ([#​1548](https://redirect.github.com/rust-random/rand/issues/1548)) - Rename `distr::DistIter` -> `distr::Iter`, `distr::DistMap` -> `distr::Map` ([#​1548](https://redirect.github.com/rust-random/rand/issues/1548)) ##### API changes: Distributions - Relax `Sized` bound on `Distribution for &D` ([#​1278](https://redirect.github.com/rust-random/rand/issues/1278)) - Remove impl of `Distribution>` for `StandardUniform` ([#​1526](https://redirect.github.com/rust-random/rand/issues/1526)) - Let distribution `StandardUniform` support all `NonZero*` types ([#​1332](https://redirect.github.com/rust-random/rand/issues/1332)) - Fns `{Uniform, UniformSampler}::{new, new_inclusive}` return a `Result` (instead of potentially panicking) ([#​1229](https://redirect.github.com/rust-random/rand/issues/1229)) - Distribution `Uniform` implements `TryFrom` instead of `From` for ranges ([#​1229](https://redirect.github.com/rust-random/rand/issues/1229)) - Add `UniformUsize` ([#​1487](https://redirect.github.com/rust-random/rand/issues/1487)) - Remove support for generating `isize` and `usize` values with `StandardUniform`, `Uniform` (except via `UniformUsize`) and `Fill` and usage as a `WeightedAliasIndex` weight ([#​1487](https://redirect.github.com/rust-random/rand/issues/1487)) - Add impl `DistString` for distributions `Slice` and `Uniform` ([#​1315](https://redirect.github.com/rust-random/rand/issues/1315)) - Add fn `Slice::num_choices` ([#​1402](https://redirect.github.com/rust-random/rand/issues/1402)) - Add fn `p()` for distribution `Bernoulli` to access probability ([#​1481](https://redirect.github.com/rust-random/rand/issues/1481)) ##### API changes: Weighted distributions - Add `pub` module `rand::distr::weighted`, moving `WeightedIndex` there ([#​1548](https://redirect.github.com/rust-random/rand/issues/1548)) - Add trait `weighted::Weight`, allowing `WeightedIndex` to trap overflow ([#​1353](https://redirect.github.com/rust-random/rand/issues/1353)) - Add fns `weight, weights, total_weight` to distribution `WeightedIndex` ([#​1420](https://redirect.github.com/rust-random/rand/issues/1420)) - Rename enum `WeightedError` to `weighted::Error`, revising variants ([#​1382](https://redirect.github.com/rust-random/rand/issues/1382)) and mark as `#[non_exhaustive]` ([#​1480](https://redirect.github.com/rust-random/rand/issues/1480)) ##### API changes: SIMD - Switch to `std::simd`, expand SIMD & docs ([#​1239](https://redirect.github.com/rust-random/rand/issues/1239)) ##### Reproducibility-breaking changes - Make `ReseedingRng::reseed` discard remaining data from the last block generated ([#​1379](https://redirect.github.com/rust-random/rand/issues/1379)) - Change fn `SmallRng::seed_from_u64` implementation ([#​1203](https://redirect.github.com/rust-random/rand/issues/1203)) - Allow `UniformFloat::new` samples and `UniformFloat::sample_single` to yield `high` ([#​1462](https://redirect.github.com/rust-random/rand/issues/1462)) - Fix portability of distribution `Slice` ([#​1469](https://redirect.github.com/rust-random/rand/issues/1469)) - Make `Uniform` for `usize` portable via `UniformUsize` ([#​1487](https://redirect.github.com/rust-random/rand/issues/1487)) - Fix `IndexdRandom::choose_multiple_weighted` for very small seeds and optimize for large input length / low memory ([#​1530](https://redirect.github.com/rust-random/rand/issues/1530)) ##### Reproducibility-breaking optimisations - Optimize fn `sample_floyd`, affecting output of `rand::seq::index::sample` and `rand::seq::SliceRandom::choose_multiple` ([#​1277](https://redirect.github.com/rust-random/rand/issues/1277)) - New, faster algorithms for `IteratorRandom::choose` and `choose_stable` ([#​1268](https://redirect.github.com/rust-random/rand/issues/1268)) - New, faster algorithms for `SliceRandom::shuffle` and `partial_shuffle` ([#​1272](https://redirect.github.com/rust-random/rand/issues/1272)) - Optimize distribution `Uniform`: use Canon's method (single sampling) / Lemire's method (distribution sampling) for faster sampling (breaks value stability; [#​1287](https://redirect.github.com/rust-random/rand/issues/1287)) - Optimize fn `sample_single_inclusive` for floats (+\~20% perf) ([#​1289](https://redirect.github.com/rust-random/rand/issues/1289)) ##### Other optimisations - Improve `SmallRng` initialization performance ([#​1482](https://redirect.github.com/rust-random/rand/issues/1482)) - Optimise SIMD widening multiply ([#​1247](https://redirect.github.com/rust-random/rand/issues/1247)) ##### Other - Add `Cargo.lock.msrv` file ([#​1275](https://redirect.github.com/rust-random/rand/issues/1275)) - Reformat with `rustfmt` and enforce ([#​1448](https://redirect.github.com/rust-random/rand/issues/1448)) - Apply Clippy suggestions and enforce ([#​1448](https://redirect.github.com/rust-random/rand/issues/1448), [#​1474](https://redirect.github.com/rust-random/rand/issues/1474)) - Move all benchmarks to new `benches` crate ([#​1329](https://redirect.github.com/rust-random/rand/issues/1329), [#​1439](https://redirect.github.com/rust-random/rand/issues/1439)) and migrate to Criterion ([#​1490](https://redirect.github.com/rust-random/rand/issues/1490)) ##### Documentation - Improve `ThreadRng` related docs ([#​1257](https://redirect.github.com/rust-random/rand/issues/1257)) - Docs: enable experimental `--generate-link-to-definition` feature ([#​1327](https://redirect.github.com/rust-random/rand/issues/1327)) - Better doc of crate features, use `doc_auto_cfg` ([#​1411](https://redirect.github.com/rust-random/rand/issues/1411), [#​1450](https://redirect.github.com/rust-random/rand/issues/1450)) ### [`v0.8.6`](https://redirect.github.com/rust-random/rand/releases/tag/0.8.6) [Compare Source](https://redirect.github.com/rust-random/rand/compare/0.8.5...0.8.6) #### What's Changed This release back-ports a fix from v0.10. See also [#​1763]. ##### Changes - Deprecate feature `log` ([#​1772]) [#​1763]: https://redirect.github.com/rust-random/rand/pull/1763 [#​1772]: https://redirect.github.com/rust-random/rand/pull/1772 - Drop the experimental `simd_support` feature. #### New Contributors - [@​nwalfield](https://redirect.github.com/nwalfield) made their first contribution in [#​1772](https://redirect.github.com/rust-random/rand/pull/1772) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - "" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Adam Gutglick Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Adam Gutglick --- Cargo.lock | 2 +- vortex-ffi/Cargo.toml | 2 +- vortex-ffi/src/lib.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad18bbb2cef..0dc36ecd49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10574,7 +10574,7 @@ dependencies = [ "object_store 0.13.2", "paste", "prost 0.14.3", - "rand 0.8.5", + "rand 0.10.1", "tempfile", "tracing", "tracing-subscriber", diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index c8daac6200d..83a0737e2bd 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -35,7 +35,7 @@ url = { workspace = true, features = [] } vortex = { workspace = true, features = ["object_store"] } [dev-dependencies] -rand = "0.8" +rand = { workspace = true } tempfile = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } diff --git a/vortex-ffi/src/lib.rs b/vortex-ffi/src/lib.rs index fb675c558bc..a72466734cd 100644 --- a/vortex-ffi/src/lib.rs +++ b/vortex-ffi/src/lib.rs @@ -85,7 +85,7 @@ mod tests { use std::ptr; use std::sync::Arc; - use rand::Rng; + use rand::RngExt; use tempfile::NamedTempFile; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; @@ -126,11 +126,11 @@ mod tests { fn random_str(length: usize) -> String { const CHARSET: &[u8] = b"0123456789"; - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); (0..length) .map(|_| { - let idx = rng.gen_range(0..CHARSET.len()); + let idx = rng.random_range(0..CHARSET.len()); CHARSET[idx] as char }) .collect() From 6ebb797cbd607c2cd81457e0562146cb05d31c5f Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Fri, 17 Apr 2026 12:02:31 +0100 Subject: [PATCH 097/250] chore: speed up CUDA kernel compilation (#7509) Previously, changing a .cu file triggered cargo to rerun vortex-cuda's build script, which recompiled all CUDA kernels and regenerated bindgen output, causing a full Rust recompilation (~20s). Now CUDA kernel compilation lives in a standalone vortex-cuda-kernel-build workspace member with no dependency link to vortex-cuda, so .cu changes only rebuild PTX without touching Rust. Changes: - New vortex-cuda/kernel-build crate owns .cu/.cuh file watching, nvcc compilation, and bit_unpack kernel generation - Timestamp-based PTX skipping: only recompiles .cu files whose PTX is older than the source or any header --------- Signed-off-by: Alexander Droste --- vortex-cuda/build.rs | 89 ++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index b8673cdda83..90e0de1d881 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -3,10 +3,11 @@ #![expect(clippy::unwrap_used)] #![expect(clippy::expect_used)] +#![expect(clippy::panic)] #![expect(clippy::use_debug)] use std::env; -use std::fs::File; +use std::fs; use std::io; use std::path::Path; use std::path::PathBuf; @@ -30,11 +31,10 @@ fn main() { // Output directory for compiled .ptx files - separate by profile. let kernels_gen = Path::new(&manifest_dir).join("kernels/gen").join(&profile); - std::fs::create_dir_all(&kernels_gen).expect("Failed to create kernels/gen directory"); + fs::create_dir_all(&kernels_gen).expect("Failed to create kernels/gen directory"); - // Always emit the kernels output directory path as a compile-time env var so any binary - // linking against vortex-cuda can find the PTX files. This must be set regardless - // of CUDA availability since the code using env!() is always compiled. + // Emit the kernels output directory path as a compile-time env var so any binary + // linking against vortex-cuda can find the PTX files. // At runtime, VORTEX_CUDA_KERNELS_DIR can be set to override this path. println!( "cargo:rustc-env=VORTEX_CUDA_KERNELS_DIR={}", @@ -43,7 +43,7 @@ fn main() { println!("cargo:rerun-if-env-changed=PROFILE"); - // Regenerate bit_unpack kernels only when the generator changes + // Regenerate bit_unpack kernels only when the generator changes. println!( "cargo:rerun-if-changed={}", Path::new(&manifest_dir) @@ -63,55 +63,73 @@ fn main() { return; } - // Watch and compile .cu and .cuh files from kernels/src to PTX in kernels/gen - if let Ok(entries) = std::fs::read_dir(&kernels_src) { - for path in entries.flatten().map(|entry| entry.path()) { - let is_generated = path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.starts_with("bit_unpack_")); + // Compile .cu files to PTX. We deliberately do NOT register .cu/.cuh files + // with rerun-if-changed so that editing a .cu file does not trigger Rust + // recompilation. + let mut cu_files = Vec::new(); + let mut newest_header = std::time::SystemTime::UNIX_EPOCH; + if let Ok(entries) = fs::read_dir(&kernels_src) { + for path in entries.flatten().map(|entry| entry.path()) { match path.extension().and_then(|e| e.to_str()) { Some("cuh") | Some("h") => { - // Only watch hand-written .cuh/.h files, not generated ones - // (generated files are rebuilt when cuda_kernel_generator changes) - if !is_generated { - println!("cargo:rerun-if-changed={}", path.display()); + if let Ok(mtime) = fs::metadata(&path).and_then(|m| m.modified()) { + newest_header = newest_header.max(mtime); } } Some("cu") => { - // Only watch hand-written .cu files, not generated ones - // (generated files are rebuilt when cuda_kernel_generator changes) - if !is_generated { - println!("cargo:rerun-if-changed={}", path.display()); - } - // Compile all .cu files to PTX in gen directory - nvcc_compile_ptx(&kernels_src, &kernels_gen, &path, &profile) - .map_err(|e| { - format!("Failed to compile CUDA kernel {}: {}", path.display(), e) - }) - .unwrap(); + cu_files.push(path); } _ => {} } } } + + // Only compile .cu files whose PTX is stale (older than the source or any header). + for cu_path in &cu_files { + let ptx_path = kernels_gen + .join(cu_path.file_name().unwrap()) + .with_extension("ptx"); + + let cu_mtime = fs::metadata(cu_path) + .and_then(|m| m.modified()) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + let newest_input = cu_mtime.max(newest_header); + + let ptx_mtime = fs::metadata(&ptx_path).and_then(|m| m.modified()).ok(); + if ptx_mtime.is_some_and(|t| t >= newest_input) { + continue; + } + + nvcc_compile_ptx(&kernels_src, &kernels_gen, cu_path, &profile) + .map_err(|e| format!("Failed to compile CUDA kernel {}: {}", cu_path.display(), e)) + .unwrap(); + } } -fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result { +fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result<()> { // Generate the lanes header (.cuh) — device functions only, no __global__ kernels. // This is what dynamic_dispatch.cu includes (via bit_unpack.cuh). let cuh_path = output_dir.join(format!("bit_unpack_{}_lanes.cuh", T::T)); - let mut cuh_file = File::create(&cuh_path)?; - generate_cuda_unpack_lanes::(&mut cuh_file)?; + let mut cuh_buf = Vec::new(); + generate_cuda_unpack_lanes::(&mut cuh_buf)?; + write_if_changed(&cuh_path, &cuh_buf); // Generate the standalone kernels (.cu) — includes the lanes header, // adds _device template + __global__ wrappers. Compiled to its own PTX. let cu_path = output_dir.join(format!("bit_unpack_{}.cu", T::T)); - let mut cu_file = File::create(&cu_path)?; - generate_cuda_unpack_kernels::(&mut cu_file, thread_count)?; + let mut cu_buf = Vec::new(); + generate_cuda_unpack_kernels::(&mut cu_buf, thread_count)?; + write_if_changed(&cu_path, &cu_buf); - Ok(cu_path) + Ok(()) +} + +fn write_if_changed(path: &Path, content: &[u8]) { + if fs::read(path).is_ok_and(|existing| existing == content) { + return; + } + fs::write(path, content).unwrap_or_else(|e| panic!("Failed to write {}: {e}", path.display())); } fn nvcc_compile_ptx( @@ -223,14 +241,13 @@ fn generate_patches_bindings(kernels_src: &Path, out_dir: &Path) { .derive_copy(true) .derive_debug(true) .generate() - .expect("Failed to generate dynamic_dispatch bindings"); + .expect("Failed to generate patches bindings"); bindings .write_to_file(out_dir.join("patches.rs")) .expect("Failed to write patches.rs"); } -/// Check if CUDA is available based on nvcc. fn is_cuda_available() -> bool { Command::new("nvcc") .arg("--version") From 7c32db06f4116b091f9215deb0abf71594808ed9 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 17 Apr 2026 12:49:32 +0100 Subject: [PATCH 098/250] Use dev profile for editable python builds (#7510) ## Summary Has this ever happened to you? You just want to make some minor change, or run some small `uv` command, and suddenly you're re-building all of Vortex in release mode and your laptop sounds like a jet taking off. Now you don't have to do that! Any editable build (which is the default thing `uv` does) build will be in `dev` profile, while things like `maturin develop -r` or our publishing will stay in release mode. --------- Signed-off-by: Adam Gutglick --- .github/workflows/publish-dry-runs.yml | 4 ++-- uv.lock | 32 +++++++++++++------------- vortex-python/pyproject.toml | 6 +++-- vortex-python/test/conftest.py | 14 ----------- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 3e5b6d32868..5a2c9763cad 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -49,8 +49,8 @@ jobs: rm -rf ../target/wheels/ uv venv - uv tool run maturin@1.10 build --interpreter python3.11 --zig - uv tool run maturin@1.10 build --interpreter python3.11 --zig --sdist + uv tool run maturin@1.10.2 build --interpreter python3.11 --zig + uv tool run maturin@1.10.2 build --interpreter python3.11 --zig --sdist file_count=$(ls -1 ../target/wheels/ | wc -l) diff --git a/uv.lock b/uv.lock index a81fafa3462..b2e234fc1c3 100644 --- a/uv.lock +++ b/uv.lock @@ -597,23 +597,23 @@ wheels = [ [[package]] name = "maturin" -version = "1.9.6" +version = "1.13.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/35/c3370188492f4c139c7a318f438d01b8185c216303c49c4bc885c98b6afb/maturin-1.9.6.tar.gz", hash = "sha256:2c2ae37144811d365509889ed7220b0598487f1278c2441829c3abf56cc6324a", size = 214846, upload-time = "2025-10-07T12:45:08.408Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/16/b284a7bc4af3dd87717c784278c1b8cb18606ad1f6f7a671c47bfd9c3df0/maturin-1.13.1.tar.gz", hash = "sha256:9a87ff3b8e4d1c6eac33ebfe8e261e8236516d98d45c0323550621819b5a1a2f", size = 340369, upload-time = "2026-04-09T15:14:07.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/55/5c/b435418ba4ba2647a1f7a95d53314991b1e556e656ae276dea993c3bce1d/maturin-1.9.6-py3-none-linux_armv6l.whl", hash = "sha256:26e3ab1a42a7145824210e9d763f6958f2c46afb1245ddd0bab7d78b1f59bb3f", size = 8134483, upload-time = "2025-10-07T12:44:44.274Z" }, - { url = "https://files.pythonhosted.org/packages/4d/1c/8e58eda6601f328b412cdeeaa88a9b6a10e591e2a73f313e8c0154d68385/maturin-1.9.6-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5263dda3f71feef2e4122baf5c4620e4b3710dbb7f2121f85a337182de214369", size = 15776470, upload-time = "2025-10-07T12:44:47.476Z" }, - { url = "https://files.pythonhosted.org/packages/6c/33/8c967cce6848cdd87a2e442c86120ac644b80c5ed4c32e3291bde6a17df8/maturin-1.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe78262c2800c92f67d1ce3c0f6463f958a692cc67bfb572e5dbf5b4b696a8ba", size = 8226557, upload-time = "2025-10-07T12:44:49.844Z" }, - { url = "https://files.pythonhosted.org/packages/58/bd/3e2675cdc8b7270700ba30c663c852a35694441732a107ac30ebd6878bd8/maturin-1.9.6-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7ab827c6e8c022eb2e1e7fb6deede54549c8460b20ccc2e9268cc6e8cde957a8", size = 8166544, upload-time = "2025-10-07T12:44:51.396Z" }, - { url = "https://files.pythonhosted.org/packages/58/1f/a2047ddf2230e700d5f8a13dd4b9af5ce806ad380c32e58105888205926e/maturin-1.9.6-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:0246202377c49449315305209f45c8ecef6e2d6bd27a04b5b6f1ab3e4ea47238", size = 8641010, upload-time = "2025-10-07T12:44:53.658Z" }, - { url = "https://files.pythonhosted.org/packages/be/1f/265d63c7aa6faf363d4a3f23396f51bc6b4d5c7680a4190ae68dba25dea2/maturin-1.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:f5bac167700fbb6f8c8ed1a97b494522554b4432d7578e11403b894b6a91d99f", size = 7965945, upload-time = "2025-10-07T12:44:55.248Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ca/a8e61979ccfe080948bcc1bddd79356157aee687134df7fb013050cec783/maturin-1.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7f53d3b1d8396d3fea3e1ee5fd37558bca5719090f3d194ba1c02b0b56327ae3", size = 7978820, upload-time = "2025-10-07T12:44:56.919Z" }, - { url = "https://files.pythonhosted.org/packages/bf/4a/81b412f8ad02a99801ef19ec059fba0822d1d28fb44cb6a92e722f05f278/maturin-1.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:7f506eb358386d94d6ec3208c003130cf4b69cab26034fc0cbbf8bf83afa4c2e", size = 10452064, upload-time = "2025-10-07T12:44:58.232Z" }, - { url = "https://files.pythonhosted.org/packages/5b/12/cc96c7a8cb51d8dcc9badd886c361caa1526fba7fa69d1e7892e613b71d4/maturin-1.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2d6984ab690af509f525dbd2b130714207c06ebb14a5814edbe1e42b17ae0de", size = 8852401, upload-time = "2025-10-07T12:44:59.8Z" }, - { url = "https://files.pythonhosted.org/packages/51/8e/653ac3c9f2c25cdd81aefb0a2d17ff140ca5a14504f5e3c7f94dcfe4dbb7/maturin-1.9.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5c2252b0956bb331460ac750c805ddf0d9b44442449fc1f16e3b66941689d0bc", size = 8425057, upload-time = "2025-10-07T12:45:01.711Z" }, - { url = "https://files.pythonhosted.org/packages/db/29/f13490328764ae9bfc1da55afc5b707cebe4fa75ad7a1573bfa82cfae0c6/maturin-1.9.6-py3-none-win32.whl", hash = "sha256:f2c58d29ebdd4346fd004e6be213d071fdd94a77a16aa91474a21a4f9dbf6309", size = 7165956, upload-time = "2025-10-07T12:45:03.766Z" }, - { url = "https://files.pythonhosted.org/packages/db/9f/dd51e5ac1fce47581b8efa03d77a03f928c0ef85b6e48a61dfa37b6b85a2/maturin-1.9.6-py3-none-win_amd64.whl", hash = "sha256:1b39a5d82572c240d20d9e8be024d722dfb311d330c5e28ddeb615211755941a", size = 8145722, upload-time = "2025-10-07T12:45:05.487Z" }, - { url = "https://files.pythonhosted.org/packages/65/f2/e97aaba6d0d78c5871771bf9dd71d4eb8dac15df9109cf452748d2207412/maturin-1.9.6-py3-none-win_arm64.whl", hash = "sha256:ac02a30083553d2a781c10cd6f5480119bf6692fd177e743267406cad2ad198c", size = 6857006, upload-time = "2025-10-07T12:45:06.813Z" }, + { url = "https://files.pythonhosted.org/packages/43/4d/a23fc95be881aa8c7a6ea353410417872e4d7065df03d7f3db8f0dbed4a7/maturin-1.13.1-py3-none-linux_armv6l.whl", hash = "sha256:416e4e01cb88b798e606ee43929df897e42c1647b722ef68283816cca99a8742", size = 10102444, upload-time = "2026-04-09T15:13:48.393Z" }, + { url = "https://files.pythonhosted.org/packages/a6/1e/65c385d65bae95cf04895d52f39dbed8b1453ae55da2903d252ade40a774/maturin-1.13.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:72888e87819ce546d0d2df900e4b385e4ef299077d92ee37b48923a5602dae94", size = 19576043, upload-time = "2026-04-09T15:14:08.685Z" }, + { url = "https://files.pythonhosted.org/packages/8f/13/f6bc868d0bfecd9314870b97f530a167e31f7878ac4945c78245c6eef69c/maturin-1.13.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:98b5fcf1a186c217830a8295ecc2989c6b1cf50945417adfc15252107b9475b7", size = 10117339, upload-time = "2026-04-09T15:13:40.559Z" }, + { url = "https://files.pythonhosted.org/packages/51/58/279e081305c11c1c1c4fccacf77df8959646c5d4de7a57ec7e787653e270/maturin-1.13.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:3da18cccf2f683c0977bff9146a0908d6ffce836d600665736ac01679f588cb9", size = 10139689, upload-time = "2026-04-09T15:13:38.291Z" }, + { url = "https://files.pythonhosted.org/packages/00/94/69391af5396c6aab723932240803f49e5f3de3dd7c57d32f02d237a0ce32/maturin-1.13.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:6b1e5916a253243e8f5f9e847b62bbc98420eec48c9ce2e2e8724c6da89d359b", size = 10551141, upload-time = "2026-04-09T15:13:42.887Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bf/4edac2667b49e3733438062ae416413b8fc8d42e1bd499ba15e1fb02fc55/maturin-1.13.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:dc91031e0619c1e28730279ef9ee5f106c9b9ec806b013f888676b242f892eb7", size = 9983094, upload-time = "2026-04-09T15:13:56.868Z" }, + { url = "https://files.pythonhosted.org/packages/79/94/a6d651cfe8fc6bf2e892c90e3cdbb25c06d81c9115140d03ea1a68a97575/maturin-1.13.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:001741c6cff56aa8ea59a0d78ae990c0550d0e3e82b00b683eedb4158a8ef7e6", size = 9949980, upload-time = "2026-04-09T15:13:59.185Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d1/82c067464f848e38af9910bce55eb54302b1c1284a279d515dbfcf5994f5/maturin-1.13.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:01c845825c917c07c1d0b2c9032c59c16a7d383d1e649a46481d3e5693c2750f", size = 13186276, upload-time = "2026-04-09T15:13:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f4/25367baf1025580f047f9b37598bb3fadc416e24536afd4f28e190335c73/maturin-1.13.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f69093ed4a0e6464e52a7fc26d714f859ce15630ec8070743398c6bf41f38a9e", size = 10891837, upload-time = "2026-04-09T15:13:35.68Z" }, + { url = "https://files.pythonhosted.org/packages/af/be/caafad8ce74974b7deafdf144d12f758993dfea4c66c9905b138f51a7792/maturin-1.13.1-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:c1490584f3c70af45466ee99065b49e6657ebdccac6b10571bb44681309c9396", size = 10351032, upload-time = "2026-04-09T15:14:01.632Z" }, + { url = "https://files.pythonhosted.org/packages/66/0e/970a721d27cfa410e8bfa0a1e32e6ef52cb8169692110a5fdabe1af3f570/maturin-1.13.1-py3-none-win32.whl", hash = "sha256:c6a720b252c99de072922dbe4432ab19662b6f80045b0355fec23bdfccb450da", size = 8855465, upload-time = "2026-04-09T15:13:51.122Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/7c1e0d65fa147d5479055a171541c82b8cdfc1c825d85a82240470f14176/maturin-1.13.1-py3-none-win_amd64.whl", hash = "sha256:a2017d2281203d0c6570240e7d746564d766d756105823b7de68bda6ae722711", size = 10230471, upload-time = "2026-04-09T15:13:53.89Z" }, + { url = "https://files.pythonhosted.org/packages/c5/2a/afe0193b673a79ffd2e01ad999511b7e9e6b49af02bb3759d82a78c3043d/maturin-1.13.1-py3-none-win_arm64.whl", hash = "sha256:2839024dcd65776abb4759e5bca29941971e095574162a4d335191da4be9ff24", size = 8905575, upload-time = "2026-04-09T15:14:03.891Z" }, ] [[package]] @@ -1934,7 +1934,7 @@ provides-extras = ["duckdb", "numpy", "pandas", "polars", "ray"] [package.metadata.requires-dev] dev = [ { name = "duckdb", specifier = ">=1.1.2" }, - { name = "maturin", specifier = ">=1.7.2" }, + { name = "maturin", specifier = ">=1.10.2" }, { name = "numpy", specifier = ">=2.2.2" }, { name = "pandas", extras = ["output-formatting"], specifier = ">=2.2.3" }, { name = "pandas-stubs", specifier = ">=2.2.3.241126" }, diff --git a/vortex-python/pyproject.toml b/vortex-python/pyproject.toml index e90bae5a41b..fbb86f53e44 100644 --- a/vortex-python/pyproject.toml +++ b/vortex-python/pyproject.toml @@ -50,13 +50,15 @@ Issues = "https://github.com/vortex-data/vortex/issues" Benchmarks = "https://bench.vortex.dev" [build-system] -requires = ["maturin>=1.7.2,<2.0"] +requires = ["maturin>=1.10.2,<2.0"] build-backend = "maturin" [tool.uv] managed = true [tool.maturin] +profile = "release" +editable-profile = "dev" python-source = "python" module-name = "vortex._lib" features = ["pyo3/extension-module", "mimalloc"] @@ -70,7 +72,7 @@ include = [ [dependency-groups] dev = [ "duckdb>=1.1.2", - "maturin>=1.7.2", + "maturin>=1.10.2", "numpy>=2.2.2", "pandas-stubs>=2.2.3.241126", "pandas[output-formatting]>=2.2.3", diff --git a/vortex-python/test/conftest.py b/vortex-python/test/conftest.py index d5111a6c336..f2c1e6dd646 100644 --- a/vortex-python/test/conftest.py +++ b/vortex-python/test/conftest.py @@ -2,19 +2,5 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors import logging -import os -import pathlib -import subprocess logging.basicConfig(level=logging.DEBUG) - - -def pytest_sessionstart(): - """Pytest plugin to trigger maturin builds before running tests.""" - if os.environ.get("CI") is None: - # Running maturin develop --skip-install builds a "linux" wheel which PyPI rejects - # (https://peps.python.org/pep-0513/#rationale). When testing an already built wheel, we - # neither want to rebuild nor pollute the target/wheels directory with a wheel that PyPI - # will reject. - working_dir = pathlib.Path(__file__).parent.parent - _ = subprocess.check_call(["maturin", "develop", "--skip-install"], cwd=working_dir) From 550f351aec67031a6ca650286e986a8b3619d1c0 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Fri, 17 Apr 2026 13:17:53 +0100 Subject: [PATCH 099/250] Revert "chore: speed up CUDA kernel compilation (#7509)" (#7511) Needs to be reverted as CUDA changes are not properly picked up by build.rs Turns out there's no way to skip Rust build time and linkage, even if only the CUDA files changed that need to be re-generated. Signed-off-by: Alexander Droste --- vortex-cuda/build.rs | 89 ++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/vortex-cuda/build.rs b/vortex-cuda/build.rs index 90e0de1d881..b8673cdda83 100644 --- a/vortex-cuda/build.rs +++ b/vortex-cuda/build.rs @@ -3,11 +3,10 @@ #![expect(clippy::unwrap_used)] #![expect(clippy::expect_used)] -#![expect(clippy::panic)] #![expect(clippy::use_debug)] use std::env; -use std::fs; +use std::fs::File; use std::io; use std::path::Path; use std::path::PathBuf; @@ -31,10 +30,11 @@ fn main() { // Output directory for compiled .ptx files - separate by profile. let kernels_gen = Path::new(&manifest_dir).join("kernels/gen").join(&profile); - fs::create_dir_all(&kernels_gen).expect("Failed to create kernels/gen directory"); + std::fs::create_dir_all(&kernels_gen).expect("Failed to create kernels/gen directory"); - // Emit the kernels output directory path as a compile-time env var so any binary - // linking against vortex-cuda can find the PTX files. + // Always emit the kernels output directory path as a compile-time env var so any binary + // linking against vortex-cuda can find the PTX files. This must be set regardless + // of CUDA availability since the code using env!() is always compiled. // At runtime, VORTEX_CUDA_KERNELS_DIR can be set to override this path. println!( "cargo:rustc-env=VORTEX_CUDA_KERNELS_DIR={}", @@ -43,7 +43,7 @@ fn main() { println!("cargo:rerun-if-env-changed=PROFILE"); - // Regenerate bit_unpack kernels only when the generator changes. + // Regenerate bit_unpack kernels only when the generator changes println!( "cargo:rerun-if-changed={}", Path::new(&manifest_dir) @@ -63,73 +63,55 @@ fn main() { return; } - // Compile .cu files to PTX. We deliberately do NOT register .cu/.cuh files - // with rerun-if-changed so that editing a .cu file does not trigger Rust - // recompilation. - let mut cu_files = Vec::new(); - let mut newest_header = std::time::SystemTime::UNIX_EPOCH; - - if let Ok(entries) = fs::read_dir(&kernels_src) { + // Watch and compile .cu and .cuh files from kernels/src to PTX in kernels/gen + if let Ok(entries) = std::fs::read_dir(&kernels_src) { for path in entries.flatten().map(|entry| entry.path()) { + let is_generated = path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("bit_unpack_")); + match path.extension().and_then(|e| e.to_str()) { Some("cuh") | Some("h") => { - if let Ok(mtime) = fs::metadata(&path).and_then(|m| m.modified()) { - newest_header = newest_header.max(mtime); + // Only watch hand-written .cuh/.h files, not generated ones + // (generated files are rebuilt when cuda_kernel_generator changes) + if !is_generated { + println!("cargo:rerun-if-changed={}", path.display()); } } Some("cu") => { - cu_files.push(path); + // Only watch hand-written .cu files, not generated ones + // (generated files are rebuilt when cuda_kernel_generator changes) + if !is_generated { + println!("cargo:rerun-if-changed={}", path.display()); + } + // Compile all .cu files to PTX in gen directory + nvcc_compile_ptx(&kernels_src, &kernels_gen, &path, &profile) + .map_err(|e| { + format!("Failed to compile CUDA kernel {}: {}", path.display(), e) + }) + .unwrap(); } _ => {} } } } - - // Only compile .cu files whose PTX is stale (older than the source or any header). - for cu_path in &cu_files { - let ptx_path = kernels_gen - .join(cu_path.file_name().unwrap()) - .with_extension("ptx"); - - let cu_mtime = fs::metadata(cu_path) - .and_then(|m| m.modified()) - .unwrap_or(std::time::SystemTime::UNIX_EPOCH); - let newest_input = cu_mtime.max(newest_header); - - let ptx_mtime = fs::metadata(&ptx_path).and_then(|m| m.modified()).ok(); - if ptx_mtime.is_some_and(|t| t >= newest_input) { - continue; - } - - nvcc_compile_ptx(&kernels_src, &kernels_gen, cu_path, &profile) - .map_err(|e| format!("Failed to compile CUDA kernel {}: {}", cu_path.display(), e)) - .unwrap(); - } } -fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result<()> { +fn generate_unpack(output_dir: &Path, thread_count: usize) -> io::Result { // Generate the lanes header (.cuh) — device functions only, no __global__ kernels. // This is what dynamic_dispatch.cu includes (via bit_unpack.cuh). let cuh_path = output_dir.join(format!("bit_unpack_{}_lanes.cuh", T::T)); - let mut cuh_buf = Vec::new(); - generate_cuda_unpack_lanes::(&mut cuh_buf)?; - write_if_changed(&cuh_path, &cuh_buf); + let mut cuh_file = File::create(&cuh_path)?; + generate_cuda_unpack_lanes::(&mut cuh_file)?; // Generate the standalone kernels (.cu) — includes the lanes header, // adds _device template + __global__ wrappers. Compiled to its own PTX. let cu_path = output_dir.join(format!("bit_unpack_{}.cu", T::T)); - let mut cu_buf = Vec::new(); - generate_cuda_unpack_kernels::(&mut cu_buf, thread_count)?; - write_if_changed(&cu_path, &cu_buf); + let mut cu_file = File::create(&cu_path)?; + generate_cuda_unpack_kernels::(&mut cu_file, thread_count)?; - Ok(()) -} - -fn write_if_changed(path: &Path, content: &[u8]) { - if fs::read(path).is_ok_and(|existing| existing == content) { - return; - } - fs::write(path, content).unwrap_or_else(|e| panic!("Failed to write {}: {e}", path.display())); + Ok(cu_path) } fn nvcc_compile_ptx( @@ -241,13 +223,14 @@ fn generate_patches_bindings(kernels_src: &Path, out_dir: &Path) { .derive_copy(true) .derive_debug(true) .generate() - .expect("Failed to generate patches bindings"); + .expect("Failed to generate dynamic_dispatch bindings"); bindings .write_to_file(out_dir.join("patches.rs")) .expect("Failed to write patches.rs"); } +/// Check if CUDA is available based on nvcc. fn is_cuda_available() -> bool { Command::new("nvcc") .arg("--version") From ad087b6b7d8cab7d391831da2dd2050d3577bc16 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 17 Apr 2026 13:44:59 +0100 Subject: [PATCH 100/250] Improve docs for DataFusion integration (#7442) ## Summary Reworking docs for the DataFusion integration, mostly just more examples. Also moved some top-level tests into a dedicated module. --------- Signed-off-by: Adam Gutglick --- docs/user-guide/datafusion.md | 8 +- vortex-datafusion/src/lib.rs | 88 +++++- .../src/persistent/access_plan.rs | 41 ++- vortex-datafusion/src/persistent/format.rs | 140 ++++++++- vortex-datafusion/src/persistent/metrics.rs | 35 ++- vortex-datafusion/src/persistent/mod.rs | 281 ++---------------- vortex-datafusion/src/persistent/reader.rs | 32 +- vortex-datafusion/src/persistent/source.rs | 169 ++++++++++- vortex-datafusion/src/persistent/tests.rs | 257 ++++++++++++++++ vortex-datafusion/src/v2/mod.rs | 33 +- vortex-datafusion/src/v2/source.rs | 156 ++++++++-- vortex-datafusion/src/v2/table.rs | 55 +++- 12 files changed, 964 insertions(+), 331 deletions(-) create mode 100644 vortex-datafusion/src/persistent/tests.rs diff --git a/docs/user-guide/datafusion.md b/docs/user-guide/datafusion.md index 650d5e1c908..0be982575b6 100644 --- a/docs/user-guide/datafusion.md +++ b/docs/user-guide/datafusion.md @@ -14,7 +14,7 @@ vortex-datafusion = "" Register the Vortex format with a `SessionContext`: -:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs +:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs :language: rust :dedent: :start-after: [setup] @@ -27,7 +27,7 @@ Register the Vortex format with a `SessionContext`: Create an external table and query it: -:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs +:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs :language: rust :dedent: :start-after: [create] @@ -49,7 +49,7 @@ You can also register a `ListingTable` directly: Write query results to Vortex using `INSERT INTO`: -:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs +:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs :language: rust :dedent: :start-after: [write] @@ -63,7 +63,7 @@ partition value. Filters and projections are pushed down into the Vortex scan: -:::{literalinclude} ../../vortex-datafusion/src/persistent/mod.rs +:::{literalinclude} ../../vortex-datafusion/src/persistent/tests.rs :language: rust :dedent: :start-after: [query] diff --git a/vortex-datafusion/src/lib.rs b/vortex-datafusion/src/lib.rs index 75057365d11..e127e3e33ee 100644 --- a/vortex-datafusion/src/lib.rs +++ b/vortex-datafusion/src/lib.rs @@ -1,7 +1,87 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Connectors to enable [DataFusion](https://docs.rs/datafusion/latest/datafusion/) to read [`Vortex`](https://docs.rs/crate/vortex/latest) data. +//! Integrations between [`Vortex`] and [DataFusion]. +//! +//! The crate exposes two main entry points: +//! +//! - [`VortexFormatFactory`] for the file-based integration used by SQL, +//! `CREATE EXTERNAL TABLE`, and +//! [`ListingTable`]. +//! - [`v2`] for direct integration from an existing Vortex +//! [`DataSourceRef`]. +//! +//! # Registering The File Format +//! +//! Most applications register [`VortexFormatFactory`] with a DataFusion +//! [`SessionContext`] and then let DataFusion create [`VortexFormat`] and +//! [`VortexSource`] instances as queries are planned: +//! +//! ```no_run +//! use std::sync::Arc; +//! +//! use datafusion::datasource::provider::DefaultTableFactory; +//! use datafusion::execution::SessionStateBuilder; +//! use datafusion::prelude::SessionContext; +//! use datafusion_common::GetExt; +//! use vortex_datafusion::VortexFormatFactory; +//! +//! # #[tokio::main] +//! # async fn main() -> Result<(), Box> { +//! let factory = Arc::new(VortexFormatFactory::new()); +//! let mut state_builder = SessionStateBuilder::new() +//! .with_default_features() +//! .with_table_factory( +//! factory.get_ext().to_uppercase(), +//! Arc::new(DefaultTableFactory::new()), +//! ); +//! +//! if let Some(file_formats) = state_builder.file_formats() { +//! file_formats.push(factory.clone() as _); +//! } +//! +//! let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table(); +//! ctx.sql( +//! "CREATE EXTERNAL TABLE metrics (service VARCHAR, value BIGINT) \ +//! STORED AS vortex LOCATION 'file:///tmp/metrics/'", +//! ) +//! .await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! # Registering An Existing Vortex Data Source +//! +//! If your application already has a Vortex [`DataSourceRef`], use +//! [`v2::VortexTable`] to register it directly with DataFusion: +//! +//! ```no_run +//! use std::sync::Arc; +//! +//! use arrow_schema::Schema; +//! use datafusion::prelude::SessionContext; +//! use vortex::VortexSessionDefault; +//! use vortex::scan::DataSourceRef; +//! use vortex::session::VortexSession; +//! use vortex_datafusion::v2::VortexTable; +//! +//! # let data_source: DataSourceRef = todo!(); +//! let table = Arc::new(VortexTable::new( +//! data_source, +//! VortexSession::default(), +//! Arc::new(Schema::empty()), +//! )); +//! +//! let ctx = SessionContext::new(); +//! ctx.register_table("vortex_data", table)?; +//! # Ok::<(), datafusion_common::DataFusionError>(()) +//! ``` +//! +//! [`Vortex`]: https://docs.rs/crate/vortex/latest +//! [DataFusion]: https://docs.rs/datafusion/latest/datafusion/ +//! [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html +//! [`DataSourceRef`]: vortex::scan::DataSourceRef +//! [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html #![deny(missing_docs)] use std::fmt::Debug; @@ -18,7 +98,11 @@ mod tests; pub use convert::exprs::ExpressionConvertor; pub use persistent::*; -/// Extension trait to convert our [`Precision`](vortex::stats::Precision) to Datafusion's [`Precision`](datafusion_common::stats::Precision) +/// Extension trait to convert our [`Precision`] to DataFusion's +/// [`DataFusionPrecision`]. +/// +/// [`Precision`]: vortex::expr::stats::Precision +/// [`DataFusionPrecision`]: datafusion_common::stats::Precision trait PrecisionExt where T: Debug + Clone + PartialEq + Eq + PartialOrd, diff --git a/vortex-datafusion/src/persistent/access_plan.rs b/vortex-datafusion/src/persistent/access_plan.rs index 78f81d01b05..99583b7c0e4 100644 --- a/vortex-datafusion/src/persistent/access_plan.rs +++ b/vortex-datafusion/src/persistent/access_plan.rs @@ -4,18 +4,46 @@ use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::scan::selection::Selection; -/// Custom Vortex-specific information that can be provided by external indexes or other sources. +/// Additional Vortex-specific scan constraints attached to a +/// [`PartitionedFile`]. /// -/// This is intended as a low-level interface for users building their own data systems, see the [advance index] example from the DataFusion repo for a similar usage with Parquet. +/// `VortexAccessPlan` is the hook to use when an external index or planner +/// already knows that only part of a file needs to be scanned. The plan is +/// attached as `extensions` on `PartitionedFile`, and the internal +/// `VortexOpener` applies it before building the Vortex scan. /// -/// [advance index]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs +/// The current access plan surface is intentionally small: it lets callers +/// provide a [`Selection`] that narrows the rows considered by the scan. +/// +/// # Example +/// +/// ```no_run +/// # use std::sync::Arc; +/// # use datafusion_datasource::PartitionedFile; +/// # use vortex::scan::selection::Selection; +/// use vortex_datafusion::VortexAccessPlan; +/// +/// # let selection: Selection = todo!(); +/// let file = PartitionedFile::new("metrics.vortex", 1024).with_extensions(Arc::new( +/// VortexAccessPlan::default().with_selection(selection), +/// )); +/// # let _ = file; +/// ``` +/// +/// This is a low-level integration point for systems building their own access +/// paths on top of DataFusion. For a conceptually similar Parquet example, see +/// DataFusion's +/// [`parquet_advanced_index`]. +/// +/// [`PartitionedFile`]: datafusion_datasource::PartitionedFile +/// [`parquet_advanced_index`]: https://github.com/apache/datafusion/blob/47df535d2cd5aac5ad5a92bdc837f38e05ea0f0f/datafusion-examples/examples/data_io/parquet_advanced_index.rs #[derive(Default)] pub struct VortexAccessPlan { selection: Option, } impl VortexAccessPlan { - /// Sets a [`Selection`] for this plan. + /// Sets the row [`Selection`] to apply when the file is opened. pub fn with_selection(mut self, selection: Selection) -> Self { self.selection = Some(selection); self @@ -28,7 +56,10 @@ impl VortexAccessPlan { self.selection.as_ref() } - /// Apply the plan to the scan's builder. + /// Applies this access plan to a [`ScanBuilder`]. + /// + /// This is used internally by the file opener after it has translated a + /// `PartitionedFile` into a Vortex scan. pub fn apply_to_builder
(&self, mut scan_builder: ScanBuilder) -> ScanBuilder where A: 'static + Send, diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index 2515b66289e..d5c7fe913f1 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -70,7 +70,53 @@ use crate::convert::TryToDataFusion; const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES: usize = MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE; -/// Vortex implementation of a DataFusion [`FileFormat`]. +/// DataFusion [`FileFormat`] implementation for `.vortex` files. +/// +/// Most applications do not construct `VortexFormat` directly. Instead, they +/// register [`VortexFormatFactory`] with a [`SessionContext`] and let +/// DataFusion instantiate `VortexFormat` as tables are planned. +/// +/// Construct `VortexFormat` directly when you are wiring a [`ListingTable`] by +/// hand and need to pass a file format into [`ListingOptions`]. +/// +/// # Example +/// +/// ```no_run +/// use std::sync::Arc; +/// +/// use datafusion::datasource::listing::ListingOptions; +/// use datafusion::datasource::listing::ListingTable; +/// use datafusion::datasource::listing::ListingTableConfig; +/// use datafusion::datasource::listing::ListingTableUrl; +/// use datafusion::prelude::SessionContext; +/// use tempfile::tempdir; +/// use vortex::VortexSessionDefault; +/// use vortex::session::VortexSession; +/// use vortex_datafusion::VortexFormat; +/// +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let ctx = SessionContext::new(); +/// let dir = tempdir()?; +/// +/// let format = Arc::new(VortexFormat::new(VortexSession::default())); +/// let table_url = ListingTableUrl::parse(dir.path().to_str().unwrap())?; +/// let config = ListingTableConfig::new(table_url) +/// .with_listing_options( +/// ListingOptions::new(format).with_session_config_options(ctx.state().config()), +/// ) +/// .infer_schema(&ctx.state()) +/// .await?; +/// +/// let table = ListingTable::try_new(config)?; +/// # let _ = table; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html +/// [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html +/// [`ListingOptions`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingOptions.html pub struct VortexFormat { session: VortexSession, opts: VortexTableOptions, @@ -85,9 +131,24 @@ impl Debug for VortexFormat { } config_namespace! { - /// Options to configure the [`VortexFormat`]. + /// Options to configure [`VortexFormat`] and [`VortexSource`]. /// - /// Can be set through a DataFusion [`SessionConfig`]. + /// These options are usually set on a [`VortexFormatFactory`] and inherited + /// by the `VortexFormat` / `VortexSource` instances created for individual + /// tables. + /// + /// # Example + /// + /// ```rust + /// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions}; + /// + /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions { + /// projection_pushdown: true, + /// scan_concurrency: Some(8), + /// ..Default::default() + /// }); + /// # let _ = factory; + /// ``` /// /// [`SessionConfig`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionConfig.html pub struct VortexTableOptions { @@ -113,7 +174,44 @@ config_namespace! { impl Eq for VortexTableOptions {} -/// Minimal factory to create [`VortexFormat`] instances. +/// Registration entry point for the file-backed Vortex integration. +/// +/// `VortexFormatFactory` is the type most applications use. Register it with a +/// DataFusion session, and DataFusion will create [`VortexFormat`] values for +/// `CREATE EXTERNAL TABLE`, [`ListingTable`], and URL-table scans. +/// +/// The factory stores a [`VortexSession`] and default [`VortexTableOptions`]. +/// Those defaults are copied into the formats and sources created for each +/// table. +/// +/// # Example +/// +/// ```no_run +/// use std::sync::Arc; +/// +/// use datafusion::datasource::provider::DefaultTableFactory; +/// use datafusion::execution::SessionStateBuilder; +/// use datafusion_common::GetExt; +/// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions}; +/// +/// let factory = Arc::new(VortexFormatFactory::new().with_options(VortexTableOptions { +/// projection_pushdown: true, +/// ..Default::default() +/// })); +/// +/// let mut state_builder = SessionStateBuilder::new() +/// .with_default_features() +/// .with_table_factory( +/// factory.get_ext().to_uppercase(), +/// Arc::new(DefaultTableFactory::new()), +/// ); +/// +/// if let Some(file_formats) = state_builder.file_formats() { +/// file_formats.push(factory.clone() as _); +/// } +/// ``` +/// +/// [`ListingTable`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingTable.html #[derive(Debug)] pub struct VortexFormatFactory { session: VortexSession, @@ -127,7 +225,7 @@ impl GetExt for VortexFormatFactory { } impl VortexFormatFactory { - /// Creates a new instance with a default [`VortexSession`] and default options. + /// Creates a factory with a default [`VortexSession`] and default options. #[expect( clippy::new_without_default, reason = "FormatFactory defines `default` method, so having `Default` implementation is confusing" @@ -139,9 +237,11 @@ impl VortexFormatFactory { } } - /// Creates a new instance with customized session and default options for all [`VortexFormat`] instances created from this factory. + /// Creates a factory with an explicit session and default options. /// - /// The options can be overridden by table-level configuration pass in [`FileFormatFactory::create`]. + /// The supplied options become the baseline for every [`VortexFormat`] + /// created by this factory. DataFusion may still override them with + /// table-level options passed into [`FileFormatFactory::create`]. pub fn new_with_options(session: VortexSession, options: VortexTableOptions) -> Self { Self { session, @@ -149,13 +249,21 @@ impl VortexFormatFactory { } } - /// Override the default options for this factory. + /// Overrides the default options for this factory. + /// + /// This is the usual way to turn on features such as projection pushdown for + /// every table created through the factory. + /// + /// # Example /// - /// For example: /// ```rust /// use vortex_datafusion::{VortexFormatFactory, VortexTableOptions}; /// - /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions::default()); + /// let factory = VortexFormatFactory::new().with_options(VortexTableOptions { + /// projection_pushdown: true, + /// ..Default::default() + /// }); + /// # let _ = factory; /// ``` pub fn with_options(mut self, options: VortexTableOptions) -> Self { self.options = Some(options); @@ -195,17 +303,23 @@ impl FileFormatFactory for VortexFormatFactory { } impl VortexFormat { - /// Create a new instance with default options. + /// Creates a format with default [`VortexTableOptions`]. + /// + /// Prefer [`VortexFormatFactory`] when registering with a session. Construct + /// `VortexFormat` directly when building [`ListingOptions`] manually. + /// + /// [`ListingOptions`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingOptions.html pub fn new(session: VortexSession) -> Self { Self::new_with_options(session, VortexTableOptions::default()) } - /// Creates a new instance with configured by a [`VortexTableOptions`]. + /// Creates a format with explicit [`VortexTableOptions`]. pub fn new_with_options(session: VortexSession, opts: VortexTableOptions) -> Self { Self { session, opts } } - /// Return the format specific configuration + /// Returns the format-specific configuration that will be copied into the + /// [`VortexSource`] created for a scan. pub fn options(&self) -> &VortexTableOptions { &self.opts } diff --git a/vortex-datafusion/src/persistent/metrics.rs b/vortex-datafusion/src/persistent/metrics.rs index 7e99430568f..d4c6456d6fa 100644 --- a/vortex-datafusion/src/persistent/metrics.rs +++ b/vortex-datafusion/src/persistent/metrics.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Vortex table provider metrics. +//! Helpers for extracting Vortex scan metrics from DataFusion execution plans. use std::sync::Arc; use std::time::Duration; @@ -27,13 +27,40 @@ use crate::persistent::source::VortexSource; pub(crate) static PARTITION_LABEL: &str = "partition"; pub(crate) static PATH_LABEL: &str = "file_path"; -/// Extracts datafusion metrics from all VortexExec instances in -/// a given physical plan. +/// Walks a physical plan and returns one [`MetricsSet`] per +/// [`DataSourceExec`]. +/// +/// For Vortex-backed scans, the returned metrics include both the metrics +/// already attached to the `DataSourceExec` and the Vortex metrics accumulated +/// in [`VortexSource::metrics_registry`]. +/// +/// This helper exists because the Vortex read path records most scan metrics in +/// a Vortex [`MetricsRegistry`] rather than in DataFusion's native metrics set. +/// +/// # Example +/// +/// ```no_run +/// # use std::sync::Arc; +/// use datafusion_physical_plan::ExecutionPlan; +/// use vortex_datafusion::metrics::VortexMetricsFinder; +/// +/// # let plan: Arc = todo!(); +/// for metrics in VortexMetricsFinder::find_all(plan.as_ref()) { +/// for metric in metrics.aggregate_by_name().sorted_for_display().iter() { +/// println!("{metric}"); +/// } +/// } +/// ``` +/// +/// [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec +/// [`VortexSource::metrics_registry`]: crate::VortexSource::metrics_registry +/// [`MetricsRegistry`]: vortex::metrics::MetricsRegistry #[derive(Default)] pub struct VortexMetricsFinder(Vec); impl VortexMetricsFinder { - /// find all metrics for VortexExec nodes. + /// Collects metrics for each `DataSourceExec` in `plan`, augmenting any + /// Vortex-backed scan with the attached Vortex registry snapshot. pub fn find_all(plan: &dyn ExecutionPlan) -> Vec { let mut finder = Self::default(); match accept(plan, &mut finder) { diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs index c8560cfb7a8..17565937a0f 100644 --- a/vortex-datafusion/src/persistent/mod.rs +++ b/vortex-datafusion/src/persistent/mod.rs @@ -1,7 +1,27 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! Persistent implementation of a Vortex table provider. +//! File-backed DataFusion integration for `.vortex` files. +//! +//! Use this module when Vortex data lives in a filesystem or object store and +//! you want DataFusion to discover files, infer schema, and build +//! [`DataSourceExec`] plans for you. +//! +//! The main entry points are: +//! +//! - [`VortexFormatFactory`] to register the `vortex` file format with +//! DataFusion. +//! - [`VortexFormat`] when constructing +//! [`ListingOptions`] directly. +//! - [`VortexSource`] for lower-level `FileScanConfig` construction. +//! - [`VortexAccessPlan`] when external indexing or custom file selection needs +//! to restrict what a scan reads. +//! - [`metrics::VortexMetricsFinder`] to collect Vortex-specific scan metrics +//! from a planned query. +//! +//! [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec +//! [`ListingOptions`]: https://docs.rs/datafusion/latest/datafusion/datasource/listing/struct.ListingOptions.html + mod access_plan; mod cache; mod format; @@ -19,261 +39,4 @@ pub use format::VortexTableOptions; pub use source::VortexSource; #[cfg(test)] -mod tests { - - use std::sync::Arc; - - use datafusion::arrow::util::pretty::pretty_format_batches; - use datafusion_physical_plan::display::DisplayableExecutionPlan; - use insta::assert_snapshot; - use rstest::rstest; - use vortex::VortexSessionDefault; - use vortex::array::IntoArray; - use vortex::array::arrays::ChunkedArray; - use vortex::array::arrays::StructArray; - use vortex::array::arrays::VarBinArray; - use vortex::array::validity::Validity; - use vortex::buffer::buffer; - use vortex::file::WriteOptionsSessionExt; - use vortex::io::VortexWrite; - use vortex::io::object_store::ObjectStoreWrite; - use vortex::session::VortexSession; - - use crate::common_tests::TestSessionContext; - - #[rstest] - #[tokio::test] - async fn test_query_file(#[values(Some(1), None)] limit: Option) -> anyhow::Result<()> { - let ctx = TestSessionContext::default(); - - let session = VortexSession::default(); - - let strings = ChunkedArray::from_iter([ - VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), - VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), - ]) - .into_array(); - - let numbers = ChunkedArray::from_iter([ - buffer![1u32, 2, 3, 4].into_array(), - buffer![5u32, 6, 7, 8].into_array(), - ]) - .into_array(); - - let st = StructArray::try_new( - ["strings", "numbers"].into(), - vec![strings, numbers], - 8, - Validity::NonNullable, - )?; - - let mut writer = - ObjectStoreWrite::new(Arc::clone(&ctx.store), &"test.vortex".into()).await?; - - let summary = session - .write_options() - .write(&mut writer, st.into_array().to_array_stream()) - .await?; - - writer.shutdown().await?; - - assert_eq!(summary.row_count(), 8); - - let read_row_count = ctx - .session - .sql("SELECT * from '/test.vortex'") - .await? - .limit(0, limit)? - .count() - .await?; - - assert_eq!(read_row_count, limit.unwrap_or(8)); - - Ok(()) - } - - #[tokio::test] - async fn test_addition_pushdown() -> anyhow::Result<()> { - let ctx = TestSessionContext::default(); - dbg!(&ctx.store); - - ctx.session - .sql( - "CREATE EXTERNAL TABLE written_data \ - (a TINYINT NOT NULL) \ - STORED AS vortex \ - LOCATION '/test/'", - ) - .await?; - - ctx.session - .sql("INSERT INTO written_data VALUES (0), (1), (2), (3), (4)") - .await? - .collect() - .await?; - - let result = ctx - .session - .sql("SELECT a, a + 5 as five, a + 6 as six FROM written_data WHERE a + 5 > 7") - .await? - .collect() - .await?; - - assert_snapshot!(pretty_format_batches(&result)?, @r" - +---+------+-----+ - | a | five | six | - +---+------+-----+ - | 3 | 8 | 9 | - | 4 | 9 | 10 | - +---+------+-----+ - "); - - Ok(()) - } - - #[tokio::test] - async fn create_table_ordered_by() -> anyhow::Result<()> { - let ctx = TestSessionContext::default(); - - // Vortex - ctx.session - .sql( - "CREATE EXTERNAL TABLE my_tbl_vx \ - (c1 VARCHAR NOT NULL, c2 INT NOT NULL) \ - STORED AS vortex \ - WITH ORDER (c1 ASC) - LOCATION '/test/'", - ) - .await?; - - ctx.session - .sql("INSERT INTO my_tbl_vx VALUES ('air', 5), ('balloon', 42)") - .await? - .collect() - .await?; - - ctx.session - .sql("INSERT INTO my_tbl_vx VALUES ('zebra', 5)") - .await? - .collect() - .await?; - - ctx.session - .sql("INSERT INTO my_tbl_vx VALUES ('texas', 2000), ('alabama', 2000)") - .await? - .collect() - .await?; - - let df = ctx - .session - .sql("SELECT * FROM my_tbl_vx ORDER BY c1 ASC limit 3") - .await?; - - let physical_plan = ctx - .session - .state() - .create_physical_plan(df.logical_plan()) - .await?; - - insta::assert_snapshot!(DisplayableExecutionPlan::new(physical_plan.as_ref()) - .tree_render().to_string(), @r" - ┌───────────────────────────┐ - │ SortPreservingMergeExec │ - │ -------------------- │ - │ c1 ASC NULLS LAST │ - │ │ - │ limit: 3 │ - └─────────────┬─────────────┘ - ┌─────────────┴─────────────┐ - │ DataSourceExec │ - │ -------------------- │ - │ files: 3 │ - │ format: vortex │ - └───────────────────────────┘ - "); - - let r = df.collect().await?; - - insta::assert_snapshot!(pretty_format_batches(&r)?.to_string(), @r" - +---------+------+ - | c1 | c2 | - +---------+------+ - | air | 5 | - | alabama | 2000 | - | balloon | 42 | - +---------+------+ - "); - - Ok(()) - } - - /// Doc example: demonstrates creating, writing, reading, and filtering a Vortex table. - #[tokio::test] - async fn doc_example() -> anyhow::Result<()> { - // [setup] - use std::sync::Arc; - - use datafusion::datasource::provider::DefaultTableFactory; - use datafusion::execution::SessionStateBuilder; - use datafusion::prelude::SessionContext; - use datafusion_common::GetExt; - use object_store::memory::InMemory; - - use crate::VortexFormatFactory; - - let factory = Arc::new(VortexFormatFactory::new()); - let state = SessionStateBuilder::new() - .with_default_features() - .with_table_factory( - factory.get_ext().to_uppercase(), - Arc::new(DefaultTableFactory::new()), - ) - .with_file_formats(vec![factory]) - .build(); - let ctx = SessionContext::new_with_state(state).enable_url_table(); - // [setup] - - // Register an in-memory object store for the test. - let store = Arc::new(InMemory::new()); - ctx.register_object_store(&url::Url::try_from("file://").unwrap(), store); - - // [create] - ctx.sql( - "CREATE EXTERNAL TABLE my_table \ - (name VARCHAR NOT NULL, age INT NOT NULL) \ - STORED AS vortex \ - LOCATION '/demo/'", - ) - .await?; - // [create] - - // [write] - ctx.sql( - "INSERT INTO my_table VALUES \ - ('Alice', 30), ('Bob', 25), ('Charlie', 35), ('Diana', 28)", - ) - .await? - .collect() - .await?; - // [write] - - // [query] - let result = ctx - .sql("SELECT name, age FROM my_table WHERE age > 28 ORDER BY age") - .await? - .collect() - .await?; - // [query] - - assert_snapshot!(pretty_format_batches(&result)?, @r" - +---------+-----+ - | name | age | - +---------+-----+ - | Alice | 30 | - | Charlie | 35 | - +---------+-----+ - "); - - Ok(()) - } -} +mod tests; diff --git a/vortex-datafusion/src/persistent/reader.rs b/vortex-datafusion/src/persistent/reader.rs index 34ff646722c..de221c28ca7 100644 --- a/vortex-datafusion/src/persistent/reader.rs +++ b/vortex-datafusion/src/persistent/reader.rs @@ -16,7 +16,16 @@ use vortex::io::object_store::ObjectStoreReadAt; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; -/// Factory to create [`VortexReadAt`] instances to read the target file. +/// Factory to create [`VortexReadAt`] instances for a `PartitionedFile`. +/// +/// Plug a custom implementation into [`VortexSource::with_vortex_reader_factory`] +/// when the default object-store reader is not sufficient, for example to: +/// +/// - reuse an application-level metadata cache, +/// - wrap reads with custom authentication or routing, +/// - coalesce I/O in a remote storage layer. +/// +/// [`VortexSource::with_vortex_reader_factory`]: crate::VortexSource::with_vortex_reader_factory pub trait VortexReaderFactory: Debug + Send + Sync + 'static { /// Create a reader for a target object. fn create_reader( @@ -26,15 +35,30 @@ pub trait VortexReaderFactory: Debug + Send + Sync + 'static { ) -> DFResult>; } -/// Default factory, creates [`ObjectStore`] backed readers for files, -/// works with multiple cloud providers. +/// Default [`VortexReaderFactory`] backed by DataFusion's [`ObjectStore`]. +/// +/// This is the reader used by [`crate::VortexSource`] and +/// [`crate::VortexFormat`] unless a +/// custom factory is supplied. It works with any object store that DataFusion +/// has registered for the scan. #[derive(Debug)] pub struct DefaultVortexReaderFactory { object_store: Arc, } impl DefaultVortexReaderFactory { - /// Creates new instance + /// Creates a new factory from an [`ObjectStore`]. + /// + /// # Example + /// + /// ```rust + /// # use std::sync::Arc; + /// # use object_store::memory::InMemory; + /// use vortex_datafusion::reader::DefaultVortexReaderFactory; + /// + /// let factory = DefaultVortexReaderFactory::new(Arc::new(InMemory::new())); + /// # let _ = factory; + /// ``` pub fn new(object_store: Arc) -> Self { Self { object_store } } diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index 537e6162d0b..fdd6244eaf5 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -41,9 +41,133 @@ use crate::convert::exprs::ExpressionConvertor; use crate::persistent::reader::DefaultVortexReaderFactory; use crate::persistent::reader::VortexReaderFactory; -/// Execution plan for reading one or more Vortex files, intended to be consumed by [`DataSourceExec`]. +/// File scan implementation for reading one or more `.vortex` files. /// +/// `VortexSource` is the lower-level read component underneath +/// [`VortexFormat`]. It is the type DataFusion stores in a [`FileScanConfig`], +/// and it is ultimately executed through [`DataSourceExec`]. +/// +/// ```text +/// ▲ +/// │ +/// │ Produce a stream of +/// │ RecordBatches +/// │ +/// ┌───────────────────────┐ +/// │ DataSourceExec │ +/// └───────────────────────┘ +/// ▲ +/// │ uses +/// │ +/// ┌───────────────────────┐ +/// │ VortexSource │ +/// └───────────────────────┘ +/// ▲ +/// │ opens `.vortex` files via +/// │ +/// ObjectStore / VortexReadAt +/// ``` +/// +/// Most applications reach `VortexSource` indirectly through +/// [`VortexFormatFactory`]. Use `VortexSource` directly when you are +/// constructing a `FileScanConfig` yourself or when you need to inject +/// lower-level behavior such as a custom [`VortexReaderFactory`], an external +/// [`VortexAccessPlan`], or a specific [`FileMetadataCache`]. +/// +/// # Example +/// +/// ```rust +/// use std::sync::Arc; +/// +/// use arrow_schema::Schema; +/// use datafusion_datasource::file_scan_config::FileScanConfigBuilder; +/// use datafusion_datasource::source::DataSourceExec; +/// use datafusion_datasource::PartitionedFile; +/// use datafusion_datasource::TableSchema; +/// use datafusion_execution::object_store::ObjectStoreUrl; +/// use vortex::VortexSessionDefault; +/// use vortex::session::VortexSession; +/// use vortex_datafusion::VortexSource; +/// +/// let file_schema = Arc::new(Schema::empty()); +/// let source = Arc::new( +/// VortexSource::new( +/// TableSchema::from_file_schema(file_schema), +/// VortexSession::default(), +/// ) +/// .with_projection_pushdown(true) +/// .with_scan_concurrency(4), +/// ); +/// +/// let config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source) +/// .with_file(PartitionedFile::new("metrics.vortex", 1024)) +/// .build(); +/// +/// let exec = DataSourceExec::from_data_source(config); +/// # let _ = exec; +/// ``` +/// +/// # What `VortexSource` Handles +/// +/// `VortexSource` is responsible for: +/// +/// - translating DataFusion filters into Vortex predicates when possible, +/// - retaining the full predicate for file pruning based on statistics and +/// partition values, +/// - configuring per-file readers and sharing parsed layout readers across +/// partitions within the same scan, +/// - carrying the table schema used for schema evolution and missing-column +/// adaptation, +/// - attaching a Vortex metrics registry to the read path. +/// +/// # Projection And Predicate Behavior +/// +/// `VortexSource` keeps two related predicate forms: +/// +/// - `full_predicate`, which is used by DataFusion's `FilePruner` to skip whole +/// files before they are opened, +/// - `vortex_predicate`, which contains only the expressions Vortex can evaluate +/// during the scan. +/// +/// Projection handling depends on +/// [`VortexTableOptions::projection_pushdown`]: +/// +/// - when disabled, `VortexSource` still prunes unreferenced top-level columns, +/// but DataFusion applies the full projection after the scan, +/// - when enabled, the scan can evaluate a Vortex-native projection and leave +/// only unsupported expressions for DataFusion. +/// +/// # Observability +/// +/// `VortexSource` owns a Vortex metrics registry for the lifetime of a physical +/// scan. The registry is passed to the reader and scan builder so I/O and scan +/// metrics accumulate as the query executes. +/// +/// Use [`VortexMetricsFinder`] to merge those metrics back into DataFusion +/// `MetricsSet` values after the plan has run. +/// +/// # Execution Flow +/// +/// At execution time: +/// +/// 1. DataFusion calls `DataSourceExec`, which delegates file opening to +/// `VortexSource`. +/// 2. `VortexSource` creates a `VortexOpener` configured with the current +/// projection, predicate, options, and metrics. +/// 3. The opener adapts filters and schema for the specific file, applies any +/// [`VortexAccessPlan`], and builds a Vortex scan. +/// 4. Scan results are converted into Arrow `RecordBatch` values for +/// DataFusion. +/// +/// [`VortexFormat`]: crate::VortexFormat +/// [`FileScanConfig`]: datafusion_datasource::file_scan_config::FileScanConfig /// [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec +/// [`VortexFormatFactory`]: crate::VortexFormatFactory +/// [`VortexReaderFactory`]: crate::reader::VortexReaderFactory +/// [`VortexAccessPlan`]: crate::VortexAccessPlan +/// [`FileMetadataCache`]: datafusion_execution::cache::cache_manager::FileMetadataCache +/// [`VortexTableOptions::projection_pushdown`]: crate::VortexTableOptions::projection_pushdown +/// [`VortexMetricsFinder`]: crate::metrics::VortexMetricsFinder #[derive(Clone)] pub struct VortexSource { pub(crate) session: VortexSession, @@ -70,10 +194,14 @@ pub struct VortexSource { } impl VortexSource { - /// Creates a new VortexSource with default configuration and a provided [`VortexSession`]. - /// Meant to be use with a [`FileScanConfig`] to scan a file with the provided schema. + /// Creates a new `VortexSource` for a table schema and [`VortexSession`]. + /// + /// The new source starts with: /// - /// Can be configured using the provided methods. + /// - all top-level columns projected, + /// - no pushed filters, + /// - a default Vortex metrics registry, + /// - default [`VortexTableOptions`]. pub fn new(table_schema: TableSchema, session: VortexSession) -> Self { let full_schema = table_schema.table_schema(); let indices = (0..full_schema.fields().len()).collect::>(); @@ -96,13 +224,21 @@ impl VortexSource { } } - /// Enable or disable expression pushdown into the underlying Vortex scan. + /// Enables or disables Vortex-native projection evaluation. + /// + /// This toggles whether `VortexSource` tries to split DataFusion projection + /// expressions into a Vortex scan projection plus a leftover DataFusion + /// projection. pub fn with_projection_pushdown(mut self, enabled: bool) -> Self { self.options.projection_pushdown = enabled; self } - /// Set a [`ExpressionConvertor`] to control how Datafusion expression should be converted and pushed down. + /// Sets the [`ExpressionConvertor`] used to translate DataFusion expressions + /// into Vortex expressions. + /// + /// Override this when the default converter is insufficient for an engine + /// integration or for a custom schema-adaptation strategy. pub fn with_expression_convertor( mut self, expr_convertor: Arc, @@ -111,7 +247,10 @@ impl VortexSource { self } - /// Set a user-defined factory to create the underlying [`VortexReadAt`] + /// Sets a custom factory for the underlying [`VortexReadAt`]. + /// + /// Use this when reads need to go through an application-specific layer + /// rather than the default DataFusion [`ObjectStore`]. /// /// [`VortexReadAt`]: vortex::io::VortexReadAt pub fn with_vortex_reader_factory( @@ -122,12 +261,16 @@ impl VortexSource { self } - /// Returns the [`MetricsRegistry`] attached to this source. + /// Returns the [`MetricsRegistry`] attached to this scan. + /// + /// The registry is populated as files are opened and scanned. In most + /// callers, [`crate::metrics::VortexMetricsFinder`] is the more convenient + /// public API for turning the registry contents into DataFusion metrics. pub fn metrics_registry(&self) -> &Arc { &self.vx_metrics_registry } - /// Override the file metadata cache + /// Overrides the metadata cache used to reuse Vortex footers across scans. pub fn with_file_metadata_cache( mut self, file_metadata_cache: Arc, @@ -136,18 +279,20 @@ impl VortexSource { self } - /// Set the underlying scan concurrency. This limit is used per Vortex scan operations. + /// Sets the per-file Vortex scan concurrency. + /// + /// This is separate from DataFusion's partition-level parallelism. pub fn with_scan_concurrency(mut self, scan_concurrency: usize) -> Self { self.options.scan_concurrency = Some(scan_concurrency); self } - /// Returns the table options for this source. + /// Returns the effective table options for this source. pub fn options(&self) -> &VortexTableOptions { &self.options } - /// Set the table options for this source. + /// Replaces the table options for this source. pub fn with_options(mut self, opts: VortexTableOptions) -> Self { self.options = opts; self diff --git a/vortex-datafusion/src/persistent/tests.rs b/vortex-datafusion/src/persistent/tests.rs new file mode 100644 index 00000000000..09c56d4fbe4 --- /dev/null +++ b/vortex-datafusion/src/persistent/tests.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::sync::Arc; + +use datafusion::arrow::util::pretty::pretty_format_batches; +use datafusion_physical_plan::display::DisplayableExecutionPlan; +use insta::assert_snapshot; +use rstest::rstest; +use vortex::VortexSessionDefault; +use vortex::array::IntoArray; +use vortex::array::arrays::ChunkedArray; +use vortex::array::arrays::StructArray; +use vortex::array::arrays::VarBinArray; +use vortex::array::validity::Validity; +use vortex::buffer::buffer; +use vortex::file::WriteOptionsSessionExt; +use vortex::io::VortexWrite; +use vortex::io::object_store::ObjectStoreWrite; +use vortex::session::VortexSession; + +use crate::common_tests::TestSessionContext; + +#[rstest] +#[tokio::test] +async fn test_query_file(#[values(Some(1), None)] limit: Option) -> anyhow::Result<()> { + let ctx = TestSessionContext::default(); + + let session = VortexSession::default(); + + let strings = ChunkedArray::from_iter([ + VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), + VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), + ]) + .into_array(); + + let numbers = ChunkedArray::from_iter([ + buffer![1u32, 2, 3, 4].into_array(), + buffer![5u32, 6, 7, 8].into_array(), + ]) + .into_array(); + + let st = StructArray::try_new( + ["strings", "numbers"].into(), + vec![strings, numbers], + 8, + Validity::NonNullable, + )?; + + let mut writer = ObjectStoreWrite::new(Arc::clone(&ctx.store), &"test.vortex".into()).await?; + + let summary = session + .write_options() + .write(&mut writer, st.into_array().to_array_stream()) + .await?; + + writer.shutdown().await?; + + assert_eq!(summary.row_count(), 8); + + let read_row_count = ctx + .session + .sql("SELECT * from '/test.vortex'") + .await? + .limit(0, limit)? + .count() + .await?; + + assert_eq!(read_row_count, limit.unwrap_or(8)); + + Ok(()) +} + +#[tokio::test] +async fn test_addition_pushdown() -> anyhow::Result<()> { + let ctx = TestSessionContext::default(); + dbg!(&ctx.store); + + ctx.session + .sql( + "CREATE EXTERNAL TABLE written_data \ + (a TINYINT NOT NULL) \ + STORED AS vortex \ + LOCATION '/test/'", + ) + .await?; + + ctx.session + .sql("INSERT INTO written_data VALUES (0), (1), (2), (3), (4)") + .await? + .collect() + .await?; + + let result = ctx + .session + .sql("SELECT a, a + 5 as five, a + 6 as six FROM written_data WHERE a + 5 > 7") + .await? + .collect() + .await?; + + assert_snapshot!(pretty_format_batches(&result)?, @r" + +---+------+-----+ + | a | five | six | + +---+------+-----+ + | 3 | 8 | 9 | + | 4 | 9 | 10 | + +---+------+-----+ + "); + + Ok(()) +} + +#[tokio::test] +async fn create_table_ordered_by() -> anyhow::Result<()> { + let ctx = TestSessionContext::default(); + + // Vortex + ctx.session + .sql( + "CREATE EXTERNAL TABLE my_tbl_vx \ + (c1 VARCHAR NOT NULL, c2 INT NOT NULL) \ + STORED AS vortex \ + WITH ORDER (c1 ASC) + LOCATION '/test/'", + ) + .await?; + + ctx.session + .sql("INSERT INTO my_tbl_vx VALUES ('air', 5), ('balloon', 42)") + .await? + .collect() + .await?; + + ctx.session + .sql("INSERT INTO my_tbl_vx VALUES ('zebra', 5)") + .await? + .collect() + .await?; + + ctx.session + .sql("INSERT INTO my_tbl_vx VALUES ('texas', 2000), ('alabama', 2000)") + .await? + .collect() + .await?; + + let df = ctx + .session + .sql("SELECT * FROM my_tbl_vx ORDER BY c1 ASC limit 3") + .await?; + + let physical_plan = ctx + .session + .state() + .create_physical_plan(df.logical_plan()) + .await?; + + insta::assert_snapshot!(DisplayableExecutionPlan::new(physical_plan.as_ref()) + .tree_render().to_string(), @r" + ┌───────────────────────────┐ + │ SortPreservingMergeExec │ + │ -------------------- │ + │ c1 ASC NULLS LAST │ + │ │ + │ limit: 3 │ + └─────────────┬─────────────┘ + ┌─────────────┴─────────────┐ + │ DataSourceExec │ + │ -------------------- │ + │ files: 3 │ + │ format: vortex │ + └───────────────────────────┘ + "); + + let r = df.collect().await?; + + insta::assert_snapshot!(pretty_format_batches(&r)?.to_string(), @r" + +---------+------+ + | c1 | c2 | + +---------+------+ + | air | 5 | + | alabama | 2000 | + | balloon | 42 | + +---------+------+ + "); + + Ok(()) +} + +/// Doc example: demonstrates creating, writing, reading, and filtering a Vortex table. +#[tokio::test] +async fn doc_example() -> anyhow::Result<()> { + // [setup] + use std::sync::Arc; + + use datafusion::datasource::provider::DefaultTableFactory; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::SessionContext; + use datafusion_common::GetExt; + use object_store::memory::InMemory; + + use crate::VortexFormatFactory; + + let factory = Arc::new(VortexFormatFactory::new()); + let state = SessionStateBuilder::new() + .with_default_features() + .with_table_factory( + factory.get_ext().to_uppercase(), + Arc::new(DefaultTableFactory::new()), + ) + .with_file_formats(vec![factory]) + .build(); + let ctx = SessionContext::new_with_state(state).enable_url_table(); + // [setup] + + // Register an in-memory object store for the test. + let store = Arc::new(InMemory::new()); + ctx.register_object_store(&url::Url::try_from("file://").unwrap(), store); + + // [create] + ctx.sql( + "CREATE EXTERNAL TABLE my_table \ + (name VARCHAR NOT NULL, age INT NOT NULL) \ + STORED AS vortex \ + LOCATION '/demo/'", + ) + .await?; + // [create] + + // [write] + ctx.sql( + "INSERT INTO my_table VALUES \ + ('Alice', 30), ('Bob', 25), ('Charlie', 35), ('Diana', 28)", + ) + .await? + .collect() + .await?; + // [write] + + // [query] + let result = ctx + .sql("SELECT name, age FROM my_table WHERE age > 28 ORDER BY age") + .await? + .collect() + .await?; + // [query] + + assert_snapshot!(pretty_format_batches(&result)?, @r" + +---------+-----+ + | name | age | + +---------+-----+ + | Alice | 30 | + | Charlie | 35 | + +---------+-----+ + "); + + Ok(()) +} diff --git a/vortex-datafusion/src/v2/mod.rs b/vortex-datafusion/src/v2/mod.rs index 1e385ca3c85..5c2e6aa542f 100644 --- a/vortex-datafusion/src/v2/mod.rs +++ b/vortex-datafusion/src/v2/mod.rs @@ -1,10 +1,37 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! An experimental implementation of the Vortex Scan API for DataFusion. +//! Direct DataFusion integration for an existing Vortex +//! [`DataSourceRef`]. //! -//! This integration directly implements `TableProvider` + `ExecutionPlan`, bypassing DataFusion's -//! `FileFormat` abstraction. +//! Use this module when some other part of the system has already selected the +//! Vortex source to query and DataFusion only needs an adapter around it. +//! +//! Typical flow: +//! +//! 1. Build or obtain a Vortex [`DataSourceRef`]. +//! 2. Wrap it in [`VortexTable`] to register it with a [`SessionContext`], or +//! build a [`VortexDataSource`] directly when constructing a +//! [`DataSourceExec`]. +//! 3. Let DataFusion apply projection, filter, and limit pushdown through the +//! resulting adapter. +//! +//! The two main types are: +//! +//! - [`VortexTable`], the higher-level +//! [`TableProvider`] for `SessionContext::register_table`. +//! - [`VortexDataSource`], the lower-level +//! [`DataSource`] used when constructing physical plans directly. +//! +//! Compared with [`crate::VortexFormatFactory`], this module starts from an +//! already-constructed Vortex source instead of asking DataFusion to discover +//! `.vortex` files. +//! +//! [`DataSourceRef`]: vortex::scan::DataSourceRef +//! [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html +//! [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec +//! [`TableProvider`]: datafusion_catalog::TableProvider +//! [`DataSource`]: datafusion_datasource::source::DataSource mod source; mod table; diff --git a/vortex-datafusion/src/v2/source.rs b/vortex-datafusion/src/v2/source.rs index ee7f6d88f26..76a78845bab 100644 --- a/vortex-datafusion/src/v2/source.rs +++ b/vortex-datafusion/src/v2/source.rs @@ -1,10 +1,71 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! [`VortexDataSource`] implements DataFusion's [`DataSource`] trait, deferring scan construction -//! to [`DataSource::open`] so that pushed-down filters and limits are included in the -//! [`ScanRequest`]. A single DataFusion partition is used; Vortex handles internal parallelism -//! by driving splits concurrently via [`TryStreamExt::try_flatten_unordered`]. +//! Use [`VortexDataSource`] to adapt an existing Vortex [`DataSourceRef`] into +//! a DataFusion [`DataSource`] without going through file discovery. +//! +//! [`VortexDataSource`] is responsible for: +//! +//! - exposing an Arrow schema and output statistics to DataFusion, +//! - translating DataFusion projection, filter, and limit pushdown into a +//! Vortex [`ScanRequest`], +//! - executing the Vortex scan and converting the results into Arrow +//! `RecordBatch` values. +//! +//! # Example: Create a `DataSourceExec` +//! +//! ```no_run +//! use std::sync::Arc; +//! +//! use arrow_schema::Schema; +//! use datafusion_datasource::source::DataSourceExec; +//! use vortex::VortexSessionDefault; +//! use vortex::scan::DataSourceRef; +//! use vortex::session::VortexSession; +//! use vortex_datafusion::v2::VortexDataSource; +//! +//! # #[tokio::main] +//! # async fn main() -> Result<(), Box> { +//! # let data_source: DataSourceRef = todo!(); +//! let data_source = VortexDataSource::builder(data_source, VortexSession::default()) +//! .with_arrow_schema(Arc::new(Schema::empty())) +//! .build() +//! .await?; +//! +//! let exec = DataSourceExec::from_data_source(data_source); +//! # let _ = exec; +//! # Ok(()) +//! # } +//! ``` +//! +//! # Execution Flow +//! +//! ```text +//! ▲ +//! │ RecordBatch stream +//! │ +//! ┌───────────────────────┐ +//! │ DataSourceExec │ +//! └───────────────────────┘ +//! ▲ +//! │ DataFusion pushdown +//! │ (projection/filter/limit) +//! ┌───────────────────────┐ +//! │ VortexDataSource │ +//! └───────────────────────┘ +//! ▲ +//! │ final ScanRequest +//! ┌───────────────────────┐ +//! │ DataSourceRef │ +//! └───────────────────────┘ +//! ``` +//! +//! Compared with [`crate::VortexSource`], this path starts from an existing +//! Vortex source rather than from DataFusion-managed file discovery. +//! +//! [`DataSource`]: datafusion_datasource::source::DataSource +//! [`DataSourceRef`]: vortex::scan::DataSourceRef +//! [`ScanRequest`]: vortex::scan::ScanRequest use std::any::Any; use std::fmt; @@ -63,7 +124,45 @@ use crate::convert::exprs::ProcessedProjection; use crate::convert::exprs::make_vortex_predicate; use crate::convert::stats::stats_set_to_df; -/// A builder for a [`VortexDataSource`]. +/// Builder for [`VortexDataSource`]. +/// +/// Use the builder to declare how an existing Vortex +/// [`DataSourceRef`] should appear to DataFusion. +/// In particular, it lets you choose: +/// +/// - the Arrow schema DataFusion should see, +/// - an initial top-level projection if the embedding system already knows +/// which columns are needed. +/// +/// The resulting [`VortexDataSource`] is ready to plug into +/// [`DataSourceExec`] or other DataFusion physical planning code. +/// +/// # Example +/// +/// ```no_run +/// use std::sync::Arc; +/// +/// use arrow_schema::Schema; +/// use vortex::VortexSessionDefault; +/// use vortex::scan::DataSourceRef; +/// use vortex::session::VortexSession; +/// use vortex_datafusion::v2::VortexDataSource; +/// +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// # let data_source: DataSourceRef = todo!(); +/// let data_source = VortexDataSource::builder(data_source, VortexSession::default()) +/// .with_arrow_schema(Arc::new(Schema::empty())) +/// .with_projection(vec![0]) +/// .build() +/// .await?; +/// # let _ = data_source; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`DataSourceRef`]: vortex::scan::DataSourceRef +/// [`DataSourceExec`]: datafusion_datasource::source::DataSourceExec pub struct VortexDataSourceBuilder { data_source: DataSourceRef, session: VortexSession, @@ -73,8 +172,10 @@ pub struct VortexDataSourceBuilder { } impl VortexDataSourceBuilder { - /// Manually configure an Arrow schema to use when reading from the Vortex source. - /// If not specified, the data source will infer an Arrow schema from the Vortex DType. + /// Sets the Arrow schema exposed to DataFusion. + /// + /// If not specified, the builder derives an Arrow schema from the Vortex + /// dtype. /// /// Note that this schema is not validated against the Vortex DType so any errors will be /// deferred until read time. @@ -83,24 +184,26 @@ impl VortexDataSourceBuilder { self } - /// Configure an initial projection using top-level field indices. + /// Configures an initial top-level projection. + /// + /// This is useful when the embedding system already knows which columns are + /// needed before DataFusion applies its own optimizer pushdown. pub fn with_projection(mut self, indices: Vec) -> Self { self.projection = Some(indices); self } - /// Configure an initial projection using top-level field indices. + /// Like [`Self::with_projection`], but accepts an optional projection. pub fn with_some_projection(mut self, indices: Option>) -> Self { self.projection = indices; self } - /// Build the [`VortexDataSource`]. + /// Builds the [`VortexDataSource`]. /// - /// FIXME(ngates): Note that due to the DataFusion API, this function eagerly resolves - /// statistics for all projected columns. That said.. we only need to do this for aggregation - /// reductions. Any stats used for pruning are handled internally. We could possibly look - /// at the plan ourselves and decide whether there is any need for the stats? + /// The builder eagerly resolves statistics for the initial projection + /// columns because DataFusion expects the `DataSource` to report output + /// statistics before execution begins. pub async fn build(self) -> VortexResult { // The projection expression let mut projection = root(); @@ -193,12 +296,22 @@ impl VortexDataSource { } } -/// A DataFusion [`DataSource`] that defers Vortex scan construction to [`open`](DataSource::open). +/// DataFusion [`DataSource`] backed by a Vortex [`DataSourceRef`]. +/// +/// `VortexDataSource` is the core execution adapter for the `v2` integration. +/// It presents DataFusion with a scanable Arrow data source while preserving the +/// underlying Vortex source until execution time. /// -/// Holds a [`DataSourceRef`] rather than pre-collected splits, so that filters and limits pushed -/// down by DataFusion's optimizer are included in the [`ScanRequest`]. A single DataFusion -/// partition is exposed; Vortex drives splits concurrently via -/// [`TryStreamExt::try_flatten_unordered`]. +/// During planning, it reports the current output schema and column statistics. +/// During execution, it builds the final Vortex [`ScanRequest`] from the +/// current projection, pushed filters, ordering hints, and row limit. +/// +/// This integration intentionally reports a single DataFusion output partition. +/// Vortex then handles split-level concurrency internally by polling multiple +/// split streams concurrently. +/// +/// Use [`crate::VortexSource`] instead when DataFusion should discover and plan +/// `.vortex` files on its own. #[derive(Clone)] pub struct VortexDataSource { /// The Vortex data source. @@ -542,7 +655,10 @@ impl DataSource for VortexDataSource { } } -/// Convert a Vortex [`Option`] to a DataFusion [`Precision`](DFPrecision). +/// Convert a Vortex [`Option`] to a DataFusion +/// [`DataFusionPrecision`]. +/// +/// [`DataFusionPrecision`]: datafusion_common::stats::Precision fn estimate_to_df_precision(est: &Option>) -> DFPrecision { match est { Some(Precision::Exact(v)) => DFPrecision::Exact(usize::try_from(*v).unwrap_or(usize::MAX)), diff --git a/vortex-datafusion/src/v2/table.rs b/vortex-datafusion/src/v2/table.rs index 05edc0f27d8..5ca6829fdb3 100644 --- a/vortex-datafusion/src/v2/table.rs +++ b/vortex-datafusion/src/v2/table.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! [`VortexTable`] implements DataFusion's [`TableProvider`] trait, providing a direct -//! integration between a Vortex [`DataSource`] and DataFusion's query engine. +//! [`VortexTable`] adapts a Vortex [`DataSourceRef`] into a DataFusion +//! [`TableProvider`]. +//! +//! [`DataSourceRef`]: vortex::scan::DataSourceRef +//! [`TableProvider`]: datafusion_catalog::TableProvider use std::any::Any; use std::fmt; @@ -26,7 +29,49 @@ use vortex::session::VortexSession; use crate::v2::source::VortexDataSource; -/// A DataFusion [`TableProvider`] backed by a Vortex [`DataSourceRef`]. +/// DataFusion [`TableProvider`] backed by a Vortex +/// [`DataSourceRef`]. +/// +/// `VortexTable` is the usual entry point into [`crate::v2`] when you want to +/// register an existing Vortex source with DataFusion. +/// +/// Use it when another part of the system has already built a Vortex source and +/// you want to expose that source through a +/// [`SessionContext`]. +/// +/// `VortexTable` handles the `TableProvider` side of the integration: +/// +/// - it exposes the table schema and coarse statistics to DataFusion, +/// - it seeds the initial top-level projection during `scan`, +/// - it hands execution off to [`VortexDataSource`] for later pushdown and +/// execution. +/// +/// # Example +/// +/// ```no_run +/// use std::sync::Arc; +/// +/// use arrow_schema::Schema; +/// use datafusion::prelude::SessionContext; +/// use vortex::VortexSessionDefault; +/// use vortex::scan::DataSourceRef; +/// use vortex::session::VortexSession; +/// use vortex_datafusion::v2::VortexTable; +/// +/// # let data_source: DataSourceRef = todo!(); +/// let table = Arc::new(VortexTable::new( +/// data_source, +/// VortexSession::default(), +/// Arc::new(Schema::empty()), +/// )); +/// +/// let ctx = SessionContext::new(); +/// ctx.register_table("vortex_data", table)?; +/// # Ok::<(), datafusion_common::DataFusionError>(()) +/// ``` +/// +/// [`DataSourceRef`]: vortex::scan::DataSourceRef +/// [`SessionContext`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionContext.html pub struct VortexTable { data_source: DataSourceRef, session: VortexSession, @@ -44,8 +89,8 @@ impl fmt::Debug for VortexTable { impl VortexTable { /// Creates a new [`VortexTable`] from a Vortex data source and session. /// - /// The Arrow schema will be used to emit the correct column names and types to DataFusion. - /// The Vortex DType of the data source should be compatible with this Arrow schema. + /// The Arrow schema is the schema DataFusion will observe for this table. + /// It should be compatible with the Vortex dtype exposed by `data_source`. pub fn new( data_source: DataSourceRef, session: VortexSession, From bd9334090aa8b47124cbfb247f95feb5112a7b8e Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 17 Apr 2026 09:00:22 -0400 Subject: [PATCH 101/250] GPU kernel for sorted patches with chunk_offsets (#7440) --- Cargo.lock | 1 + vortex-cuda/Cargo.toml | 5 +- vortex-cuda/benches/transpose_patches.rs | 81 ---- vortex-cuda/kernels/src/bit_unpack_16.cu | 19 +- vortex-cuda/kernels/src/bit_unpack_32.cu | 19 +- vortex-cuda/kernels/src/bit_unpack_64.cu | 19 +- vortex-cuda/kernels/src/bit_unpack_8.cu | 19 +- vortex-cuda/kernels/src/patches.cuh | 90 ++++- vortex-cuda/kernels/src/patches.h | 22 +- vortex-cuda/src/bit_unpack_gen.rs | 19 +- vortex-cuda/src/kernel/encodings/bitpacked.rs | 180 ++++++++- vortex-cuda/src/kernel/mod.rs | 1 - vortex-cuda/src/kernel/patches/types.rs | 382 ++++-------------- vortex-cuda/src/lib.rs | 1 - 14 files changed, 409 insertions(+), 449 deletions(-) delete mode 100644 vortex-cuda/benches/transpose_patches.rs diff --git a/Cargo.lock b/Cargo.lock index 0dc36ecd49c..c7fc01c5ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10392,6 +10392,7 @@ dependencies = [ "futures", "itertools 0.14.0", "kanal", + "num-traits", "object_store 0.13.2", "parking_lot", "prost 0.14.3", diff --git a/vortex-cuda/Cargo.toml b/vortex-cuda/Cargo.toml index 9d21976f92b..edf91b00def 100644 --- a/vortex-cuda/Cargo.toml +++ b/vortex-cuda/Cargo.toml @@ -30,6 +30,7 @@ cudarc = { workspace = true, features = ["f16"] } futures = { workspace = true, features = ["executor"] } itertools = { workspace = true } kanal = { workspace = true } +num-traits = { workspace = true } object_store = { workspace = true, features = ["fs"] } parking_lot = { workspace = true } prost = { workspace = true } @@ -89,7 +90,3 @@ harness = false [[bench]] name = "throughput_cuda" harness = false - -[[bench]] -name = "transpose_patches" -harness = false diff --git a/vortex-cuda/benches/transpose_patches.rs b/vortex-cuda/benches/transpose_patches.rs deleted file mode 100644 index d713a304a07..00000000000 --- a/vortex-cuda/benches/transpose_patches.rs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#![expect(clippy::unwrap_used)] - -use std::time::Duration; - -use criterion::BenchmarkId; -use criterion::Criterion; -use criterion::Throughput; -use futures::executor::block_on; -use vortex::array::IntoArray; -use vortex::array::arrays::PrimitiveArray; -use vortex::array::dtype::PType; -use vortex::array::patches::Patches; -use vortex::buffer::Buffer; -use vortex::buffer::buffer; -use vortex::session::VortexSession; -use vortex_array::validity::Validity; -use vortex_cuda::CudaSession; -use vortex_cuda::transpose_patches; -use vortex_cuda_macros::cuda_available; -use vortex_cuda_macros::cuda_not_available; -use vortex_error::VortexExpect; - -fn benchmark_transpose(c: &mut Criterion) { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context"); - - let patches = block_on(async { - // Assume that we have 64k values, and we have 1024 patches evenly disbursed across - // the range. - let indices = (0..1024).map(|x| x * 64).collect::>(); - - let values = buffer![-1.0f32; 1024]; - - let device_indices = cuda_ctx.copy_to_device(indices)?.await?; - let device_values = cuda_ctx.copy_to_device(values)?.await?; - - Patches::new( - 64 * 1024, - 0, - PrimitiveArray::from_buffer_handle(device_indices, PType::U32, Validity::NonNullable) - .into_array(), - PrimitiveArray::from_buffer_handle(device_values, PType::F32, Validity::NonNullable) - .into_array(), - None, - ) - }) - .unwrap(); - - let mut group = c.benchmark_group("transpose"); - group.sample_size(100); - group.measurement_time(Duration::from_secs(10)); - - group.throughput(Throughput::Bytes( - patches.indices().nbytes() + patches.values().nbytes(), - )); - - group.bench_with_input( - BenchmarkId::new("transpose_patches", 0), - &patches, - |b, patches| { - b.iter(|| block_on(async { transpose_patches(patches, &mut cuda_ctx).await.unwrap() })) - }, - ); -} - -criterion::criterion_group! { - name = benches; - config = Criterion::default().without_plots() - .warm_up_time(Duration::from_nanos(1)) - .nresamples(10); - targets = benchmark_transpose -} - -#[cuda_available] -criterion::criterion_main!(benches); - -#[cuda_not_available] -fn main() {} diff --git a/vortex-cuda/kernels/src/bit_unpack_16.cu b/vortex-cuda/kernels/src/bit_unpack_16.cu index 63eb5f19bf9..3c05baf2011 100644 --- a/vortex-cuda/kernels/src/bit_unpack_16.cu +++ b/vortex-cuda/kernels/src/bit_unpack_16.cu @@ -5,21 +5,28 @@ template __device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint16_t shared_out[1024]; + + // Step 1: Unpack into shared memory #pragma unroll for (int i = 0; i < 2; i++) { _bit_unpack_16_lane(in, shared_out, reference, thread_idx * 2 + i); } __syncwarp(); + + // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); + while (patch.index != 1024) { + shared_out[patch.index] = patch.value; + patch = cursor.next(); + } + __syncwarp(); + + // Step 3: Copy to global memory + #pragma unroll for (int i = 0; i < 32; i++) { auto idx = i * 32 + thread_idx; - if (idx == patch.index) { - out[idx] = patch.value; - patch = cursor.next(); - } else { - out[idx] = shared_out[idx]; - } + out[idx] = shared_out[idx]; } } diff --git a/vortex-cuda/kernels/src/bit_unpack_32.cu b/vortex-cuda/kernels/src/bit_unpack_32.cu index 521183eee7c..97906f612c4 100644 --- a/vortex-cuda/kernels/src/bit_unpack_32.cu +++ b/vortex-cuda/kernels/src/bit_unpack_32.cu @@ -5,21 +5,28 @@ template __device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint32_t shared_out[1024]; + + // Step 1: Unpack into shared memory #pragma unroll for (int i = 0; i < 1; i++) { _bit_unpack_32_lane(in, shared_out, reference, thread_idx * 1 + i); } __syncwarp(); + + // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); + while (patch.index != 1024) { + shared_out[patch.index] = patch.value; + patch = cursor.next(); + } + __syncwarp(); + + // Step 3: Copy to global memory + #pragma unroll for (int i = 0; i < 32; i++) { auto idx = i * 32 + thread_idx; - if (idx == patch.index) { - out[idx] = patch.value; - patch = cursor.next(); - } else { - out[idx] = shared_out[idx]; - } + out[idx] = shared_out[idx]; } } diff --git a/vortex-cuda/kernels/src/bit_unpack_64.cu b/vortex-cuda/kernels/src/bit_unpack_64.cu index 9be1262f0f4..6270f4f8261 100644 --- a/vortex-cuda/kernels/src/bit_unpack_64.cu +++ b/vortex-cuda/kernels/src/bit_unpack_64.cu @@ -5,21 +5,28 @@ template __device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint64_t shared_out[1024]; + + // Step 1: Unpack into shared memory #pragma unroll for (int i = 0; i < 1; i++) { _bit_unpack_64_lane(in, shared_out, reference, thread_idx * 1 + i); } __syncwarp(); + + // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 16); auto patch = cursor.next(); + while (patch.index != 1024) { + shared_out[patch.index] = patch.value; + patch = cursor.next(); + } + __syncwarp(); + + // Step 3: Copy to global memory + #pragma unroll for (int i = 0; i < 64; i++) { auto idx = i * 16 + thread_idx; - if (idx == patch.index) { - out[idx] = patch.value; - patch = cursor.next(); - } else { - out[idx] = shared_out[idx]; - } + out[idx] = shared_out[idx]; } } diff --git a/vortex-cuda/kernels/src/bit_unpack_8.cu b/vortex-cuda/kernels/src/bit_unpack_8.cu index c933eda4df2..064970d650f 100644 --- a/vortex-cuda/kernels/src/bit_unpack_8.cu +++ b/vortex-cuda/kernels/src/bit_unpack_8.cu @@ -5,21 +5,28 @@ template __device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, int thread_idx, GPUPatches& patches) { __shared__ uint8_t shared_out[1024]; + + // Step 1: Unpack into shared memory #pragma unroll for (int i = 0; i < 4; i++) { _bit_unpack_8_lane(in, shared_out, reference, thread_idx * 4 + i); } __syncwarp(); + + // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); + while (patch.index != 1024) { + shared_out[patch.index] = patch.value; + patch = cursor.next(); + } + __syncwarp(); + + // Step 3: Copy to global memory + #pragma unroll for (int i = 0; i < 32; i++) { auto idx = i * 32 + thread_idx; - if (idx == patch.index) { - out[idx] = patch.value; - patch = cursor.next(); - } else { - out[idx] = shared_out[idx]; - } + out[idx] = shared_out[idx]; } } diff --git a/vortex-cuda/kernels/src/patches.cuh b/vortex-cuda/kernels/src/patches.cuh index 24b1705164f..66a97b935e9 100644 --- a/vortex-cuda/kernels/src/patches.cuh +++ b/vortex-cuda/kernels/src/patches.cuh @@ -5,6 +5,21 @@ #include "patches.h" +/// Load a chunk offset value, dispatching on the runtime type. +__device__ inline uint32_t load_chunk_offset(const GPUPatches &patches, uint32_t idx) { + switch (patches.chunk_offset_type) { + case CO_U8: + return reinterpret_cast(patches.chunk_offsets)[idx]; + case CO_U16: + return reinterpret_cast(patches.chunk_offsets)[idx]; + case CO_U32: + return reinterpret_cast(patches.chunk_offsets)[idx]; + case CO_U64: + return static_cast(reinterpret_cast(patches.chunk_offsets)[idx]); + } + return 0; +} + /// A single patch: a within-chunk index and its replacement value. /// A sentinel patch has index == 1024, which can never match a valid /// within-chunk position (0–1023). @@ -14,46 +29,78 @@ struct Patch { T value; }; -/// Cursor for iterating over a single lane's patches within a chunk. +/// Cursor for iterating over a thread's portion of patches within a chunk. /// -/// Usage in the generated merge-loop: +/// Patches are divided evenly among threads. Each thread applies its patches +/// to shared memory, then all threads sync and copy to global memory. +/// +/// Usage in the generated kernel: /// /// PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); /// auto patch = cursor.next(); -/// for (int i = 0; i < 32; i++) { -/// auto idx = i * 32 + thread_idx; -/// if (idx == patch.index) { -/// out[idx] = patch.value; -/// patch = cursor.next(); -/// } else { -/// out[idx] = shared_out[idx]; -/// } +/// while (patch.index != 1024) { +/// shared_out[patch.index] = patch.value; +/// patch = cursor.next(); /// } template class PatchesCursor { public: - /// Construct a cursor positioned at the patches for the given (chunk, lane). - /// n_lanes is a compile-time constant emitted by the code generator (16 or 32). - __device__ PatchesCursor(const GPUPatches &patches, uint32_t chunk, uint32_t lane, uint32_t n_lanes) { - if (patches.lane_offsets == nullptr) { + /// Construct a cursor for this thread's portion of patches in the chunk. + __device__ + PatchesCursor(const GPUPatches &patches, uint32_t chunk, uint32_t thread_idx, uint32_t n_threads) { + if (patches.chunk_offsets == nullptr) { indices = nullptr; values = nullptr; remaining = 0; return; } - auto slot = chunk * n_lanes + lane; - auto start = patches.lane_offsets[slot]; - remaining = patches.lane_offsets[slot + 1] - start; + + // mirrors the logic from vortex-array/src/arrays/primitive/array/patch.rs + + // Compute base_offset from the first chunk offset. + uint32_t base_offset = load_chunk_offset(patches, 0); + + uint32_t patches_start_idx = load_chunk_offset(patches, chunk) - base_offset; + patches_start_idx -= min(patches_start_idx, patches.offset_within_chunk); + + // calculate the ending index. + uint32_t patches_end_idx; + if ((chunk + 1) < patches.n_chunks) { + patches_end_idx = load_chunk_offset(patches, chunk + 1) - base_offset; + // if this is the end of times, we should drop it out here... + patches_end_idx -= min(patches_end_idx, patches.offset_within_chunk); + } else { + patches_end_idx = patches.num_patches; + } + + // calculate how many patches are in the chunk + uint32_t num_patches = patches_end_idx - patches_start_idx; + + // Divide patches among threads (ceil division) + uint32_t patches_per_thread = (num_patches + n_threads - 1) / n_threads; + uint32_t my_start = min(thread_idx * patches_per_thread, num_patches); + uint32_t my_end = min((thread_idx + 1) * patches_per_thread, num_patches); + + uint32_t start = patches_start_idx + my_start; + remaining = my_end - my_start; indices = patches.indices + start; values = reinterpret_cast(patches.values) + start; + + // The iterator returns indices relative to the start of the chunk. + // `chunk_base` is the index of the first element within a chunk, accounting + // for the slice offset. + chunk_base = chunk * 1024 + patches.offset; + chunk_base -= min(chunk_base, patches.offset % 1024); } - /// Return the current patch and advance, or a sentinel {1024, 0} if exhausted. + /// Return the current patch (with within-chunk index) and advance, + /// or a sentinel {1024, 0} if exhausted. __device__ Patch next() { if (remaining == 0) { return {1024, T {}}; } - Patch patch = {*indices, *values}; + uint16_t within_chunk = static_cast(*indices - chunk_base); + Patch patch = {within_chunk, *values}; indices++; values++; remaining--; @@ -61,7 +108,8 @@ public: } private: - const uint16_t *indices; + const uint32_t *indices; const T *values; uint8_t remaining; -}; \ No newline at end of file + uint32_t chunk_base; +}; diff --git a/vortex-cuda/kernels/src/patches.h b/vortex-cuda/kernels/src/patches.h index acc628ae595..ed7240821e6 100644 --- a/vortex-cuda/kernels/src/patches.h +++ b/vortex-cuda/kernels/src/patches.h @@ -9,18 +9,26 @@ extern "C" { #endif +/// Type tag for chunk_offsets pointer. +typedef enum { CO_U8 = 0, CO_U16 = 1, CO_U32 = 2, CO_U64 = 3 } ChunkOffsetType; + /// GPU-resident patches for fused exception patching during bit-unpacking. /// -/// Patches are stored in a lane-wise transposed layout: for each (chunk, lane) pair, -/// the corresponding patch indices and values are stored contiguously. The lane_offsets -/// array is a CSR-style offset array of size (n_chunks * n_lanes + 1) that maps each -/// (chunk, lane) slot to its range in the indices and values arrays. +/// Patches are stored in sorted order within each chunk. The chunk_offsets +/// array maps each chunk to the start of its range in the indices/values arrays. +/// The array has n_chunks elements (not n_chunks+1); the final offset is implicit +/// and equals num_patches. /// -/// A NULL lane_offsets pointer indicates no patches are present. +/// A NULL chunk_offsets pointer indicates no patches are present. typedef struct { - uint32_t *lane_offsets; - uint16_t *indices; + void *chunk_offsets; + ChunkOffsetType chunk_offset_type; + uint32_t *indices; void *values; + uint32_t offset; + uint32_t offset_within_chunk; + uint32_t num_patches; + uint32_t n_chunks; } GPUPatches; #ifdef __cplusplus diff --git a/vortex-cuda/src/bit_unpack_gen.rs b/vortex-cuda/src/bit_unpack_gen.rs index e114f319de8..8d5eda920cd 100644 --- a/vortex-cuda/src/bit_unpack_gen.rs +++ b/vortex-cuda/src/bit_unpack_gen.rs @@ -153,21 +153,28 @@ fn generate_device_kernel_template( r#"template __device__ void _bit_unpack_{bits}_device(const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, int thread_idx, GPUPatches& patches) {{ __shared__ uint{bits}_t shared_out[1024]; + + // Step 1: Unpack into shared memory #pragma unroll for (int i = 0; i < {per_thread_loop_count}; i++) {{ _bit_unpack_{bits}_lane(in, shared_out, reference, thread_idx * {per_thread_loop_count} + i); }} __syncwarp(); + + // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, {thread_count}); auto patch = cursor.next(); + while (patch.index != 1024) {{ + shared_out[patch.index] = patch.value; + patch = cursor.next(); + }} + __syncwarp(); + + // Step 3: Copy to global memory + #pragma unroll for (int i = 0; i < {shared_copy_ncount}; i++) {{ auto idx = i * {thread_count} + thread_idx; - if (idx == patch.index) {{ - out[idx] = patch.value; - patch = cursor.next(); - }} else {{ - out[idx] = shared_out[idx]; - }} + out[idx] = shared_out[idx]; }} }} "# diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 5fb576b5089..d29f6115889 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -21,6 +21,7 @@ use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedDataParts; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use vortex::error::vortex_err; @@ -28,8 +29,13 @@ use crate::CudaBufferExt; use crate::CudaDeviceBuffer; use crate::executor::CudaExecute; use crate::executor::CudaExecutionCtx; +use crate::kernel::patches::gpu::ChunkOffsetType; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U8; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U16; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U32; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U64; use crate::kernel::patches::gpu::GPUPatches; -use crate::kernel::patches::types::transpose_patches; +use crate::kernel::patches::types::load_patches; /// CUDA decoder for bit-packed arrays. #[derive(Debug)] @@ -86,6 +92,17 @@ pub fn bitpacked_cuda_launch_config(output_width: usize, len: usize) -> VortexRe unsafe impl DeviceRepr for GPUPatches {} +/// Convert a PType to the corresponding ChunkOffsetType for GPU patches. +fn ptype_to_chunk_offset_type(ptype: vortex::dtype::PType) -> VortexResult { + match ptype { + vortex::dtype::PType::U8 => Ok(ChunkOffsetType_CO_U8), + vortex::dtype::PType::U16 => Ok(ChunkOffsetType_CO_U16), + vortex::dtype::PType::U32 => Ok(ChunkOffsetType_CO_U32), + vortex::dtype::PType::U64 => Ok(ChunkOffsetType_CO_U64), + _ => vortex_bail!("Invalid PType for chunk_offsets: {:?}", ptype), + } +} + #[instrument(skip_all)] pub(crate) async fn decode_bitpacked( array: BitPackedArray, @@ -124,23 +141,34 @@ where // We hold this here to keep the device buffers alive. let device_patches = if let Some(patches) = patches { - Some(transpose_patches(&patches, ctx).await?) + Some(load_patches(&patches, ctx).await?) } else { None }; + #[expect(clippy::cast_possible_truncation)] let patches_arg = if let Some(p) = &device_patches { GPUPatches { - lane_offsets: p.lane_offsets.cuda_device_ptr()? as _, + chunk_offsets: p.chunk_offsets.cuda_device_ptr()? as _, + chunk_offset_type: ptype_to_chunk_offset_type(p.chunk_offset_ptype)?, indices: p.indices.cuda_device_ptr()? as _, values: p.values.cuda_device_ptr()? as _, + offset: p.offset as u32, + offset_within_chunk: p.offset_within_chunk as u32, + num_patches: p.num_patches as u32, + n_chunks: p.n_chunks as u32, } } else { - // NULL lane_offsets signals no patches to the kernel + // NULL chunk_offsets signals no patches to the kernel GPUPatches { - lane_offsets: std::ptr::null_mut(), + chunk_offsets: std::ptr::null_mut(), + chunk_offset_type: ChunkOffsetType_CO_U32, indices: std::ptr::null_mut(), values: std::ptr::null_mut(), + offset: 0, + offset_within_chunk: 0, + num_patches: 0, + n_chunks: 0, } }; @@ -175,6 +203,7 @@ mod tests { use vortex::array::dtype::NativePType; use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; + use vortex::buffer::buffer; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::error::VortexExpect; use vortex::session::VortexSession; @@ -535,4 +564,145 @@ mod tests { Ok(()) } + + /// Test slicing a bitpacked array with patches where the slice boundary + /// falls in the middle of a chunk's patch range, creating a non-zero + /// offset_within_chunk. + #[crate::test] + fn test_cuda_bitunpack_sliced_patches_offset_within_chunk() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Create an array with values that will generate patches. + // We use values 0-511 (fits in 9 bits) but include some larger values + // that will become patches. + let primitive_array = PrimitiveArray::new(buffer![100u8, 101, 102, 3, 4, 5], NonNullable); + + // Encode with bit width 4. First 3 elements patched, remainder will pack. + let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 4)?; + assert!( + bitpacked_array.patches().is_some(), + "Expected patches to be present" + ); + + let sliced_array = bitpacked_array.into_array().slice(2..6)?; + assert!(sliced_array.is::()); + + let cpu_result = sliced_array + .clone() + .execute::(cuda_ctx.execution_ctx())?; + let gpu_result = block_on(async { + BitPackedExecutor + .execute(sliced_array, &mut cuda_ctx) + .await + .vortex_expect("GPU decompression failed") + .into_host() + .await + .map(|a| a.into_array()) + })?; + + assert_arrays_eq!(cpu_result.into_array(), gpu_result); + + Ok(()) + } + + /// Test slicing a bitpacked array multiple times, accumulating offset_within_chunk. + #[crate::test] + fn test_cuda_bitunpack_double_sliced_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Create an array with values that will generate patches. + let mut values: Vec = Vec::with_capacity(3072); + for i in 0u16..3072 { + if i == 50 || i == 100 || i == 200 || i == 300 || i == 400 || i == 1100 || i == 2100 { + values.push(600); + } else { + values.push(i % 512); + } + } + + let primitive_array = + PrimitiveArray::new(Buffer::from_iter(values.iter().copied()), NonNullable); + + let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 9)?; + assert!( + bitpacked_array.patches().is_some(), + "Expected patches to be present" + ); + + // First slice: drop the patch at index 50 from the front of chunk 0. + let first_slice = bitpacked_array.into_array().slice(75..3000)?; + // Second slice (relative to first): drop patch at original index 100. + // The second slice's range is kept wide enough that num_blocks still + // covers every chunk in the packed buffer. + let second_slice = first_slice.slice(50..2900)?; + assert!(second_slice.is::()); + + let cpu_result = second_slice + .clone() + .execute::(cuda_ctx.execution_ctx())?; + let gpu_result = block_on(async { + BitPackedExecutor + .execute(second_slice, &mut cuda_ctx) + .await + .vortex_expect("GPU decompression failed") + .into_host() + .await + .map(|a| a.into_array()) + })?; + + assert_arrays_eq!(cpu_result.into_array(), gpu_result); + + Ok(()) + } + + /// Test slicing to skip an entire chunk's worth of patches. + #[crate::test] + fn test_cuda_bitunpack_sliced_skip_first_chunk_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Create patches in first chunk only, then slice past them all. + let mut values: Vec = Vec::with_capacity(3072); + for i in 0u16..3072 { + if i == 100 || i == 200 || i == 300 { + values.push(600); + } else if i == 1500 || i == 2500 { + values.push(700); + } else { + values.push(i % 512); + } + } + + let primitive_array = + PrimitiveArray::new(Buffer::from_iter(values.iter().copied()), NonNullable); + + let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 9)?; + assert!( + bitpacked_array.patches().is_some(), + "Expected patches to be present" + ); + + // Slice to skip past all first chunk patches + let sliced_array = bitpacked_array.into_array().slice(1024..3072)?; + assert!(sliced_array.is::()); + + let cpu_result = sliced_array + .clone() + .execute::(cuda_ctx.execution_ctx())?; + let gpu_result = block_on(async { + BitPackedExecutor + .execute(sliced_array, &mut cuda_ctx) + .await + .vortex_expect("GPU decompression failed") + .into_host() + .await + .map(|a| a.into_array()) + })?; + + assert_arrays_eq!(cpu_result.into_array(), gpu_result); + + Ok(()) + } } diff --git a/vortex-cuda/src/kernel/mod.rs b/vortex-cuda/src/kernel/mod.rs index 93ffd768df5..68f40a15005 100644 --- a/vortex-cuda/src/kernel/mod.rs +++ b/vortex-cuda/src/kernel/mod.rs @@ -34,7 +34,6 @@ pub use encodings::ZstdKernelPrep; pub use encodings::zstd_kernel_prepare; pub(crate) use encodings::*; pub(crate) use filter::FilterExecutor; -pub use patches::types::transpose_patches; pub(crate) use slice::SliceExecutor; use crate::CudaKernelEvents; diff --git a/vortex-cuda/src/kernel/patches/types.rs b/vortex-cuda/src/kernel/patches/types.rs index 380fefb2d94..350c5210e8a 100644 --- a/vortex-cuda/src/kernel/patches/types.rs +++ b/vortex-cuda/src/kernel/patches/types.rs @@ -1,354 +1,138 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! An implementation of lane-wise patches instead of linear patches. This layout for exception -//! patching enables fully parallel GPU execution, as outlined by Hepkema et al. in -//! "G-ALP: Rethinking Light-weight Encodings for GPUs" +//! GPU patches loading for fused exception patching during bit-unpacking. -use vortex::array::Canonical; +use num_traits::ToPrimitive; use vortex::array::buffer::BufferHandle; -use vortex::array::dtype::IntegerPType; -use vortex::array::dtype::NativePType; use vortex::buffer::Buffer; use vortex::buffer::BufferMut; -use vortex_array::match_each_native_ptype; +use vortex::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_error::VortexResult; +use vortex_error::vortex_bail; use crate::CudaExecutionCtx; +use crate::executor::CudaArrayExt; -/// A set of device-resident patches that live in the GPU. -/// -/// These are dynamically typed. -#[repr(C)] +/// A set of device-resident patches. pub struct DevicePatches { - pub(crate) lane_offsets: BufferHandle, + pub(crate) chunk_offsets: BufferHandle, + pub(crate) chunk_offset_ptype: PType, pub(crate) indices: BufferHandle, pub(crate) values: BufferHandle, + pub(crate) offset: usize, + pub(crate) offset_within_chunk: usize, + pub(crate) num_patches: usize, + pub(crate) n_chunks: usize, } -/// Number of lanes used at patch time for a value of type `V`. +/// Load patches for GPU use. /// -/// This is *NOT* equal to the number of FastLanes lanes for the type `V`, rather this will -/// correspond with the number of CUDA threads dedicated to executing each 1024-element vector. -const fn patch_lanes() -> usize { - // For types 32-bits or smaller, we use a 32 lane configuration, and for 64-bit we use 16 lanes. - // This matches up with the number of lanes we use to execute copying results from bit-unpacking - // from shared to global memory. - if size_of::() < 8 { 32 } else { 16 } -} - -/// A set of patches of values `V` existing in host buffers. -#[allow(dead_code)] -pub struct HostPatches { - n_chunks: usize, - n_lanes: usize, - lane_offsets: Buffer, - indices: Buffer, - /// Values. This is a buffer handle which might live on the new buffer type here - values: Buffer, -} - -#[cfg(test)] -struct LanePatches<'a, V> { - indices: &'a [u16], - values: &'a [V], -} - -impl HostPatches { - /// Get number of patches for a specific lane. - #[cfg(test)] - fn patch_count(&self, chunk: usize, lane: usize) -> usize { - let start = chunk * self.n_lanes + lane; - let end = start + 1; - let count = self.lane_offsets[end] - self.lane_offsets[start]; - - count as usize - } - - /// Get an ordered list of patches for the given chunk/lane. - #[cfg(test)] - fn patches(&self, chunk: usize, lane: usize) -> LanePatches<'_, V> { - let start = chunk * self.n_lanes + lane; - let end = start + 1; - - let lane_start = self.lane_offsets[start] as usize; - let lane_stop = self.lane_offsets[end] as usize; - - LanePatches { - indices: &self.indices[lane_start..lane_stop], - values: &self.values[lane_start..lane_stop], - } - } - - /// Apply the patches on top of the other buffer. - #[cfg(test)] - fn apply(&self, output: &mut BufferMut) { - for chunk in 0..self.n_chunks { - for lane in 0..self.n_lanes { - let patches = self.patches(chunk, lane); - for (&index, &value) in std::iter::zip(patches.indices, patches.values) { - let full_index = chunk * 1024 + (index as usize); - output[full_index] = value; - } - } - } - } - - /// Export the patches for use on the device associated with the provided execution context. - pub async fn export_to_device( - mut self, - ctx: &mut CudaExecutionCtx, - ) -> VortexResult { - let lane_offsets = std::mem::take(&mut self.lane_offsets); - let indices = std::mem::take(&mut self.indices); - let values = std::mem::take(&mut self.values); - - // Convert each into a handle that can be passed around. - let lane_offsets_handle = BufferHandle::new_host(lane_offsets.into_byte_buffer()); - let indices_handle = BufferHandle::new_host(indices.into_byte_buffer()); - let values_handle = BufferHandle::new_host(values.into_byte_buffer()); - - let lane_offsets_handle = ctx.ensure_on_device(lane_offsets_handle).await?; - let indices_handle = ctx.ensure_on_device(indices_handle).await?; - let values_handle = ctx.ensure_on_device(values_handle).await?; - - Ok(DevicePatches { - lane_offsets: lane_offsets_handle, - indices: indices_handle, - values: values_handle, - }) - } -} - -/// Transpose a set of patches from the default sorted layout into the data parallel layout. -#[expect(clippy::cognitive_complexity)] -pub async fn transpose_patches( +/// # Errors +/// +/// If the patches do not have `chunk_offsets`. They have been written by +/// default in new Vortex files since 0.54.0. +pub(crate) async fn load_patches( patches: &Patches, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let array_len = patches.array_len(); let offset = patches.offset(); + let offset_within_chunk = patches.offset_within_chunk().unwrap_or_default(); + let array_len = patches.array_len(); + + // Get or compute chunk_offsets + let Some(co) = patches.chunk_offsets() else { + vortex_bail!("cannot execute_cuda for patched BitPacked array without chunk_offsets") + }; + + let (chunk_offsets, chunk_offset_ptype) = { + let co_canonical = co.clone().execute_cuda(ctx).await?.into_primitive(); + let ptype = co_canonical.ptype(); + (co_canonical.buffer_handle().clone(), ptype) + }; + // Load indices - must be converted to u32 for GPU use let indices = patches .indices() .clone() - .execute::(ctx.execution_ctx())? + .execute_cuda(ctx) + .await? .into_primitive(); + let indices_ptype = indices.ptype(); + #[expect(clippy::expect_used)] + let indices = if indices_ptype == PType::U32 { + indices.buffer_handle().clone() + } else { + // Convert indices to u32 + let indices_buf = indices.buffer_handle().to_host().await; + let indices_u32 = match_each_unsigned_integer_ptype!(indices_ptype, |I| { + let src: Buffer = Buffer::from_byte_buffer(indices_buf); + let mut dst: BufferMut = BufferMut::with_capacity(src.len()); + for &idx in src.as_slice() { + // Indices are limited to u32 range for GPU + dst.push(idx.to_u32().expect("index should fit in u32")); + } + dst.freeze() + }); + BufferHandle::new_host(indices_u32.into_byte_buffer()) + }; + // Load values let values = patches .values() .clone() - .execute::(ctx.execution_ctx())? + .execute_cuda(ctx) + .await? .into_primitive(); - let indices_ptype = indices.ptype(); - let values_ptype = values.ptype(); - - let indices = indices.buffer_handle().to_host().await; - let values = values.buffer_handle().to_host().await; - - match_each_unsigned_integer_ptype!(indices_ptype, |I| { - match_each_native_ptype!(values_ptype, |V| { - let indices: Buffer = Buffer::from_byte_buffer(indices); - let values: Buffer = Buffer::from_byte_buffer(values); - - let host_patches = transpose(indices.as_slice(), values.as_slice(), offset, array_len); - - host_patches.export_to_device(ctx).await - }) - }) -} + // Ensure all on device + let chunk_offsets = ctx.ensure_on_device(chunk_offsets).await?; + let indices = ctx.ensure_on_device(indices).await?; + let values = ctx.ensure_on_device(values.buffer_handle().clone()).await?; -#[expect(clippy::cast_possible_truncation)] -fn transpose( - indices_in: &[I], - values_in: &[V], - offset: usize, - array_len: usize, -) -> HostPatches { - // Total number of slots is number of chunks times number of lanes. + let num_patches = patches.num_patches(); let n_chunks = array_len.div_ceil(1024); - assert!( - n_chunks <= u32::MAX as usize, - "Cannot transpose patches for array with >= 4 trillion elements" - ); - - let n_lanes = patch_lanes::(); - - // We know upfront how many indices and values we'll have. - let mut indices_buffer = BufferMut::with_capacity(indices_in.len()); - let mut values_buffer = BufferMut::with_capacity(values_in.len()); - - // number of patches in each chunk. - let mut lane_offsets: BufferMut = BufferMut::zeroed(n_chunks * n_lanes + 1); - // Scan the index/values once to get chunk/lane counts - for index in indices_in { - let index = index.as_() - offset; - let chunk = index / 1024; - let lane = index % n_lanes; - - lane_offsets[chunk * n_lanes + lane + 1] += 1; - } - - // Prefix-sum sizes -> offsets - for index in 1..lane_offsets.len() { - lane_offsets[index] += lane_offsets[index - 1]; - } - - // Loop over patches, writing them to final positions - let indices_out = indices_buffer.spare_capacity_mut(); - let values_out = values_buffer.spare_capacity_mut(); - for (index, &value) in std::iter::zip(indices_in, values_in) { - let index = index.as_() - offset; - let chunk = index / 1024; - let lane = index % n_lanes; - - let position = &mut lane_offsets[chunk * n_lanes + lane]; - indices_out[*position as usize].write((index % 1024) as u16); - values_out[*position as usize].write(value); - *position += 1; - } - - // SAFETY: we know there are exactly indices_in.len() indices/values, and we just - // set them to the appropriate values in the loop above. - unsafe { - indices_buffer.set_len(indices_in.len()); - values_buffer.set_len(values_in.len()); - } - - // Now, pass over all the indices and values again and subtract out the position increments. - for index in indices_in { - let index = index.as_() - offset; - let chunk = index / 1024; - let lane = index % n_lanes; - - lane_offsets[chunk * n_lanes + lane] -= 1; - } - - HostPatches { + Ok(DevicePatches { + chunk_offsets, + chunk_offset_ptype, + indices, + values, + offset, + offset_within_chunk, + num_patches, n_chunks, - n_lanes, - lane_offsets: lane_offsets.freeze(), - indices: indices_buffer.freeze(), - values: values_buffer.freeze(), - } + }) } #[cfg(test)] mod tests { - use vortex::array::ExecutionCtx; - use vortex::buffer::BufferMut; - use vortex::buffer::buffer; - use vortex::buffer::buffer_mut; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::arrays::PrimitiveArray; - use vortex_array::assert_arrays_eq; - use vortex_array::dtype::NativePType; use vortex_array::patches::Patches; use vortex_error::VortexResult; - use crate::kernel::patches::types::transpose; - - #[crate::test] - fn test_transpose_patches() { - let patch_values = buffer![0u32, 10, 20, 30, 40, 50, 60, 70, 80]; - - let mut patch_indices = BufferMut::empty(); - // CHUNK 0. patch_values have value type i32, which means there will be 32 lanes. - patch_indices.extend_from_slice(&[0, 31, 63, 64]); - - // CHUNK 1. - patch_indices.extend_from_slice(&[1024, 1056, 1058]); - - // CHUNK 2: empty - patch_indices.extend_from_slice(&[]); - - // CHUNK 3 - patch_indices.extend_from_slice(&[3073, 3076]); - - let patch_indices = patch_indices.freeze(); - - let transposed = transpose( - patch_indices.as_slice(), - patch_values.as_slice(), - 0, - 1024 * 5, - ); - - // Chunk 0 should have patches in lanes 0, 31 - assert_eq!(transposed.patches(0, 0).values, &[0, 30]); - assert_eq!(transposed.patches(0, 0).indices, &[0, 64]); - - assert_eq!(transposed.patches(0, 31).values, &[10, 20]); - assert_eq!(transposed.patches(0, 31).indices, &[31, 63]); - - // Chunk 1 should have patches in lanes 0, 2 - assert_eq!(transposed.patches(1, 0).values, &[40, 50]); - assert_eq!(transposed.patches(1, 0).indices, &[0, 32]); - assert_eq!(transposed.patches(1, 2).values, &[60]); - assert_eq!(transposed.patches(1, 2).indices, &[34]); - - // Chunk 2 should be empty - for lane in 0..transposed.n_lanes { - assert_eq!(transposed.patch_count(2, lane), 0); - } - - // Chunk 3 contains patches at lanes 1, 4 - assert_eq!(transposed.patches(3, 1).values, &[70]); - assert_eq!(transposed.patches(3, 1).indices, &[1]); - assert_eq!(transposed.patches(3, 4).values, &[80]); - assert_eq!(transposed.patches(3, 4).indices, &[4]); - } - #[test] - #[expect(clippy::cast_possible_truncation)] - fn test_transpose_complex() -> VortexResult<()> { - test_case(1024, 0, &[0], &[0f32])?; - test_case(512, 512, &[512, 513, 514], &[10i8, 20, 30])?; - test_case(10_000, 100, &[500, 1_000, 1_001, 1_002], &[1i16, 2, 3, 4])?; - - for len in (1..4096).step_by(10) { - let offset = len / 2; - - let indices: Vec = (offset..len).map(|x| x as u32).collect(); - - test_case(len, offset, &indices, &indices)?; - } - - Ok(()) - } - - fn test_case( - len: usize, - offset: usize, - patch_indices: &[u32], - patch_values: &[V], - ) -> VortexResult<()> { - let mut data = buffer_mut![V::default(); len]; - let array = PrimitiveArray::from_iter(data.iter().copied()); + fn test_patches_with_chunk_offsets() -> VortexResult<()> { + // Test creating patches with pre-computed chunk_offsets + let indices = PrimitiveArray::from_iter([0u32, 500, 1024, 2000]); + let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); + let chunk_offsets = PrimitiveArray::from_iter([0u32, 2, 3, 4]); let patches = Patches::new( - len, - offset, - PrimitiveArray::from_iter(patch_indices.iter().copied()).into_array(), - PrimitiveArray::from_iter(patch_values.iter().copied()).into_array(), - None, + 3072, + 0, + indices.into_array(), + values.into_array(), + Some(chunk_offsets.into_array()), )?; - // Verify that the outputs match between Patches and transpose_patches(). - let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let patched = array.patch(&patches, &mut ctx)?.into_array(); - - let transposed = transpose(patch_indices, patch_values, offset, len); - transposed.apply(&mut data); - - let patched_transposed = data.freeze().into_array(); - - assert_arrays_eq!(patched, patched_transposed); + assert!(patches.chunk_offsets().is_some()); + assert_eq!(patches.chunk_offset_at(0)?, 0); + assert_eq!(patches.chunk_offset_at(1)?, 2); + assert_eq!(patches.chunk_offset_at(2)?, 3); Ok(()) } diff --git a/vortex-cuda/src/lib.rs b/vortex-cuda/src/lib.rs index 7ce8fa09b84..7d9caf6aa99 100644 --- a/vortex-cuda/src/lib.rs +++ b/vortex-cuda/src/lib.rs @@ -47,7 +47,6 @@ use kernel::ZigZagExecutor; use kernel::ZstdBuffersExecutor; use kernel::ZstdExecutor; pub use kernel::ZstdKernelPrep; -pub use kernel::transpose_patches; pub use kernel::zstd_kernel_prepare; pub use pinned::PinnedByteBufferPool; pub use pinned::PinnedPoolStats; From 47f1158cc1faab14edcfaddf87db2e80c1608e29 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:57:19 -0400 Subject: [PATCH 102/250] Add `unstable_encodings` feature to `vortex-tui` (#7515) ## Summary Without this, we cannot view unstable encodings (like the tensor encodings). ## Testing N/A Signed-off-by: Connor Tsui --- vortex-tui/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/vortex-tui/Cargo.toml b/vortex-tui/Cargo.toml index e671ce935f5..33cd1958ba2 100644 --- a/vortex-tui/Cargo.toml +++ b/vortex-tui/Cargo.toml @@ -30,6 +30,7 @@ native = [ "vortex/tokio", "vortex/zstd", ] +unstable_encodings = ["vortex/unstable_encodings"] [lib] crate-type = ["cdylib", "rlib"] From 1ece6946ad4be356bb0b3b711bcc6ee1bd4d24a8 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:42:15 -0400 Subject: [PATCH 103/250] Hack to support forcing normalization (#7517) ## Summary We need to support normalization of vector arrays to perform faster compute. However, this will essentially always create larger arrays on disk. This is a hack specifically for `L2Denorm` that will allow us write normalized vector arrays to disk. Given that writing scalar fn arrays to disk is already gated by an environment variable and the unstable encodings feature flag, I think this hack is fine for now. ## Testing This just allows some arrays that are larger than canonical. Signed-off-by: Connor Tsui --- vortex-compressor/src/compressor.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index e23353f43df..26585a352a9 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -28,6 +28,7 @@ use vortex_array::arrays::list::ListArrayExt; use vortex_array::arrays::listview::ListViewArrayExt; use vortex_array::arrays::listview::list_from_list_view; use vortex_array::arrays::primitive::PrimitiveArrayExt; +use vortex_array::arrays::scalar_fn::AnyScalarFn; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -254,6 +255,11 @@ impl CascadingCompressor { return Ok(result); } + // TODO(connor): HACK TO SUPPORT L2 DENORMALIZATION!!! + if result.is::() { + return Ok(result); + } + // Otherwise, fall back to compressing the underlying storage array. let compressed_storage = self.compress(ext_array.storage_array())?; @@ -337,6 +343,11 @@ impl CascadingCompressor { // TODO(connor): Add a tracing warning here too. return Ok(compressed); } + + // TODO(connor): HACK TO SUPPORT L2 DENORMALIZATION!!! + if compressed.is::() { + return Ok(compressed); + } } // No scheme improved on the original. From 102de51ce977183a48649eb717e7d116c4621c1d Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 17 Apr 2026 17:17:41 +0100 Subject: [PATCH 104/250] Propagate min/max/string length statistics to duckdb (#7416) 1. Open all files eagerly in duckdb: this is needed to get file statistics 2. Save these statistics, if present, in duckdb table function bind data 3. Use this data to propagate min/max numeric and min/max/max_lendth string statistics Signed-off-by: Mikhail Kot --- vortex-array/src/stats/stats_set.rs | 27 ++- vortex-cuda/src/layout.rs | 5 + .../cpp/include/duckdb_vx/table_function.h | 19 +- vortex-duckdb/cpp/table_function.cpp | 112 ++++++++++- vortex-duckdb/src/datasource.rs | 178 +++++++++++++++--- .../src/duckdb/table_function/mod.rs | 19 +- .../src/duckdb/table_function/statistics.rs | 31 +++ vortex-duckdb/src/duckdb/value.rs | 1 + .../src/e2e_test/object_cache_test.rs | 9 + vortex-duckdb/src/multi_file.rs | 11 +- vortex-file/public-api.lock | 6 +- vortex-file/src/multi/mod.rs | 11 +- vortex-file/src/v2/file_stats_reader.rs | 8 + vortex-layout/public-api.lock | 14 ++ vortex-layout/src/layouts/chunked/reader.rs | 4 + vortex-layout/src/layouts/dict/reader.rs | 4 + vortex-layout/src/layouts/flat/reader.rs | 4 + vortex-layout/src/layouts/row_idx/mod.rs | 4 + vortex-layout/src/layouts/struct_/reader.rs | 4 + vortex-layout/src/layouts/zoned/reader.rs | 4 + vortex-layout/src/reader.rs | 3 + vortex-layout/src/scan/multi.rs | 6 +- vortex-layout/src/scan/scan_builder.rs | 12 ++ 23 files changed, 438 insertions(+), 58 deletions(-) create mode 100644 vortex-duckdb/src/duckdb/table_function/statistics.rs diff --git a/vortex-array/src/stats/stats_set.rs b/vortex-array/src/stats/stats_set.rs index 9496720d47e..a0ca186fed6 100644 --- a/vortex-array/src/stats/stats_set.rs +++ b/vortex-array/src/stats/stats_set.rs @@ -459,11 +459,28 @@ impl MutTypedStatsSetRef<'_, '_> { ) { (Some(m1), Some(m2)) => { // If the combine sum is exact, then we can sum them. - if let Some(scalar_value) = m1.zip(m2).as_exact().and_then(|(s1, s2)| { - s1.as_primitive() - .checked_add(&s2.as_primitive()) - .and_then(|pscalar| pscalar.pvalue().map(ScalarValue::Primitive)) - }) { + if let Some(scalar_value) = + m1.zip(m2).as_exact().and_then(|(s1, s2)| match s1.dtype() { + DType::Primitive(..) => s1 + .as_primitive() + .checked_add(&s2.as_primitive()) + .and_then(|pscalar| pscalar.pvalue().map(ScalarValue::Primitive)), + DType::Decimal(..) => s1 + .as_decimal() + .checked_binary_numeric( + &s2.as_decimal(), + crate::scalar::NumericOperator::Add, + ) + .map(|scalar| { + ScalarValue::Decimal( + scalar + .decimal_value() + .vortex_expect("no decimal value in scalar"), + ) + }), + _ => None, + }) + { self.set(Stat::Sum, Precision::Exact(scalar_value)); } } diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index ed69e235ad4..3bec31ba163 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -3,6 +3,7 @@ //! A CUDA-optimized flat layout that inlines small constant array buffers into layout metadata. +use std::any::Any; use std::collections::BTreeSet; use std::ops::BitAnd; use std::ops::Range; @@ -381,6 +382,10 @@ impl LayoutReader for CudaFlatReader { } .boxed()) } + + fn as_any(&self) -> &dyn Any { + self + } } /// A [`LayoutStrategy`] that writes a [`CudaFlatLayout`] with constant array buffers inlined diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h index e8f483514a5..b0e4532f7d0 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h @@ -97,6 +97,18 @@ typedef struct { bool has_max_cardinality; } duckdb_vx_node_statistics; +typedef struct { + // Set only for strings and primitive types + duckdb_value min; + duckdb_value max; + // upper bit: "length is set". lower 32 bits: DuckDB's max string length. + // set only for strings + uint64_t max_string_length; + bool has_null; +} duckdb_column_statistics; + +typedef idx_t column_t; + // A transparent DuckDB table function vtable, which can be used to configure a table function. // See duckdb/include/function/tfunc.hpp for details on each field. typedef struct { @@ -137,7 +149,12 @@ typedef struct { // void *in_out_function; // void *in_out_function_final; - void *statistics; + + // false if statistics are not available + bool (*statistics)(duckdb_client_context context, + const void *bind_data, + size_t column_index, + duckdb_column_statistics *stats_out); // void *dependency; void (*cardinality)(void *bind_data, duckdb_vx_node_statistics *node_stats_out); diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index 6154e91f738..45ab61a743e 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "duckdb_vx/table_function.h" #include "duckdb_vx/duckdb_diagnostics.h" DUCKDB_INCLUDES_BEGIN @@ -30,8 +31,10 @@ struct CTableFunctionInfo final : TableFunctionInfo { }; struct CTableBindData final : TableFunctionData { - CTableBindData(unique_ptr info_p, unique_ptr ffi_data_p) - : info(std::move(info_p)), ffi_data(std::move(ffi_data_p)) { + CTableBindData(unique_ptr info_p, + unique_ptr ffi_data_p, + const vector &types) + : info(std::move(info_p)), ffi_data(std::move(ffi_data_p)), types(types) { } unique_ptr Copy() const override { @@ -43,11 +46,13 @@ struct CTableBindData final : TableFunctionData { throw BinderException(IntoErrString(error_out)); } return make_uniq(make_uniq(info->vtab), - unique_ptr(reinterpret_cast(copied_ffi_data))); + unique_ptr(reinterpret_cast(copied_ffi_data)), + types); } unique_ptr info; unique_ptr ffi_data; + vector types; }; struct CTableGlobalData final : GlobalTableFunctionState { @@ -88,6 +93,103 @@ double c_table_scan_progress(ClientContext &context, return bind.info->vtab.table_scan_progress(c_ctx, c_bind_data, c_global_state); } +static Value &UnwrapValue(duckdb_value value) { + return *(reinterpret_cast(value)); +} + +unique_ptr numeric_stats(duckdb_column_statistics &stats, LogicalType type) { + BaseStatistics out = StringStats::CreateUnknown(type); + if (stats.min) { + NumericStats::SetMin(out, UnwrapValue(stats.min)); + duckdb_destroy_value(&stats.min); + } + if (stats.max) { + NumericStats::SetMax(out, UnwrapValue(stats.max)); + duckdb_destroy_value(&stats.max); + } + if (!stats.has_null) { + out.Set(StatsInfo::CANNOT_HAVE_NULL_VALUES); + } + return out.ToUnique(); +} + +unique_ptr string_stats(duckdb_column_statistics &stats, LogicalType type) { + BaseStatistics out = StringStats::CreateUnknown(type); + if (stats.min) { + StringStats::SetMin(out, StringValue::Get(UnwrapValue(stats.min))); + duckdb_destroy_value(&stats.min); + } + if (stats.max) { + StringStats::SetMax(out, StringValue::Get(UnwrapValue(stats.max))); + duckdb_destroy_value(&stats.max); + } + if (stats.max_string_length >> 63) { + StringStats::SetMaxStringLength(out, uint32_t(stats.max_string_length)); + } + if (!stats.has_null) { + out.Set(StatsInfo::CANNOT_HAVE_NULL_VALUES); + } + + return out.ToUnique(); +} + +unique_ptr base_stats(duckdb_column_statistics &stats, LogicalType type) { + BaseStatistics out = StringStats::CreateUnknown(type); + if (!stats.has_null) { + out.Set(StatsInfo::CANNOT_HAVE_NULL_VALUES); + } + return out.ToUnique(); +} + +unique_ptr +c_statistics(ClientContext &context, const FunctionData *bind_data, column_t column_index) { + if (IsVirtualColumn(column_index)) { + return {}; + } + + const auto &bind = bind_data->Cast(); + void *const ffi_bind = bind.ffi_data->DataPtr(); + + duckdb_client_context c_ctx = reinterpret_cast(&context); + duckdb_column_statistics statistics = {}; + if (!bind.info->vtab.statistics(c_ctx, ffi_bind, column_index, &statistics)) { + return {}; + } + + const LogicalType type = bind.types[column_index]; + + switch (type.id()) { + case LogicalTypeId::BOOLEAN: + case LogicalTypeId::TINYINT: + case LogicalTypeId::SMALLINT: + case LogicalTypeId::INTEGER: + case LogicalTypeId::BIGINT: + case LogicalTypeId::FLOAT: + case LogicalTypeId::DOUBLE: + case LogicalTypeId::UTINYINT: + case LogicalTypeId::USMALLINT: + case LogicalTypeId::UINTEGER: + case LogicalTypeId::UBIGINT: + case LogicalTypeId::UHUGEINT: + case LogicalTypeId::HUGEINT: { + return numeric_stats(statistics, type); + } + case LogicalTypeId::VARCHAR: + case LogicalTypeId::BLOB: { + return string_stats(statistics, type); + } + case LogicalTypeId::STRUCT: { + // TODO(myrrc) + // Duckdb's has_null has a different semantics for structs. + // If we propagate our has_null, this breaks Duckdb optimizer. + // You can reproduce it in struct.slt test in vortex-sqllogictests: + return {}; + } + default: + return base_stats(statistics, type); + } +} + unique_ptr c_bind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, @@ -111,7 +213,8 @@ unique_ptr c_bind(ClientContext &context, } return make_uniq(make_uniq(info.vtab), - unique_ptr(reinterpret_cast(ffi_bind_data))); + unique_ptr(reinterpret_cast(ffi_bind_data)), + return_types); } unique_ptr c_init_global(ClientContext &context, TableFunctionInitInput &input) { @@ -363,6 +466,7 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d tf.get_virtual_columns = c_get_virtual_columns; tf.to_string = c_to_string; tf.table_scan_progress = c_table_scan_progress; + tf.statistics = c_statistics; // Set up the parameters tf.arguments.reserve(vtab->parameter_count); diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index 47a1eba2b4f..e7a795621cb 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -26,6 +26,7 @@ use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; use vortex::array::arrays::scalar_fn::ScalarFnArrayExt; use vortex::array::optimizer::ArrayOptimizer; +use vortex::array::stats::StatsSet; use vortex::dtype::DType; use vortex::dtype::FieldNames; use vortex::error::VortexExpect; @@ -37,23 +38,31 @@ use vortex::expr::col; use vortex::expr::root; use vortex::expr::select; use vortex::expr::stats::Precision; +use vortex::expr::stats::Stat; +use vortex::file::v2::FileStatsLayoutReader; use vortex::io::kanal_ext::KanalExt; use vortex::io::runtime::BlockingRuntime; use vortex::io::runtime::current::ThreadSafeIterator; +use vortex::layout::scan::multi::MultiLayoutChild; +use vortex::layout::scan::multi::MultiLayoutDataSource; use vortex::metrics::tracing::get_global_labels; +use vortex::scalar::Scalar; +use vortex::scalar::ScalarValue; use vortex::scalar_fn::fns::pack::Pack; -use vortex::scan::DataSourceRef; +use vortex::scan::DataSource; use vortex::scan::ScanRequest; use vortex_utils::aliases::hash_set::HashSet; use crate::RUNTIME; use crate::SESSION; +use crate::convert::ToDuckDBScalar; use crate::convert::try_from_bound_expression; use crate::convert::try_from_table_filter; use crate::duckdb::BindInputRef; use crate::duckdb::BindResultRef; use crate::duckdb::Cardinality; use crate::duckdb::ClientContextRef; +use crate::duckdb::ColumnStatistics; use crate::duckdb::DataChunkRef; use crate::duckdb::ExpressionRef; use crate::duckdb::LogicalType; @@ -97,15 +106,21 @@ pub(crate) trait DataSourceTableFunction: Sized + Debug { } /// Bind the table function and return a [`DataSourceRef`]. - fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult; + fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult; +} + +#[derive(Debug, Clone)] +struct DuckdbField { + name: String, + logical_type: LogicalType, + dtype: DType, } /// Bind data produced by a [`DataSourceTableFunction`]. pub struct DataSourceBindData { - data_source: DataSourceRef, + data_source: Arc, filter_exprs: Vec, - column_names: Vec, - column_types: Vec, + column_fields: Vec, } impl Clone for DataSourceBindData { @@ -114,8 +129,7 @@ impl Clone for DataSourceBindData { data_source: Arc::clone(&self.data_source), // filter_exprs are consumed once in `init_global`. filter_exprs: vec![], - column_names: self.column_names.clone(), - column_types: self.column_types.clone(), + column_fields: self.column_fields.clone(), } } } @@ -123,8 +137,7 @@ impl Clone for DataSourceBindData { impl Debug for DataSourceBindData { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DataSourceBindData") - .field("column_names", &self.column_names) - .field("column_types", &self.column_types) + .field("column_fields", &self.column_fields) .field( "filter_exprs", &self @@ -163,6 +176,83 @@ fn progress(bytes_read: &AtomicU64, bytes_total: &AtomicU64) -> f64 { read as f64 / total as f64 * 100. } +impl ColumnStatistics { + fn from(stats: &ColumnStatisticsAggregate, dtype: DType) -> Self { + let min = stats.min.as_ref().map(|value| { + let value = value.clone(); + Scalar::try_new(dtype.clone(), Some(value)) + .vortex_expect("scalar dtype and value are incompatible") + .try_to_duckdb_scalar() + .vortex_expect("can't convert Scalar to duckdb Value") + }); + let max = stats.max.as_ref().map(|value| { + Scalar::try_new(dtype.clone(), Some(value.clone())) + .vortex_expect("scalar dtype and value are incompatible") + .try_to_duckdb_scalar() + .vortex_expect("can't convert Scalar to duckdb Value") + }); + + let max_string_length = stats + .max_string_length + .map_or(0, |len| (1u64 << 63) | (len as u64)); + + // Useful estimate if we didn't get null count stats + let has_null = stats.has_null && dtype.is_nullable(); + + Self { + min, + max, + max_string_length, + has_null, + } + } +} + +#[derive(Default)] +pub struct ColumnStatisticsAggregate { + pub min: Option, + pub max: Option, + pub max_string_length: Option, + /// May be true if null count stat isn't present + pub has_null: bool, +} + +impl ColumnStatisticsAggregate { + pub fn new(stats: &StatsSet) -> Self { + let min = match stats.get(Stat::Min) { + Some(Precision::Exact(min)) => Some(min), + _ => None, + }; + let max = match stats.get(Stat::Max) { + Some(Precision::Exact(max)) => Some(max), + _ => None, + }; + + let max_string_length = + if let Some(Precision::Exact(value)) = stats.get(Stat::UncompressedSizeInBytes) { + // DuckDB's string length is u32 + #[allow(clippy::cast_possible_truncation)] + Some(value.as_primitive().as_u64().vortex_expect("not a u64") as u32) + } else { + None + }; + + let has_null = match stats.get(Stat::NullCount) { + Some(Precision::Exact(cnt)) => { + cnt.as_primitive().as_u64().vortex_expect("not a u64") > 0 + } + _ => true, + }; + + Self { + min, + max, + max_string_length, + has_null, + } + } +} + // --------------------------------------------------------------------------- // Blanket TableFunction implementation for any DataSourceTableFunction // --------------------------------------------------------------------------- @@ -190,18 +280,14 @@ impl TableFunction for T { result: &mut BindResultRef, ) -> VortexResult { let data_source = T::bind(ctx, input)?; - - let (column_names, column_types) = extract_schema_from_dtype(data_source.dtype())?; - - for (column_name, column_type) in column_names.iter().zip(&column_types) { - result.add_result_column(column_name, column_type); + let column_fields = extract_schema_from_dtype(data_source.dtype())?; + for fields in &column_fields { + result.add_result_column(&fields.name, &fields.logical_type); } - Ok(DataSourceBindData { - data_source, + data_source: Arc::new(data_source), filter_exprs: vec![], - column_names, - column_types, + column_fields, }) } @@ -213,11 +299,11 @@ impl TableFunction for T { let projection_ids = init_input.projection_ids(); let projection_expr = - extract_projection_expr(projection_ids, column_ids, &bind_data.column_names); + extract_projection_expr(projection_ids, column_ids, &bind_data.column_fields); let filter_expr = extract_table_filter_expr( init_input.table_filter_set(), column_ids, - &bind_data.column_names, + &bind_data.column_fields, &bind_data.filter_exprs, bind_data.data_source.dtype(), )?; @@ -412,6 +498,33 @@ impl TableFunction for T { Ok(false) } + /// Get column-wise statistics. Available only if we're reading a single + /// file. + fn statistics( + _client_context: &ClientContextRef, + bind_data: &Self::BindData, + column_index: usize, + ) -> Option { + let children = bind_data.data_source.children(); + // Otherwise we'd have to open all files eagerly which is a performance + // regression. Duckdb's Parquet reader only gets metadata for multiple + // files with a UNION BY NAME and we don't support it (yet) + // https://github.com/duckdb/duckdb/blob/471de9f0e0e157ae672e56710e8c43b132a5ddc4/src/include/duckdb/common/multi_file/multi_file_function.hpp#L691 + if children.len() != 1 { + return None; + } + let MultiLayoutChild::Opened(ref reader) = children[0] else { + return None; + }; + let stats_sets = match reader.as_any().downcast_ref::() { + Some(inner) => inner.file_stats().stats_sets(), + None => return None, + }; + let stats_aggregate = ColumnStatisticsAggregate::new(&stats_sets[column_index]); + let dtype = bind_data.column_fields[column_index].dtype.clone(); + Some(ColumnStatistics::from(&stats_aggregate, dtype)) + } + fn cardinality(bind_data: &Self::BindData) -> Cardinality { match bind_data.data_source.row_count() { Some(Precision::Exact(v)) => Cardinality::Maximum(v), @@ -453,28 +566,30 @@ impl TableFunction for T { // --------------------------------------------------------------------------- /// Extracts DuckDB column names and logical types from a Vortex struct DType. -fn extract_schema_from_dtype(dtype: &DType) -> VortexResult<(Vec, Vec)> { +fn extract_schema_from_dtype(dtype: &DType) -> VortexResult> { let struct_dtype = dtype .as_struct_fields_opt() .ok_or_else(|| vortex_err!("Vortex file must contain a struct array at the top level"))?; - let mut column_names = Vec::new(); - let mut column_types = Vec::new(); + let len = struct_dtype.names().len(); + let mut fields = Vec::with_capacity(len); for (field_name, field_dtype) in struct_dtype.names().iter().zip(struct_dtype.fields()) { let logical_type = LogicalType::try_from(&field_dtype)?; - column_names.push(field_name.to_string()); - column_types.push(logical_type); + fields.push(DuckdbField { + name: field_name.to_string(), + logical_type, + dtype: field_dtype, + }); } - - Ok((column_names, column_types)) + Ok(fields) } /// Creates a projection expression from raw projection/column ID slices and column names. fn extract_projection_expr( projection_ids: Option<&[u64]>, column_ids: &[u64], - column_names: &[String], + column_fields: &[DuckdbField], ) -> Expression { // Projection ids may be empty, in which case you need to use projection_ids // https://github.com/duckdb/duckdb/blob/6e211da91657a94803c465fd0ce585f4c6754b54/src/planner/operator/logical_get.cpp#L168 @@ -494,9 +609,10 @@ fn extract_projection_expr( } #[expect(clippy::cast_possible_truncation)] - column_names + &column_fields .get(*idx as usize) .vortex_expect("prune idx in column names") + .name }) .map(|s| Arc::from(s.as_str())) .collect::(); @@ -509,7 +625,7 @@ fn extract_projection_expr( fn extract_table_filter_expr( table_filter_set: Option<&TableFilterSetRef>, column_ids: &[u64], - column_names: &[String], + column_fields: &[DuckdbField], additional_filters: &[Expression], dtype: &DType, ) -> VortexResult> { @@ -519,7 +635,7 @@ fn extract_table_filter_expr( .map(|(idx, ex)| { let idx_u: usize = idx.as_(); let col_idx: usize = column_ids[idx_u].as_(); - let name = column_names.get(col_idx).vortex_expect("exists"); + let name = &column_fields.get(col_idx).vortex_expect("exists").name; try_from_table_filter(ex, &col(name.as_str()), dtype) }) .collect::>>>()? diff --git a/vortex-duckdb/src/duckdb/table_function/mod.rs b/vortex-duckdb/src/duckdb/table_function/mod.rs index f20e844d381..36cd1464cc6 100644 --- a/vortex-duckdb/src/duckdb/table_function/mod.rs +++ b/vortex-duckdb/src/duckdb/table_function/mod.rs @@ -14,6 +14,7 @@ mod cardinality; mod init; mod partition; mod pushdown_complex_filter; +mod statistics; mod table_scan_progress; mod virtual_columns; @@ -28,16 +29,26 @@ use crate::duckdb::ClientContext; use crate::duckdb::DataChunk; use crate::duckdb::DatabaseRef; use crate::duckdb::LogicalType; +use crate::duckdb::Value; use crate::duckdb::client_context::ClientContextRef; use crate::duckdb::data_chunk::DataChunkRef; use crate::duckdb::expr::ExpressionRef; use crate::duckdb::table_function::cardinality::cardinality_callback; use crate::duckdb::table_function::partition::get_partition_data_callback; use crate::duckdb::table_function::pushdown_complex_filter::pushdown_complex_filter_callback; +use crate::duckdb::table_function::statistics::statistics; use crate::duckdb::table_function::table_scan_progress::table_scan_progress_callback; use crate::duckdb::table_function::virtual_columns::get_virtual_columns_callback; use crate::duckdb_try; +#[derive(Debug, Default)] +pub struct ColumnStatistics { + pub min: Option, + pub max: Option, + pub max_string_length: u64, + pub has_null: bool, +} + /// A trait that defines the supported operations for a table function in DuckDB. /// /// This trait does not yet cover the full C++ API, see table_function.hpp. @@ -84,6 +95,12 @@ pub trait TableFunction: Sized + Debug { result: &mut BindResultRef, ) -> VortexResult; + fn statistics( + client_context: &ClientContextRef, + bind_data: &Self::BindData, + column_index: usize, + ) -> Option; + /// The function is called during query execution and is responsible for producing the output fn scan( client_context: &ClientContextRef, @@ -188,7 +205,7 @@ impl DatabaseRef { init_global: Some(init_global_callback::), init_local: Some(init_local_callback::), function: Some(function::), - statistics: ptr::null_mut::(), + statistics: Some(statistics::), cardinality: Some(cardinality_callback::), pushdown_complex_filter: Some(pushdown_complex_filter_callback::), pushdown_expression: ptr::null_mut::(), diff --git a/vortex-duckdb/src/duckdb/table_function/statistics.rs b/vortex-duckdb/src/duckdb/table_function/statistics.rs new file mode 100644 index 00000000000..7f923dddefd --- /dev/null +++ b/vortex-duckdb/src/duckdb/table_function/statistics.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ffi::c_void; +use std::ptr; + +use vortex::error::VortexExpect; + +use crate::cpp; +use crate::duckdb::ClientContext; +use crate::duckdb::TableFunction; + +pub(crate) unsafe extern "C-unwind" fn statistics( + ctx: cpp::duckdb_client_context, + bind_data: *const c_void, + column_index: usize, + stats_out: *mut cpp::duckdb_column_statistics, +) -> bool { + let stats_out = unsafe { &mut *stats_out }; + let client_context = unsafe { ClientContext::borrow(ctx) }; + let bind_data = + unsafe { bind_data.cast::().as_ref() }.vortex_expect("bind_data null pointer"); + let Some(stats) = T::statistics(client_context, bind_data, column_index) else { + return false; + }; + stats_out.min = stats.min.map_or(ptr::null_mut(), |v| v.into_ptr()); + stats_out.max = stats.max.map_or(ptr::null_mut(), |v| v.into_ptr()); + stats_out.max_string_length = stats.max_string_length; + stats_out.has_null = stats.has_null; + true +} diff --git a/vortex-duckdb/src/duckdb/value.rs b/vortex-duckdb/src/duckdb/value.rs index d79c802a625..752f1ccca70 100644 --- a/vortex-duckdb/src/duckdb/value.rs +++ b/vortex-duckdb/src/duckdb/value.rs @@ -378,6 +378,7 @@ impl From<&[u8]> for Value { } /// An enum for extracting the underlying typed value from a `Value`. +#[derive(Debug)] pub enum ExtractedValue { Null, TinyInt(i8), diff --git a/vortex-duckdb/src/e2e_test/object_cache_test.rs b/vortex-duckdb/src/e2e_test/object_cache_test.rs index 712d2a41209..4aab6d131d7 100644 --- a/vortex-duckdb/src/e2e_test/object_cache_test.rs +++ b/vortex-duckdb/src/e2e_test/object_cache_test.rs @@ -12,6 +12,7 @@ use crate::cpp::DUCKDB_TYPE; use crate::duckdb::BindInputRef; use crate::duckdb::BindResultRef; use crate::duckdb::ClientContextRef; +use crate::duckdb::ColumnStatistics; use crate::duckdb::DataChunkRef; use crate::duckdb::LogicalType; use crate::duckdb::TableFunction; @@ -112,6 +113,14 @@ impl TableFunction for TestTableFunction { ) -> VortexResult { Ok(0) } + + fn statistics( + _client_context: &ClientContextRef, + _bind_data: &Self::BindData, + _column_index: usize, + ) -> Option { + None + } } use crate::duckdb::Database; diff --git a/vortex-duckdb/src/multi_file.rs b/vortex-duckdb/src/multi_file.rs index 14c8e02befb..ae89c5ef08c 100644 --- a/vortex-duckdb/src/multi_file.rs +++ b/vortex-duckdb/src/multi_file.rs @@ -13,7 +13,7 @@ use vortex::error::vortex_err; use vortex::file::multi::MultiFileDataSource; use vortex::io::filesystem::FileSystemRef; use vortex::io::runtime::BlockingRuntime; -use vortex::scan::DataSourceRef; +use vortex::layout::scan::multi::MultiLayoutDataSource; use vortex_utils::aliases::hash_map::HashMap; use crate::RUNTIME; @@ -76,7 +76,7 @@ impl DataSourceTableFunction for VortexMultiFileScan { vec![LogicalType::varchar()] } - fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { + fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { bind_multi_file_scan(ctx, input) } } @@ -89,7 +89,7 @@ impl DataSourceTableFunction for VortexMultiFileScanList { ] } - fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { + fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult { bind_multi_file_scan(ctx, input) } } @@ -98,7 +98,7 @@ impl DataSourceTableFunction for VortexMultiFileScanList { fn bind_multi_file_scan( ctx: &ClientContextRef, input: &BindInputRef, -) -> VortexResult { +) -> VortexResult { let glob_url_parameter = input .get_parameter(0) .ok_or_else(|| vortex_err!("Missing file glob parameter"))?; @@ -151,8 +151,7 @@ fn bind_multi_file_scan( builder = builder.with_glob(glob_url.path(), Some(fs)); } - let ds = builder.build().await?; - Ok(Arc::new(ds) as DataSourceRef) + builder.build().await }) } diff --git a/vortex-file/public-api.lock b/vortex-file/public-api.lock index f5be6aa8f23..5e26eeb1a1f 100644 --- a/vortex-file/public-api.lock +++ b/vortex-file/public-api.lock @@ -6,7 +6,7 @@ pub struct vortex_file::multi::MultiFileDataSource impl vortex_file::multi::MultiFileDataSource -pub async fn vortex_file::multi::MultiFileDataSource::build(self) -> vortex_error::VortexResult +pub async fn vortex_file::multi::MultiFileDataSource::build(self) -> vortex_error::VortexResult pub fn vortex_file::multi::MultiFileDataSource::new(session: vortex_session::VortexSession) -> Self @@ -68,6 +68,8 @@ pub struct vortex_file::v2::FileStatsLayoutReader impl vortex_file::v2::FileStatsLayoutReader +pub fn vortex_file::v2::FileStatsLayoutReader::file_stats(&self) -> &vortex_file::FileStatistics + pub fn vortex_file::v2::FileStatsLayoutReader::new(child: vortex_layout::reader::LayoutReaderRef, file_stats: vortex_file::FileStatistics, session: vortex_session::VortexSession) -> Self impl vortex_array::expr::pruning::StatsCatalog for vortex_file::v2::FileStatsLayoutReader @@ -76,6 +78,8 @@ pub fn vortex_file::v2::FileStatsLayoutReader::stats_ref(&self, field_path: &vor impl vortex_layout::reader::LayoutReader for vortex_file::v2::FileStatsLayoutReader +pub fn vortex_file::v2::FileStatsLayoutReader::as_any(&self) -> &dyn core::any::Any + pub fn vortex_file::v2::FileStatsLayoutReader::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_file::v2::FileStatsLayoutReader::filter_evaluation(&self, row_range: &core::ops::range::Range, expr: &vortex_array::expr::expression::Expression, mask: vortex_array::mask_future::MaskFuture) -> vortex_error::VortexResult diff --git a/vortex-file/src/multi/mod.rs b/vortex-file/src/multi/mod.rs index bf687f97ec1..5e7f5466921 100644 --- a/vortex-file/src/multi/mod.rs +++ b/vortex-file/src/multi/mod.rs @@ -98,7 +98,7 @@ impl MultiFileDataSource { /// /// Discovers files via glob, opens the first file eagerly to determine the schema, /// and creates lazy factories for the remaining files. - pub async fn build(self) -> VortexResult { + pub async fn build(self) -> VortexResult { if self.glob_sources.is_empty() { vortex_bail!("MultiFileDataSource requires at least one glob pattern"); } @@ -140,13 +140,8 @@ impl MultiFileDataSource { // Open first file eagerly for dtype. let (first_file_listing, first_fs) = &all_files[0]; - let first_file = open_file( - first_fs, - first_file_listing, - &self.session, - self.open_options_fn.as_ref(), - ) - .await?; + let open_fn = self.open_options_fn.as_ref(); + let first_file = open_file(first_fs, first_file_listing, &self.session, open_fn).await?; let first_reader = layout_reader_with_stats(&first_file)?; let factories: Vec> = all_files[1..] diff --git a/vortex-file/src/v2/file_stats_reader.rs b/vortex-file/src/v2/file_stats_reader.rs index 347670f050f..e5b504d1af4 100644 --- a/vortex-file/src/v2/file_stats_reader.rs +++ b/vortex-file/src/v2/file_stats_reader.rs @@ -106,6 +106,10 @@ impl FileStatsLayoutReader { Ok(result.as_bool().value() == Some(true)) } + + pub fn file_stats(&self) -> &FileStatistics { + &self.file_stats + } } /// Implements [`StatsCatalog`] to provide file-level stats to expressions during pruning evaluation. @@ -192,6 +196,10 @@ impl LayoutReader for FileStatsLayoutReader { ) -> VortexResult { self.child.projection_evaluation(row_range, expr, mask) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[cfg(test)] diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 4458a5a779c..22494a81f23 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -624,6 +624,8 @@ pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::new(row_offset: u64, impl vortex_layout::LayoutReader for vortex_layout::layouts::row_idx::RowIdxLayoutReader +pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::as_any(&self) -> &dyn core::any::Any + pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::filter_evaluation(&self, row_range: &core::ops::range::Range, expr: &vortex_array::expr::expression::Expression, mask: vortex_array::mask_future::MaskFuture) -> vortex_error::VortexResult @@ -1004,10 +1006,18 @@ pub fn vortex_layout::scan::layout::LayoutReaderDataSource::scan<'life0, 'async_ pub mod vortex_layout::scan::multi +pub enum vortex_layout::scan::multi::MultiLayoutChild + +pub vortex_layout::scan::multi::MultiLayoutChild::Deferred(alloc::sync::Arc) + +pub vortex_layout::scan::multi::MultiLayoutChild::Opened(vortex_layout::LayoutReaderRef) + pub struct vortex_layout::scan::multi::MultiLayoutDataSource impl vortex_layout::scan::multi::MultiLayoutDataSource +pub fn vortex_layout::scan::multi::MultiLayoutDataSource::children(&self) -> &alloc::vec::Vec + pub fn vortex_layout::scan::multi::MultiLayoutDataSource::new_deferred(dtype: vortex_array::dtype::DType, factories: alloc::vec::Vec>, session: &vortex_session::VortexSession) -> Self pub fn vortex_layout::scan::multi::MultiLayoutDataSource::new_with_first(first: vortex_layout::LayoutReaderRef, remaining: alloc::vec::Vec>, session: &vortex_session::VortexSession) -> Self @@ -1836,6 +1846,8 @@ pub fn vortex_layout::LayoutEncodingAdapter::id(&self) -> vortex_layout::Layo pub trait vortex_layout::LayoutReader: 'static + core::marker::Send + core::marker::Sync +pub fn vortex_layout::LayoutReader::as_any(&self) -> &dyn core::any::Any + pub fn vortex_layout::LayoutReader::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_layout::LayoutReader::filter_evaluation(&self, row_range: &core::ops::range::Range, expr: &vortex_array::expr::expression::Expression, mask: vortex_array::mask_future::MaskFuture) -> vortex_error::VortexResult @@ -1852,6 +1864,8 @@ pub fn vortex_layout::LayoutReader::row_count(&self) -> u64 impl vortex_layout::LayoutReader for vortex_layout::layouts::row_idx::RowIdxLayoutReader +pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::as_any(&self) -> &dyn core::any::Any + pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_layout::layouts::row_idx::RowIdxLayoutReader::filter_evaluation(&self, row_range: &core::ops::range::Range, expr: &vortex_array::expr::expression::Expression, mask: vortex_array::mask_future::MaskFuture) -> vortex_error::VortexResult diff --git a/vortex-layout/src/layouts/chunked/reader.rs b/vortex-layout/src/layouts/chunked/reader.rs index 756e5573704..376b6654707 100644 --- a/vortex-layout/src/layouts/chunked/reader.rs +++ b/vortex-layout/src/layouts/chunked/reader.rs @@ -324,6 +324,10 @@ impl LayoutReader for ChunkedReader { } .boxed()) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[cfg(test)] diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 9807f97b5ad..446df56611d 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -251,6 +251,10 @@ impl LayoutReader for DictReader { } .boxed()) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[cfg(test)] diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 29627e2b4fd..66f9553c0de 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -216,6 +216,10 @@ impl LayoutReader for FlatReader { } .boxed()) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[cfg(test)] diff --git a/vortex-layout/src/layouts/row_idx/mod.rs b/vortex-layout/src/layouts/row_idx/mod.rs index f435d6bd992..05218e0994d 100644 --- a/vortex-layout/src/layouts/row_idx/mod.rs +++ b/vortex-layout/src/layouts/row_idx/mod.rs @@ -249,6 +249,10 @@ impl LayoutReader for RowIdxLayoutReader { } } } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } // Returns a SequenceArray representing the row indices for the given row range, diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index 6c39a8f1086..b0777f3ee89 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -384,6 +384,10 @@ impl LayoutReader for StructReader { } })) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[cfg(test)] diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 1fd1ab6b0cc..4485440515e 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -330,6 +330,10 @@ impl LayoutReader for ZonedReader { self.data_child()? .projection_evaluation(row_range, expr, mask) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } /// A wrapper for the result of pruning an expression against a zone map such that we can refresh diff --git a/vortex-layout/src/reader.rs b/vortex-layout/src/reader.rs index e103c163efe..f1467212c38 100644 --- a/vortex-layout/src/reader.rs +++ b/vortex-layout/src/reader.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::any::Any; use std::collections::BTreeSet; use std::ops::Range; use std::sync::Arc; @@ -31,6 +32,8 @@ pub trait LayoutReader: 'static + Send + Sync { /// Returns the name of the layout reader for debugging. fn name(&self) -> &Arc; + fn as_any(&self) -> &dyn Any; + /// Returns the un-projected dtype of the layout reader. fn dtype(&self) -> &DType; diff --git a/vortex-layout/src/scan/multi.rs b/vortex-layout/src/scan/multi.rs index 307aca187cc..600d14ccffd 100644 --- a/vortex-layout/src/scan/multi.rs +++ b/vortex-layout/src/scan/multi.rs @@ -84,7 +84,7 @@ pub struct MultiLayoutDataSource { concurrency: usize, } -enum MultiLayoutChild { +pub enum MultiLayoutChild { Opened(LayoutReaderRef), Deferred(Arc), } @@ -141,6 +141,10 @@ impl MultiLayoutDataSource { } } + pub fn children(&self) -> &Vec { + &self.children + } + /// Sets the concurrency for opening deferred readers. /// /// Controls how many file opens run in parallel via `buffer_unordered`. diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index e1b652631ec..6053cd6ebf5 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -555,6 +555,10 @@ mod test { unreachable!("scan should not be polled in this test") })) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[test] @@ -650,6 +654,10 @@ mod test { let array = PrimitiveArray::from_iter(values?).into_array(); Ok(Box::pin(async move { Ok(array) })) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[test] @@ -750,6 +758,10 @@ mod test { unreachable!("scan should not be polled in this test") })) } + + fn as_any(&self) -> &dyn std::any::Any { + self + } } #[test] From 4d73f979eac8b019faeea763e8c6ff9f6d9e75d4 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 17 Apr 2026 13:22:11 -0400 Subject: [PATCH 105/250] fix potential UB in ByteBool encoding (#7518) ## Summary Our existing ByteBool constructor accepts arbitrary bytes as input, and will attempt to transmute it to a `&[bool]`. If those bytes are not `0x00` or `0x01` that can trigger UB on release builds. ## API Changes We eliminate the `ByteBool::as_slice() -> &[bool]` method, replacing it with an accessor to the truthy byte values. We don't really need to access this as a bool-slice anyway since we can do all operations on the bytes and wrap them back up as a ByteBool. ## Tests No additional tests needed --------- Signed-off-by: Andrew Duffy --- encodings/bytebool/public-api.lock | 16 +----- encodings/bytebool/src/array.rs | 91 ++++++++++-------------------- encodings/bytebool/src/compute.rs | 14 +++-- encodings/bytebool/src/lib.rs | 39 +++++++++++++ vortex-buffer/public-api.lock | 4 ++ vortex-buffer/src/bit/buf_mut.rs | 7 +++ 6 files changed, 92 insertions(+), 79 deletions(-) diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 8edd17d2912..53c71e3bfb6 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -76,17 +76,15 @@ pub struct vortex_bytebool::ByteBoolData impl vortex_bytebool::ByteBoolData -pub fn vortex_bytebool::ByteBoolData::as_slice(&self) -> &[bool] - pub fn vortex_bytebool::ByteBoolData::buffer(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_bytebool::ByteBoolData::from_vec>(data: alloc::vec::Vec, validity: V) -> Self - pub fn vortex_bytebool::ByteBoolData::is_empty(&self) -> bool pub fn vortex_bytebool::ByteBoolData::len(&self) -> usize -pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHandle) -> Self + +pub fn vortex_bytebool::ByteBoolData::truthy_bytes(&self) -> &[u8] pub fn vortex_bytebool::ByteBoolData::validate(buffer: &vortex_array::buffer::BufferHandle, validity: &vortex_array::validity::Validity, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> @@ -94,14 +92,6 @@ impl core::clone::Clone for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::clone(&self) -> vortex_bytebool::ByteBoolData -impl core::convert::From> for vortex_bytebool::ByteBoolData - -pub fn vortex_bytebool::ByteBoolData::from(value: alloc::vec::Vec) -> Self - -impl core::convert::From>> for vortex_bytebool::ByteBoolData - -pub fn vortex_bytebool::ByteBoolData::from(value: alloc::vec::Vec>) -> Self - impl core::fmt::Debug for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 1043aefa967..e1604b7642e 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -29,7 +29,7 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; -use vortex_buffer::BitBuffer; +use vortex_buffer::BitBufferMut; use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -131,7 +131,7 @@ impl VTable for ByteBool { } let buffer = buffers[0].clone(); - let data = ByteBoolData::new(buffer, validity.clone()); + let data = ByteBoolData::new(buffer); let slots = ByteBoolData::make_slots(&validity, len); Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots)) } @@ -149,7 +149,8 @@ impl VTable for ByteBool { } fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { - let boolean_buffer = BitBuffer::from(array.as_slice()); + // convert truthy values to set/unset bits + let boolean_buffer = BitBufferMut::from(array.truthy_bytes()).freeze(); let validity = array.validity()?; Ok(ExecutionResult::done( BoolArray::new(boolean_buffer, validity).into_array(), @@ -198,9 +199,17 @@ pub struct ByteBool; impl ByteBool { pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { + if let Some(len) = validity.maybe_len() { + assert_eq!( + buffer.len(), + len, + "ByteBool validity and bytes must have same length" + ); + } let dtype = DType::Bool(validity.nullability()); + let slots = ByteBoolData::make_slots(&validity, buffer.len()); - let data = ByteBoolData::new(buffer, validity); + let data = ByteBoolData::new(buffer); let len = data.len(); unsafe { Array::from_parts_unchecked( @@ -212,29 +221,22 @@ impl ByteBool { /// Construct a [`ByteBoolArray`] from a `Vec` and validity. pub fn from_vec>(data: Vec, validity: V) -> ByteBoolArray { let validity = validity.into(); - let data = ByteBoolData::from_vec(data, validity.clone()); - let dtype = DType::Bool(validity.nullability()); - let len = data.len(); - let slots = ByteBoolData::make_slots(&validity, len); - unsafe { - Array::from_parts_unchecked( - ArrayParts::new(ByteBool, dtype, len, data).with_slots(slots), - ) - } + // NOTE: this will not cause allocation on release builds + let bytes: Vec = data.into_iter().map(|b| b as u8).collect(); + let handle = BufferHandle::new_host(ByteBuffer::from(bytes)); + ByteBool::new(handle, validity) } /// Construct a [`ByteBoolArray`] from optional bools. pub fn from_option_vec(data: Vec>) -> ByteBoolArray { let validity = Validity::from_iter(data.iter().map(|v| v.is_some())); - let data = ByteBoolData::from(data); - let dtype = DType::Bool(validity.nullability()); - let len = data.len(); - let slots = ByteBoolData::make_slots(&validity, len); - unsafe { - Array::from_parts_unchecked( - ArrayParts::new(ByteBool, dtype, len, data).with_slots(slots), - ) - } + // NOTE: this will not cause allocation on release builds + let bytes: Vec = data + .into_iter() + .map(|b| b.unwrap_or_default() as u8) + .collect(); + let handle = BufferHandle::new_host(ByteBuffer::from(bytes)); + ByteBool::new(handle, validity) } } @@ -265,17 +267,7 @@ impl ByteBoolData { vec![validity_to_child(validity, len)] } - pub fn new(buffer: BufferHandle, validity: Validity) -> Self { - let length = buffer.len(); - if let Some(vlen) = validity.maybe_len() - && length != vlen - { - vortex_panic!( - "Buffer length ({}) does not match validity length ({})", - length, - vlen - ); - } + pub fn new(buffer: BufferHandle) -> Self { Self { buffer } } @@ -289,21 +281,15 @@ impl ByteBoolData { self.buffer.len() == 0 } - // TODO(ngates): deprecate construction from vec - pub fn from_vec>(data: Vec, validity: V) -> Self { - let validity = validity.into(); - // SAFETY: we are transmuting a Vec into a Vec - let data: Vec = unsafe { std::mem::transmute(data) }; - Self::new(BufferHandle::new_host(ByteBuffer::from(data)), validity) - } - pub fn buffer(&self) -> &BufferHandle { &self.buffer } - pub fn as_slice(&self) -> &[bool] { - // Safety: The internal buffer contains byte-sized bools - unsafe { std::mem::transmute(self.buffer().as_host().as_slice()) } + /// Get access to the underlying 8-bit truthy values. + /// + /// The zero byte indicates `false`, and any non-zero byte is a `true`. + pub fn truthy_bytes(&self) -> &[u8] { + self.buffer().as_host().as_slice() } } @@ -326,23 +312,6 @@ impl OperationsVTable for ByteBool { } } -impl From> for ByteBoolData { - fn from(value: Vec) -> Self { - Self::from_vec(value, Validity::AllValid) - } -} - -impl From>> for ByteBoolData { - fn from(value: Vec>) -> Self { - let validity = Validity::from_iter(value.iter().map(|v| v.is_some())); - - // This doesn't reallocate, and the compiler even vectorizes it - let data = value.into_iter().map(Option::unwrap_or_default).collect(); - - Self::from_vec(data, validity) - } -} - #[cfg(test)] mod tests { use vortex_array::ArrayContext; diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 0fd75cb8123..609a7bc553f 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -8,11 +8,13 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::TakeExecute; +use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::match_each_integer_ptype; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_array::validity::Validity; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use super::ByteBool; @@ -58,23 +60,25 @@ impl TakeExecute for ByteBool { ctx: &mut ExecutionCtx, ) -> VortexResult> { let indices = indices.clone().execute::(ctx)?; - let bools = array.as_slice(); + let values = array.truthy_bytes(); // This handles combining validity from both source array and nullable indices let validity = array.validity()?.take(&indices.clone().into_array())?; - let taken_bools = match_each_integer_ptype!(indices.ptype(), |I| { + let taken = match_each_integer_ptype!(indices.ptype(), |I| { indices .as_slice::() .iter() .map(|&idx| { let idx: usize = idx.as_(); - bools[idx] + values[idx] }) - .collect::>() + .collect::() }); - Ok(Some(ByteBool::from_vec(taken_bools, validity).into_array())) + Ok(Some( + ByteBool::new(BufferHandle::new_host(taken), validity).into_array(), + )) } } diff --git a/encodings/bytebool/src/lib.rs b/encodings/bytebool/src/lib.rs index 35579a89dc8..3258341cdb9 100644 --- a/encodings/bytebool/src/lib.rs +++ b/encodings/bytebool/src/lib.rs @@ -1,6 +1,45 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +//! A Vortex encoding that mirrors Arrow's [8-bit Boolean canonical extension type][spec]. +//! +//! Each element is stored as a single byte. The zero byte represents `false` and any +//! non-zero byte represents `true`, matching the truthy semantics of the Arrow spec. This +//! trades 8x the storage of the bit-packed `Bool` layout for cheaper per-byte access — +//! useful when data arrives from a C ABI or other source that already emits byte-wide +//! booleans. On execution the array materializes into the standard bit-packed +//! [`BoolArray`][vortex_array::arrays::BoolArray]. +//! +//! # Examples +//! +//! Any non-zero byte in the backing buffer is treated as `true` when the array executes +//! to a canonical [`BoolArray`][vortex_array::arrays::BoolArray]: +//! +//! ``` +//! # use vortex_array::{IntoArray, LEGACY_SESSION, VortexSessionExecute}; +//! # use vortex_array::arrays::BoolArray; +//! # use vortex_array::arrays::bool::BoolArrayExt; +//! # use vortex_array::buffer::BufferHandle; +//! # use vortex_array::validity::Validity; +//! # use vortex_buffer::ByteBuffer; +//! # use vortex_bytebool::ByteBool; +//! # use vortex_error::VortexResult; +//! # fn main() -> VortexResult<()> { +//! # let mut ctx = LEGACY_SESSION.create_execution_ctx(); +//! let handle = BufferHandle::new_host(ByteBuffer::from(vec![0u8, 1, 42, 0])); +//! let array = ByteBool::new(handle, Validity::NonNullable); +//! +//! let bits = array.into_array().execute::(&mut ctx)?.to_bit_buffer(); +//! assert!(!bits.value(0)); +//! assert!(bits.value(1)); +//! assert!(bits.value(2)); // byte 42 is truthy +//! assert!(!bits.value(3)); +//! # Ok(()) +//! # } +//! ``` +//! +//! [spec]: https://arrow.apache.org/docs/format/CanonicalExtensions.html#bit-boolean + pub use array::*; mod array; diff --git a/vortex-buffer/public-api.lock b/vortex-buffer/public-api.lock index c89264da253..cd9e0900247 100644 --- a/vortex-buffer/public-api.lock +++ b/vortex-buffer/public-api.lock @@ -504,6 +504,10 @@ impl core::convert::From<&[bool]> for vortex_buffer::BitBufferMut pub fn vortex_buffer::BitBufferMut::from(value: &[bool]) -> Self +impl core::convert::From<&[u8]> for vortex_buffer::BitBufferMut + +pub fn vortex_buffer::BitBufferMut::from(value: &[u8]) -> Self + impl core::convert::From> for vortex_buffer::BitBufferMut pub fn vortex_buffer::BitBufferMut::from(value: alloc::vec::Vec) -> Self diff --git a/vortex-buffer/src/bit/buf_mut.rs b/vortex-buffer/src/bit/buf_mut.rs index f46a00a8fe0..bf42e92d571 100644 --- a/vortex-buffer/src/bit/buf_mut.rs +++ b/vortex-buffer/src/bit/buf_mut.rs @@ -573,6 +573,13 @@ impl From<&[bool]> for BitBufferMut { } } +// allow building a buffer from a set of truthy byte values. +impl From<&[u8]> for BitBufferMut { + fn from(value: &[u8]) -> Self { + BitBufferMut::collect_bool(value.len(), |i| value[i] > 0) + } +} + impl From> for BitBufferMut { fn from(value: Vec) -> Self { value.as_slice().into() From 9b11e576c5e06d8cd28b9822d9b72625917751a3 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:33:31 -0400 Subject: [PATCH 106/250] Fix reduction rule (#7521) ## Summary Changes a parent reduction rule on `AnyArray` to just a reduction rule. Also my lsp reordered some things. ## Testing Existing tests pass (except 1 which I had to update). Signed-off-by: Connor Tsui --- .../src/arrays/extension/compute/rules.rs | 55 +++++++-------- .../src/arrays/extension/vtable/mod.rs | 67 ++++++++++--------- 2 files changed, 60 insertions(+), 62 deletions(-) diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 7408488a0f1..568fe1c061c 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -14,52 +14,44 @@ use crate::arrays::Filter; use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::slice::SliceReduceAdaptor; -use crate::matcher::AnyArray; use crate::optimizer::rules::ArrayParentReduceRule; +use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; +use crate::optimizer::rules::ReduceRuleSet; use crate::scalar::Scalar; use crate::scalar_fn::fns::cast::CastReduceAdaptor; use crate::scalar_fn::fns::mask::MaskReduceAdaptor; -pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ - ParentRuleSet::lift(&ExtensionConstantParentRule), - ParentRuleSet::lift(&ExtensionFilterPushDownRule), - ParentRuleSet::lift(&CastReduceAdaptor(Extension)), - ParentRuleSet::lift(&FilterReduceAdaptor(Extension)), - ParentRuleSet::lift(&MaskReduceAdaptor(Extension)), - ParentRuleSet::lift(&SliceReduceAdaptor(Extension)), -]); +pub(crate) const RULES: ReduceRuleSet = ReduceRuleSet::new(&[&ExtensionConstantRule]); /// Normalize `Extension(Constant(storage))` children to `Constant(Extension(storage))`. #[derive(Debug)] -struct ExtensionConstantParentRule; - -impl ArrayParentReduceRule for ExtensionConstantParentRule { - type Parent = AnyArray; +struct ExtensionConstantRule; - fn reduce_parent( - &self, - child: ArrayView<'_, Extension>, - parent: &ArrayRef, - child_idx: usize, - ) -> VortexResult> { - let Some(const_array) = child.storage_array().as_opt::() else { +impl ArrayReduceRule for ExtensionConstantRule { + fn reduce(&self, array: ArrayView<'_, Extension>) -> VortexResult> { + let Some(const_array) = array.storage_array().as_opt::() else { return Ok(None); }; let storage_scalar = const_array.scalar().clone(); - let ext_scalar = Scalar::extension_ref(child.ext_dtype().clone(), storage_scalar); + let ext_scalar = Scalar::extension_ref(array.ext_dtype().clone(), storage_scalar); let constant_with_extension_scalar = - ConstantArray::new(ext_scalar, child.len()).into_array(); + ConstantArray::new(ext_scalar, array.len()).into_array(); - parent - .clone() - .with_slot(child_idx, constant_with_extension_scalar) - .map(Some) + Ok(Some(constant_with_extension_scalar.into_array())) } } +pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ + ParentRuleSet::lift(&ExtensionFilterPushDownRule), + ParentRuleSet::lift(&CastReduceAdaptor(Extension)), + ParentRuleSet::lift(&FilterReduceAdaptor(Extension)), + ParentRuleSet::lift(&MaskReduceAdaptor(Extension)), + ParentRuleSet::lift(&SliceReduceAdaptor(Extension)), +]); + /// Push filter operations into the storage array of an extension array. #[derive(Debug)] struct ExtensionFilterPushDownRule; @@ -99,6 +91,7 @@ mod tests { use crate::arrays::ExtensionArray; use crate::arrays::FilterArray; use crate::arrays::PrimitiveArray; + use crate::arrays::ScalarFnVTable; use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnFactoryExt; @@ -227,8 +220,8 @@ mod tests { .try_new_array(3, Operator::Lt, [constant_ext, ext_array]) .unwrap(); - let optimized = scalar_fn_array.optimize().unwrap(); - let scalar_fn = optimized.as_opt::().unwrap(); + let optimized = scalar_fn_array.optimize_recursive().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); let children = scalar_fn.children(); let constant = children[0] .as_opt::() @@ -291,7 +284,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // The first child should still be an ExtensionArray (no pushdown happened) - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when ext types differ" @@ -316,7 +309,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // No pushdown should happen because sibling is not a constant - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when sibling is not constant" @@ -339,7 +332,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // No pushdown should happen because constant is not an extension scalar - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when constant is not extension" diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index b82d4b8e0df..a3fde75f79b 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -31,10 +31,14 @@ use crate::arrays::extension::ExtensionData; use crate::arrays::extension::array::SLOT_NAMES; use crate::arrays::extension::array::STORAGE_SLOT; use crate::arrays::extension::compute::rules::PARENT_RULES; +use crate::arrays::extension::compute::rules::RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; +#[derive(Clone, Debug)] +pub struct Extension; + /// A [`Extension`]-encoded Vortex array. pub type ExtensionArray = Array; @@ -59,29 +63,6 @@ impl VTable for Extension { *ID } - fn nbuffers(_array: ArrayView<'_, Self>) -> usize { - 0 - } - - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - vortex_panic!("ExtensionArray buffer index {idx} out of bounds") - } - - fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { - None - } - - fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - SLOT_NAMES[idx].to_string() - } - - fn serialize( - _array: ArrayView<'_, Self>, - _session: &VortexSession, - ) -> VortexResult>> { - Ok(Some(vec![])) - } - fn validate( &self, data: &ExtensionData, @@ -111,6 +92,25 @@ impl VTable for Extension { Ok(()) } + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("ExtensionArray buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { + None + } + + fn serialize( + _array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(Some(vec![])) + } + fn deserialize( &self, dtype: &DType, @@ -143,27 +143,32 @@ impl VTable for Extension { .with_slots(vec![Some(storage)])) } + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(array)) } - fn reduce_parent( + fn execute_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, child_idx: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult> { - PARENT_RULES.evaluate(array, parent, child_idx) + PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn execute_parent( + fn reduce(array: ArrayView<'_, Self>) -> VortexResult> { + RULES.evaluate(array) + } + + fn reduce_parent( array: ArrayView<'_, Self>, parent: &ArrayRef, child_idx: usize, - ctx: &mut ExecutionCtx, ) -> VortexResult> { - PARENT_KERNELS.execute(array, parent, child_idx, ctx) + PARENT_RULES.evaluate(array, parent, child_idx) } } - -#[derive(Clone, Debug)] -pub struct Extension; From f308dfe25c0f863a2450a84fe7415efab261d924 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 17 Apr 2026 20:23:47 +0100 Subject: [PATCH 107/250] add unchecked array slot take and put (#7514) Adds crate-private unchecked slot take/put helpers on `ArrayRef`. This allows for in-place swapping of array children is the array is exclusively owned. --------- Signed-off-by: Joe Isaacs --- vortex-array/src/array/erased.rs | 51 ++++++++++++++++++++++++ vortex-array/src/array/mod.rs | 11 ++++++ vortex-array/src/executor.rs | 68 ++++++++++++++++++++++++++------ 3 files changed, 118 insertions(+), 12 deletions(-) diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index 84367076314..3046e2ba57c 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -431,6 +431,56 @@ impl ArrayRef { self.with_slots(slots) } + /// Take a slot for executor-owned physical rewrites. This has the result that the array may + /// either be taken or cloned from the parent. + /// + /// The array can be put back with [`put_slot_unchecked`]. + /// + /// # Safety + /// The caller must put back a slot with the same logical dtype and length before exposing the + /// parent array, and must only use this for physical rewrites. + pub(crate) unsafe fn take_slot_unchecked( + mut self, + slot_idx: usize, + ) -> VortexResult<(ArrayRef, ArrayRef)> { + let child = if let Some(inner) = Arc::get_mut(&mut self.0) { + // # Safety: ensured by the caller. + unsafe { inner.slots_mut()[slot_idx].take() } + .vortex_expect("take_slot_unchecked cannot take an absent slot") + } else { + self.slots()[slot_idx] + .as_ref() + .vortex_expect("take_slot_unchecked cannot take an absent slot") + .clone() + }; + + Ok((self, child)) + } + + /// Puts an array into `slot_idx` by either, cloning the inner array if the Arc is not exclusive + /// or replacing the slot in this `ArrayRef`. + /// This is the mirror of [`take_slot_unchecked`]. + /// + /// # Safety + /// The replacement must have the same logical dtype and length as the taken slot, and this + /// must only be used for physical rewrites. + pub(crate) unsafe fn put_slot_unchecked( + mut self, + slot_idx: usize, + replacement: ArrayRef, + ) -> VortexResult { + if let Some(inner) = Arc::get_mut(&mut self.0) { + // # Safety: ensured by the caller. + unsafe { inner.slots_mut()[slot_idx] = Some(replacement) }; + return Ok(self); + } + + let mut slots = self.slots().to_vec(); + slots[slot_idx] = Some(replacement); + let inner = Arc::clone(&self.0); + inner.with_slots(self, slots) + } + /// Returns a new array with the provided slots. /// /// This is only valid for physical rewrites: slot count, presence, logical `DType`, and @@ -611,6 +661,7 @@ impl Matcher for V { fn try_match<'a>(array: &'a ArrayRef) -> Option> { let inner = array.0.as_any().downcast_ref::>()?; + // # Safety checked by `downcast_ref`. Some(unsafe { ArrayView::new_unchecked(array, &inner.data) }) } } diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 8193c128b1e..888e7bf6893 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -68,6 +68,13 @@ pub(crate) trait DynArray: 'static + private::Sealed + Send + Sync + Debug { /// Returns the slots of the array. fn slots(&self) -> &[Option]; + /// Returns mutable slots of the array. + /// + /// # Safety: any slot (Some(child)) that replaces an existing slot must have a compatible + /// DType and length. Currently compatible means equal, but there is no reason why that must + /// be the case. + unsafe fn slots_mut(&mut self) -> &mut [Option]; + /// Returns the encoding ID of the array. fn encoding_id(&self) -> ArrayId; @@ -212,6 +219,10 @@ impl DynArray for ArrayInner { &self.slots } + unsafe fn slots_mut(&mut self) -> &mut [Option] { + &mut self.slots + } + fn encoding_id(&self) -> ArrayId { self.vtable.id() } diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 8e0c86ff44f..4e2b2dfce53 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -33,6 +33,7 @@ use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::dtype::DType; use crate::matcher::Matcher; use crate::memory::HostAllocatorRef; use crate::memory::MemorySessionExt; @@ -107,22 +108,21 @@ impl ArrayRef { /// maximum (default 128, override with `VORTEX_MAX_ITERATIONS`). pub fn execute_until(self, ctx: &mut ExecutionCtx) -> VortexResult { let mut current = self.optimize()?; - // Stack frames: (parent, slot_idx, done_predicate_for_slot) - let mut stack: Vec<(ArrayRef, usize, DonePredicate)> = Vec::new(); + let mut stack: Vec = Vec::new(); for _ in 0..max_iterations() { // Step 1: done / canonical — splice back into stacked parent or return. let is_done = stack .last() - .map_or(M::matches as DonePredicate, |frame| frame.2); + .map_or(M::matches as DonePredicate, |frame| frame.done); if is_done(¤t) || AnyCanonical::matches(¤t) { match stack.pop() { None => { ctx.log(format_args!("-> {}", current)); return Ok(current); } - Some((parent, slot_idx, _)) => { - current = parent.with_slot(slot_idx, current)?.optimize()?; + Some(frame) => { + current = frame.put_back(current)?.optimize()?; continue; } } @@ -139,8 +139,8 @@ impl ArrayRef { current, rewritten )); current = rewritten.optimize()?; - if let Some((parent, slot_idx, _)) = stack.pop() { - current = parent.with_slot(slot_idx, current)?.optimize()?; + if let Some(frame) = stack.pop() { + current = frame.put_back(current)?.optimize()?; } continue; } @@ -150,14 +150,15 @@ impl ArrayRef { let (array, step) = result.into_parts(); match step { ExecutionStep::ExecuteSlot(i, done) => { - let child = array.slots()[i] - .clone() - .vortex_expect("ExecuteSlot index in bounds"); + // SAFETY: we record the child's dtype and len, and assert they are preserved + // when the slot is put back via `put_slot_unchecked`. + let (parent, child) = unsafe { array.take_slot_unchecked(i) }?; ctx.log(format_args!( "ExecuteSlot({i}): pushing {}, focusing on {}", - array, child + parent, child )); - stack.push((array, i, done)); + let frame = StackFrame::new(parent, i, done, &child); + stack.push(frame); current = child.optimize()?; } ExecutionStep::Done => { @@ -174,6 +175,49 @@ impl ArrayRef { } } +/// A stack frame for the iterative executor, tracking the parent array whose slot is being +/// executed and the original child's dtype/len for validation on put-back. +struct StackFrame { + parent: ArrayRef, + slot_idx: usize, + done: DonePredicate, + original_dtype: DType, + original_len: usize, +} + +impl StackFrame { + fn new(parent: ArrayRef, slot_idx: usize, done: DonePredicate, child: &ArrayRef) -> Self { + Self { + parent, + slot_idx, + done, + original_dtype: child.dtype().clone(), + original_len: child.len(), + } + } + + fn put_back(self, replacement: ArrayRef) -> VortexResult { + debug_assert_eq!( + replacement.dtype(), + &self.original_dtype, + "slot {} dtype changed from {} to {} during execution", + self.slot_idx, + self.original_dtype, + replacement.dtype() + ); + debug_assert_eq!( + replacement.len(), + self.original_len, + "slot {} len changed from {} to {} during execution", + self.slot_idx, + self.original_len, + replacement.len() + ); + // SAFETY: we assert above that dtype and len are preserved. + unsafe { self.parent.put_slot_unchecked(self.slot_idx, replacement) } + } +} + /// Execution context for batch CPU compute. #[derive(Debug, Clone)] pub struct ExecutionCtx { From 609e9aae3caaa5b91ede2888a0a76986f0af3131 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:16:55 -0400 Subject: [PATCH 108/250] Vector similarity search scan benchmarks (#7499) ## Summary Tracking Issue: https://github.com/vortex-data/vortex/issues/7297 Adds a basic vector similarity search benchmark to the Vortex benchmark suite as a binary. Here is an example of how to run this: ```sh cargo run -p vector-search-bench --release -- --dataset cohere-small-100k --layout single --iterations 100 cargo run -p vector-search-bench --release -- --dataset bioasq-large-10m --layout partitioned-shuffled --iterations 1 ``` The main scan logic is in `scan_one_file` in `benchmarks/vector-search-bench/src/scan.rs`, and everything else is just setup for that. ## Future Work - This does not measure recall, but that can come in a followup PR. - We will want to have a handrolled baseline implementation that is the "theoretical" minimum over uncompressed f32 vectors that we can compare against. We can then add more flavors of compression and quantization in the future. - No filter pushdown still (doing filter pushdown before similarity search), have to figure out why that isn't working when we eventually benchmark that - Recall over filtered cosine search ## Testing The benchmark running successfully for all datasets is sufficient. Signed-off-by: Connor Tsui --- .../vector-search-bench/src/compression.rs | 2 - benchmarks/vector-search-bench/src/display.rs | 133 +++++++++++++ benchmarks/vector-search-bench/src/ingest.rs | 96 +++++---- benchmarks/vector-search-bench/src/lib.rs | 3 + benchmarks/vector-search-bench/src/main.rs | 188 ++++++++++++++++++ benchmarks/vector-search-bench/src/prepare.rs | 48 ++--- benchmarks/vector-search-bench/src/query.rs | 120 +++++++++++ benchmarks/vector-search-bench/src/scan.rs | 177 +++++++++++++++++ 8 files changed, 690 insertions(+), 77 deletions(-) create mode 100644 benchmarks/vector-search-bench/src/display.rs create mode 100644 benchmarks/vector-search-bench/src/main.rs create mode 100644 benchmarks/vector-search-bench/src/query.rs create mode 100644 benchmarks/vector-search-bench/src/scan.rs diff --git a/benchmarks/vector-search-bench/src/compression.rs b/benchmarks/vector-search-bench/src/compression.rs index 65bf080db85..9b40cb15544 100644 --- a/benchmarks/vector-search-bench/src/compression.rs +++ b/benchmarks/vector-search-bench/src/compression.rs @@ -8,8 +8,6 @@ //! //! The benchmark writes one `.vortex` file per flavor per data file, then scans them all with the //! same query so the comparison is apples-to-apples with the Parquet files. -//! -//! Note that the handrolled `&[f32]` parquet baseline is **not** a flavor here. use clap::ValueEnum; use vortex::array::ArrayId; diff --git a/benchmarks/vector-search-bench/src/display.rs b/benchmarks/vector-search-bench/src/display.rs new file mode 100644 index 00000000000..a74eec46b01 --- /dev/null +++ b/benchmarks/vector-search-bench/src/display.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Local table renderer for the vector-search benchmark. +//! +//! Groups columns by **flavor** (`vortex-uncompressed`, `vortex-turboquant`) rather than by +//! [`vortex_bench::Format`], because the two Vortex flavors share a single +//! `Format::OnDiskVortex`/`Format::VortexLossy` pair and the generic +//! [`vortex_bench::display::render_table`] groups by Format. Local renderer keeps the +//! column-per-flavor invariant intact without introducing a new global Format value. +//! +//! Output rows: +//! +//! ```text +//! Metric | vortex-uncompressed | vortex-turboquant +//! ------------------ + ------------------- + ----------------- +//! scan wall (mean) | 485 ms | 212 ms +//! scan wall (median) | 490 ms | 215 ms +//! matches | 42 | 39 +//! rows scanned | 10,000,000 | 10,000,000 +//! bytes scanned | 30.5 GB | 7.62 GB +//! rows / sec | 5.2e6 | 1.2e7 +//! ``` + +use std::io::Write; + +use anyhow::Result; +use tabled::settings::Style; + +use crate::compression::VectorFlavor; +use crate::prepare::CompressedVortexDataset; +use crate::scan::ScanTiming; + +/// Final column-per-flavor row set for one dataset. +pub struct DatasetReport<'a> { + pub dataset_name: &'a str, + pub vortex_results: &'a [(VectorFlavor, &'a CompressedVortexDataset, &'a ScanTiming)], +} + +/// Render the full report into the given writer as a tabled table. +pub fn render(report: &DatasetReport<'_>, writer: &mut dyn Write) -> Result<()> { + let mut headers: Vec = vec!["metric".to_owned()]; + for &(flavor, ..) in report.vortex_results { + headers.push(flavor.label().to_owned()); + } + + let rows: Vec> = vec![ + make_row("scan wall (mean)", report, |_, _, scan| { + format_duration(scan.mean) + }), + make_row("scan wall (median)", report, |_, _, scan| { + format_duration(scan.median) + }), + make_row("matches", report, |_, _, scan| scan.matches.to_string()), + make_row("rows scanned", report, |_, _, scan| { + scan.rows_scanned.to_string() + }), + make_row("bytes scanned", report, |_, _, scan| { + format_bytes(scan.bytes_scanned) + }), + make_row("rows / sec", report, |_, _, scan| { + format_throughput_rows(scan.rows_scanned, scan.mean) + }), + ]; + + writeln!(writer, "## {}", report.dataset_name)?; + let mut builder = tabled::builder::Builder::new(); + builder.push_record(headers); + for row in rows { + builder.push_record(row); + } + let mut table = builder.build(); + table.with(Style::modern()); + writeln!(writer, "{table}")?; + Ok(()) +} + +fn make_row(metric: &str, report: &DatasetReport<'_>, vortex_cell: F) -> Vec +where + F: Fn(VectorFlavor, &CompressedVortexDataset, &ScanTiming) -> String, +{ + let mut row = vec![metric.to_owned()]; + for &(flavor, prep, scan) in report.vortex_results { + row.push(vortex_cell(flavor, prep, scan)); + } + row +} + +fn format_duration(d: std::time::Duration) -> String { + let secs = d.as_secs_f64(); + if secs >= 1.0 { + format!("{secs:.2} s") + } else if secs >= 1e-3 { + format!("{:.1} ms", secs * 1e3) + } else { + format!("{:.1} µs", secs * 1e6) + } +} + +fn format_bytes(bytes: u64) -> String { + const UNITS: &[&str] = &["B", "KiB", "MiB", "GiB", "TiB"]; + let mut value = bytes as f64; + let mut unit = UNITS[0]; + for next in &UNITS[1..] { + if value < 1024.0 { + break; + } + value /= 1024.0; + unit = next; + } + if unit == "B" { + format!("{bytes} B") + } else { + format!("{value:.2} {unit}") + } +} + +fn format_throughput_rows(rows: u64, wall: std::time::Duration) -> String { + let secs = wall.as_secs_f64(); + if secs <= 0.0 { + return "—".to_owned(); + } + let rps = rows as f64 / secs; + if rps >= 1e9 { + format!("{:.2}G", rps / 1e9) + } else if rps >= 1e6 { + format!("{:.2}M", rps / 1e6) + } else if rps >= 1e3 { + format!("{:.2}K", rps / 1e3) + } else { + format!("{rps:.0}") + } +} diff --git a/benchmarks/vector-search-bench/src/ingest.rs b/benchmarks/vector-search-bench/src/ingest.rs index 48904530170..f3fe04f4422 100644 --- a/benchmarks/vector-search-bench/src/ingest.rs +++ b/benchmarks/vector-search-bench/src/ingest.rs @@ -8,7 +8,9 @@ //! 1. Project the `emb` column out of each struct chunk. //! 2. Rewrap the `emb` column as `Extension>` via //! [`vortex_bench::vector_dataset::list_to_vector_ext`]. -//! 3. Cast the FSL element buffer from `f64` -> `f32` if the source is `f64`. After this point all +//! 3. Detect the FSL element ptype at runtime and cast `f64` -> `f32` when needed. Detection is +//! from the arrow schema rather than a catalog declaration so upstream parquets whose actual +//! precision disagrees with the catalog still ingest correctly. After this point all //! downstream code (compression, scan, recall) is f32-only. //! 4. Optionally project the `scalar_labels` column through unchanged so future filtered-search //! benchmarks have it without re-ingest. @@ -39,48 +41,50 @@ use vortex_bench::vector_dataset::list_to_vector_ext; use vortex_tensor::vector::AnyVector; use vortex_tensor::vector::Vector; -/// Configuration passed alongside each chunk so the transform can stay stateless. -#[derive(Debug, Clone, Copy)] -pub struct ChunkTransform { - /// Source element ptype as declared by the dataset catalog. Used purely to decide whether the - /// f64 -> f32 cast is needed. - pub src_ptype: PType, - // /// Whether to project the `scalar_labels` column through the output struct. - // pub include_scalar_labels: bool, -} +/// Apply the transform to a single struct chunk and return the rebuilt chunk. +/// +/// `chunk` must be a non-chunked `Struct { id: i64, emb: List }`, where all of the list +/// elements are +/// +/// The returned array is always a `Struct { id: i64, emb: Vector }`. +pub fn transform_chunk(chunk: ArrayRef, ctx: &mut ExecutionCtx) -> Result { + let struct_view = chunk + .as_opt::() + .with_context(|| format!("ingest: expected struct chunk, got dtype {}", chunk.dtype()))?; + + let id = struct_view + .unmasked_field_by_name("id") + .context("ingest: chunk missing `id` column")? + .clone(); + let emb = struct_view + .unmasked_field_by_name("emb") + .context("ingest: chunk missing `emb` column")? + .clone(); + + let emb_ext: ExtensionArray = list_to_vector_ext(emb)?.execute(ctx)?; + + // Detect the actual FSL element ptype from the extension storage dtype. The dataset catalog + // cannot be trusted here: at least one upstream parquet (`sift-medium-5m`) ships f64 + // embeddings despite the catalog advertising f32. + let element_ptype = { + let storage_dtype = emb_ext.storage_array().dtype(); + match storage_dtype { + DType::FixedSizeList(elem, ..) => match elem.as_ref() { + DType::Primitive(ptype, _) => *ptype, + other => bail!("ingest: expected primitive FSL element dtype, got {other}"), + }, + other => bail!("ingest: expected FSL storage dtype, got {other}"), + } + }; -impl ChunkTransform { - /// Apply the transform to a single struct chunk and return the rebuilt chunk. - /// - /// `chunk` must be a non-chunked `Struct { id: i64, emb: List }`, where all of the list - /// elements are - /// - /// The returned array is always a `Struct { id: i64, emb: Vector }`. - pub fn apply(&self, chunk: ArrayRef, ctx: &mut ExecutionCtx) -> Result { - let struct_view = chunk.as_opt::().with_context(|| { - format!("ingest: expected struct chunk, got dtype {}", chunk.dtype()) - })?; - - let id = struct_view - .unmasked_field_by_name("id") - .context("ingest: chunk missing `id` column")? - .clone(); - let emb = struct_view - .unmasked_field_by_name("emb") - .context("ingest: chunk missing `emb` column")? - .clone(); - - let emb_ext: ExtensionArray = list_to_vector_ext(emb)?.execute(ctx)?; - - let f32_vector_array = if self.src_ptype == PType::F64 { - convert_f64_to_f32_vectors(&emb_ext, ctx)? - } else { - emb_ext.into_array() - }; + let f32_vector_array = match element_ptype { + PType::F32 => emb_ext.into_array(), + PType::F64 => convert_f64_to_f32_vectors(&emb_ext, ctx)?, + other => bail!("ingest: unsupported emb element ptype {other}, expected f32 or f64"), + }; - let fields = [("id", id), ("emb", f32_vector_array)]; - Ok(StructArray::from_fields(&fields)?.into_array()) - } + let fields = [("id", id), ("emb", f32_vector_array)]; + Ok(StructArray::from_fields(&fields)?.into_array()) } /// Convert a `Vector` extension array down to `Vector`. @@ -164,10 +168,7 @@ mod tests { let emb = list_chunk_f64(&[&[1.0, 2.0, 3.0], &[4.0, 5.0, 6.0]]); let chunk = StructArray::from_fields(&[("id", id_array(&[0, 1])), ("emb", emb)])?.into_array(); - let transform = ChunkTransform { - src_ptype: PType::F64, - }; - let out = transform.apply(chunk, &mut ctx)?; + let out = transform_chunk(chunk, &mut ctx)?; let out_struct = out.as_opt::().expect("returns Struct"); let out_emb = out_struct.unmasked_field_by_name("emb").unwrap().clone(); let DType::Extension(ext) = out_emb.dtype() else { @@ -207,10 +208,7 @@ mod tests { let chunk = StructArray::from_fields(&[("id", id_array(&[0, 1])), ("emb", emb)])?.into_array(); - let transform = ChunkTransform { - src_ptype: PType::F32, - }; - let out = transform.apply(chunk, &mut ctx)?; + let out = transform_chunk(chunk, &mut ctx)?; let out_struct = out.as_opt::().expect("returns Struct"); assert_eq!(out_struct.len(), 2); Ok(()) diff --git a/benchmarks/vector-search-bench/src/lib.rs b/benchmarks/vector-search-bench/src/lib.rs index ea41e773f47..643cbb5bd0d 100644 --- a/benchmarks/vector-search-bench/src/lib.rs +++ b/benchmarks/vector-search-bench/src/lib.rs @@ -4,9 +4,12 @@ //! `vector-search-bench` vector similarity-search benchmark over several datasets. pub mod compression; +pub mod display; pub mod expression; pub mod ingest; pub mod prepare; +pub mod query; +pub mod scan; use std::sync::LazyLock; diff --git a/benchmarks/vector-search-bench/src/main.rs b/benchmarks/vector-search-bench/src/main.rs new file mode 100644 index 00000000000..626e3bfce50 --- /dev/null +++ b/benchmarks/vector-search-bench/src/main.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! `vector-search-bench` — on-disk cosine-similarity scan benchmark. +//! +//! ```sh +//! cargo run -p vector-search-bench --release -- \ +//! --dataset cohere-large-10m \ +//! --layout partitioned \ +//! --flavors vortex-uncompressed,vortex-turboquant \ +//! --iterations 3 \ +//! --threshold 0.8 +//! ``` + +use std::path::PathBuf; + +use anyhow::Context; +use anyhow::Result; +use clap::Parser; +use vector_search_bench::compression::ALL_VECTOR_FLAVORS; +use vector_search_bench::compression::VectorFlavor; +use vector_search_bench::display::DatasetReport; +use vector_search_bench::display::render; +use vector_search_bench::prepare::CompressedVortexDataset; +use vector_search_bench::prepare::prepare_all; +use vector_search_bench::query::get_random_query_vector; +use vector_search_bench::scan::ScanConfig; +use vector_search_bench::scan::ScanTiming; +use vector_search_bench::scan::run_search_scan; +use vortex_bench::setup_logging_and_tracing; +use vortex_bench::vector_dataset; +use vortex_bench::vector_dataset::TrainLayout; +use vortex_bench::vector_dataset::VectorDataset; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + /// Dataset to benchmark. Single dataset per CLI invocation by design — large datasets + /// are intentionally babysat one at a time. + #[arg(long, value_enum)] + dataset: VectorDataset, + + /// Train-split layout. Required when the dataset publishes more than one layout. + /// Defaults to the catalog's first hosted layout when omitted. + #[arg(long, value_enum)] + layout: Option, + + /// Comma-separated list of flavors to run. Each Vortex flavor produces one `.vortex` file per + /// train shard. + #[arg( + long, + value_delimiter = ',', + value_enum, + default_values_t = ALL_VECTOR_FLAVORS.to_vec(), + )] + flavors: Vec, + + /// Number of timed scan iterations per flavor. Mean and median are reported. + #[arg(long, default_value_t = 5)] + iterations: usize, + + /// Cosine threshold passed to the filter expression. + #[arg(long, default_value_t = 0.85)] + threshold: f32, + + /// Seed for the test-parquet query sampler. + #[arg(long, default_value_t = 42)] + query_seed: u64, + + /// Optional path to write the rendered table to instead of stdout. + #[arg(long)] + output_path: Option, + + /// Emit verbose tracing. + #[arg(short, long)] + verbose: bool, + + /// Enable perfetto tracing output. + #[arg(long)] + tracing: bool, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + setup_logging_and_tracing(args.verbose, args.tracing)?; + + let dataset = args.dataset; + let layout = resolve_layout(dataset, args.layout)?; + tracing::info!( + "running {} on layout {} ({} dims, {} rows)", + dataset.name(), + layout, + dataset.dim(), + dataset.num_rows() + ); + + if args.flavors.is_empty() { + anyhow::bail!("no flavors selected, please pass at least one to --flavors"); + } + + // Load the source embeddings parquet files. + let datasets_paths = vector_dataset::download(dataset, layout) + .await + .with_context(|| format!("download {}", dataset.name()))?; + + // Load all vortex files needed, compressing new ones if needed. + let prepared = prepare_all(dataset, layout, &datasets_paths, &args.flavors).await?; + + let query_vector = get_random_query_vector( + &datasets_paths.test, + dataset.dim(), + dataset.element_ptype(), + args.query_seed, + ) + .await?; + tracing::info!( + "sampled query id {} (dim={})", + query_vector.id, + query_vector.query.len() + ); + + let scan_config = ScanConfig { + iterations: args.iterations, + threshold: args.threshold, + }; + + // Run all scans and record how long each takes. + let mut scan_timings: Vec = Vec::with_capacity(prepared.len()); + for prep in &prepared { + let timing = run_search_scan(prep, &query_vector.query, &scan_config).await?; + scan_timings.push(timing); + } + + // Collect the benchmark results. + let pairs: Vec<(VectorFlavor, &CompressedVortexDataset, &ScanTiming)> = prepared + .iter() + .zip(scan_timings.iter()) + .map(|(prep, scan)| (prep.flavor, prep, scan)) + .collect(); + let report = DatasetReport { + dataset_name: dataset.name(), + vortex_results: &pairs, + }; + + // Print the results. + if let Some(path) = args.output_path { + let mut file = + std::fs::File::create(&path).with_context(|| format!("create {}", path.display()))?; + render(&report, &mut file)?; + } else { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + render(&report, &mut handle)?; + } + + Ok(()) +} + +/// Every benchmark has different sets of possible dataset layouts available. The user **must** +/// provide one if there are multiple layouts. But if a dataset only has 1 layout, we can choose +/// that for them as the default. +fn resolve_layout(dataset: VectorDataset, requested: Option) -> Result { + let layouts = dataset.layouts(); + + match requested { + Some(layout) => { + dataset.validate_layout(layout)?; + Ok(layout) + } + None => { + if layouts.len() == 1 { + Ok(layouts[0].layout()) + } else { + let allowed = layouts + .iter() + .map(|s| s.layout().label()) + .collect::>() + .join(", "); + anyhow::bail!( + "dataset {} hosts multiple layouts ([{}]): pass --layout to pick one", + dataset.name(), + allowed, + ); + } + } + } +} diff --git a/benchmarks/vector-search-bench/src/prepare.rs b/benchmarks/vector-search-bench/src/prepare.rs index 3674b353eac..78b8716e4a1 100644 --- a/benchmarks/vector-search-bench/src/prepare.rs +++ b/benchmarks/vector-search-bench/src/prepare.rs @@ -4,9 +4,9 @@ //! Per-flavor on-disk ingest. //! //! For each `(dataset, layout, flavor)` triple, [`prepare_flavor`] streams every parquet shard -//! through the [`crate::ingest::ChunkTransform`] and writes one `.vortex` file per shard. The -//! pipeline is idempotent (existing `.vortex` files are skipped) and reports end-to-end wall-clock -//! time, summed input parquet bytes, and total output bytes. +//! and writes one `.vortex` file per shard. The pipeline is idempotent (existing `.vortex` files +//! are skipped) and reports end-to-end wall-clock time, summed input parquet bytes, and total +//! output bytes. use std::path::Path; use std::path::PathBuf; @@ -19,6 +19,8 @@ use tokio::fs::File; use tokio::io::AsyncWriteExt; use tracing::info; use tracing::warn; +use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::array::VortexSessionExecute; use vortex::array::stream::ArrayStreamAdapter; use vortex::array::stream::ArrayStreamExt; @@ -33,25 +35,25 @@ use vortex_bench::vector_dataset::VectorDataset; use crate::SESSION; use crate::compression::VectorFlavor; -use crate::ingest::ChunkTransform; +use crate::ingest::transform_chunk; /// The paths of the vortex files that result from preparing one `(dataset, layout, flavor)` triple. #[derive(Debug, Clone)] -pub struct CompressedVortexDataSet { +pub struct CompressedVortexDataset { pub dataset: VectorDataset, pub layout: TrainLayout, pub flavor: VectorFlavor, pub vortex_files: Vec, } -/// Drive [`prepare_flavor`] across a list of flavors, returning a [`CompressedVortexDataSet`] per flavor -/// in input order. +/// Drive [`prepare_flavor`] across a list of flavors, returning a [`CompressedVortexDataset`] per +/// flavor in input order. pub async fn prepare_all( dataset: VectorDataset, layout: TrainLayout, paths_for_dataset: &DatasetPaths, flavors: &[VectorFlavor], -) -> Result> { +) -> Result> { let mut results = Vec::with_capacity(flavors.len()); for &flavor in flavors { @@ -62,7 +64,6 @@ pub async fn prepare_all( Ok(results) } -// TODO(connor): This should probably download things in parallel? /// Prepare one flavor of one dataset by writing one `.vortex` file per train shard. /// /// This function is sequential (for now). @@ -71,11 +72,7 @@ pub async fn prepare_flavor( layout: TrainLayout, paths_for_dataset: &DatasetPaths, flavor: VectorFlavor, -) -> Result { - let transform = ChunkTransform { - src_ptype: dataset.element_ptype(), - }; - +) -> Result { let mut vortex_files = Vec::with_capacity(paths_for_dataset.train_files.len()); for parquet_path in &paths_for_dataset.train_files { @@ -99,14 +96,14 @@ pub async fn prepare_flavor( } let written_path = idempotent_async(vortex_path.as_path(), |tmp| async move { - write_shard_streaming(&parquet_path, &tmp, flavor, transform).await + write_shard_streaming(&parquet_path, &tmp, flavor).await }) .await?; vortex_files.push(written_path); } - Ok(CompressedVortexDataSet { + Ok(CompressedVortexDataset { dataset, layout, flavor, @@ -122,7 +119,6 @@ async fn write_shard_streaming( parquet_path: &Path, vortex_path: &Path, flavor: VectorFlavor, - transform: ChunkTransform, ) -> Result<()> { let file = File::open(parquet_path).await?; let builder = ParquetRecordBatchStreamBuilder::new(file).await?; @@ -132,7 +128,7 @@ async fn write_shard_streaming( // We need to get the first chunk so that we know what the dtype of the file is. let first = match array_stream.next().await { - Some(chunk) => transform_chunk(transform, chunk, &mut ctx, parquet_path, 1)?, + Some(chunk) => transform_chunk_with_error(chunk, &mut ctx, parquet_path, 1)?, None => { return Err(vortex_err!( "ingest: parquet shard {} produced no chunks", @@ -148,8 +144,7 @@ async fn write_shard_streaming( futures::stream::iter(std::iter::once(Ok(first))).chain(array_stream.enumerate().map( move |(chunk_offset, chunk_or_err)| { let mut local_ctx = SESSION.create_execution_ctx(); - transform_chunk( - transform, + transform_chunk_with_error( chunk_or_err, &mut local_ctx, &shard_path, @@ -167,6 +162,8 @@ async fn write_shard_streaming( .open(vortex_path) .await?; + // This will write in parallel, using `std::thread::available_parallelism()`. + // See `CompressingStrategy` for more details. flavor .create_write_options(&SESSION) .write(&mut output, stream) @@ -176,13 +173,12 @@ async fn write_shard_streaming( Ok(()) } -fn transform_chunk( - transform: ChunkTransform, - chunk_or_err: VortexResult, - ctx: &mut vortex::array::ExecutionCtx, +fn transform_chunk_with_error( + chunk_or_err: VortexResult, + ctx: &mut ExecutionCtx, parquet_path: &Path, chunk_idx: usize, -) -> VortexResult { +) -> VortexResult { let chunk = chunk_or_err.map_err(|err| { vortex_err!( "ingest: failed to read chunk {} from {}: {err:#}", @@ -191,7 +187,7 @@ fn transform_chunk( ) })?; - transform.apply(chunk, ctx).map_err(|err| { + transform_chunk(chunk, ctx).map_err(|err| { vortex_err!( "ingest: failed to transform chunk {} from {}: {err:#}", chunk_idx, diff --git a/benchmarks/vector-search-bench/src/query.rs b/benchmarks/vector-search-bench/src/query.rs new file mode 100644 index 00000000000..cbd9d3bf38e --- /dev/null +++ b/benchmarks/vector-search-bench/src/query.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Sample one query vector from `test.parquet`. +//! +//! The vector datasets ship a `test.parquet` alongside the train split: these are the query vectors +//! meant to be issued against the index. +//! +//! The benchmark picks a single random row (seeded for reproducibility) and uses it as the query +//! for the scan. + +use std::path::Path; + +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use anyhow::ensure; +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::StructArray; +use vortex::array::arrays::struct_::StructArrayExt; +use vortex::dtype::PType; +use vortex::error::VortexExpect; +use vortex::error::vortex_err; +use vortex_bench::conversions::parquet_to_vortex_chunks; + +use crate::SESSION; + +/// One query vector sampled from `test.parquet`. +#[derive(Debug, Clone)] +pub struct QuerySample { + /// The ID of the vector. + pub id: i64, + /// f32 query values, length `dim`. + pub query: Vec, +} + +/// Sample one query row from `test.parquet`. +/// +/// The cast to f32 happens here when the source is f64 (matching the prepare-side cast), so that +/// all downstream code is uniformly f32. +pub async fn get_random_query_vector( + test_parquet: &Path, + expected_dim: u32, + src_ptype: PType, + seed: u64, +) -> Result { + let mut ctx = SESSION.create_execution_ctx(); + + let chunked = parquet_to_vortex_chunks(test_parquet.to_path_buf()) + .await + .with_context(|| format!("read test parquet {}", test_parquet.display()))?; + // The `test.parquet` files are generally small enough that this is not a big deal. + let struct_array: StructArray = chunked.into_array().execute(&mut ctx)?; + + let id = struct_array + .unmasked_field_by_name("id") + .context("test parquet missing `id` column")? + .clone(); + let emb = struct_array + .unmasked_field_by_name("emb") + .context("test parquet missing `emb` column")? + .clone(); + + let mut rng = StdRng::seed_from_u64(seed); + let query_row_idx = rng.random_range(0..id.len()); + + let id_scalar = id.execute_scalar(query_row_idx, &mut ctx)?; + let emb_scalar = emb.execute_scalar(query_row_idx, &mut ctx)?; + + ensure!(emb_scalar.as_list().len() == expected_dim as usize); + + let id = id_scalar + .as_primitive() + .as_::() + .ok_or_else(|| vortex_err!("vector ID was not a i64"))?; + + let query_vector = match src_ptype { + PType::F32 => emb_scalar + .as_list() + .elements() + .vortex_expect("somehow had a null test vector") + .iter() + .map(|element| { + element + .as_primitive() + .as_::() + .vortex_expect("value was not a f32") + }) + .collect(), + PType::F64 => + { + #[expect( + clippy::cast_possible_truncation, + reason = "this is intentionally lossy" + )] + emb_scalar + .as_list() + .elements() + .vortex_expect("somehow had a null test vector") + .iter() + .map(|element| { + element + .as_primitive() + .as_::() + .vortex_expect("value was not a f64") as f32 + }) + .collect() + } + ptype => bail!("source ptype {ptype} was somehow not f32 or f64"), + }; + + Ok(QuerySample { + query: query_vector, + id, + }) +} diff --git a/benchmarks/vector-search-bench/src/scan.rs b/benchmarks/vector-search-bench/src/scan.rs new file mode 100644 index 00000000000..81a054f74b3 --- /dev/null +++ b/benchmarks/vector-search-bench/src/scan.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Per-iteration scan driver. +//! +//! Each iteration re-opens every `.vortex` shard fresh (so the segment cache is re-primed +//! per run), pushes the cosine-similarity filter through the scan, and drains the resulting +//! [`vortex::array::stream::ArrayStream`]. The wall-clock around the entire per-iteration +//! pass is the headline number; we track the mean and median across iterations. + +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Context; +use anyhow::Result; +use futures::TryStreamExt; +use vortex::array::ArrayRef; +use vortex::file::OpenOptionsSessionExt; + +use crate::SESSION; +use crate::compression::VectorFlavor; +use crate::expression::similarity_filter; +use crate::prepare::CompressedVortexDataset; + +/// Inputs to a scan run. +#[derive(Debug, Clone)] +pub struct ScanConfig { + /// Number of timed iterations (best-of-N). + pub iterations: usize, + /// Cosine threshold passed to the filter expression. + pub threshold: f32, +} + +/// Aggregate timing + counters for one `(flavor)` scan. +#[derive(Debug, Clone)] +pub struct ScanTiming { + /// Which compression flavor's `.vortex` files were scanned. + pub flavor: VectorFlavor, + /// Arithmetic mean of the per-iteration wall times. + pub mean: Duration, + /// Median of the per-iteration wall times. + pub median: Duration, + /// Per-iteration wall times in run order. + pub all_runs: Vec, + /// Number of rows that survived the filter (constant across iterations because the + /// filter is deterministic). + pub matches: u64, + /// Total rows scanned (sum of file row counts) as a sanity check that the iteration + /// actually walked the files. + pub rows_scanned: u64, + /// Total on-disk size of the scanned `.vortex` files, in bytes. + pub bytes_scanned: u64, +} + +/// Scan every shard in a [`CompressedVortexDataset`] under the given config. +pub async fn run_search_scan( + dataset: &CompressedVortexDataset, + query: &[f32], + config: &ScanConfig, +) -> Result { + anyhow::ensure!( + config.iterations > 0, + "scan iterations must be >= 1, got {}", + config.iterations + ); + + let bytes_scanned = total_file_size(&dataset.vortex_files)?; + + let mut all_runs = Vec::with_capacity(config.iterations); + let mut matches = 0u64; + let mut rows_scanned = 0u64; + + for iter_idx in 0..config.iterations { + let (wall, iter_matches, iter_rows) = + run_one_iteration(&dataset.vortex_files, query, config.threshold).await?; + tracing::debug!( + "{} iter {} -> {:?} ({} matches, {} rows)", + dataset.flavor.label(), + iter_idx, + wall, + iter_matches, + iter_rows, + ); + // Matches and row counts are deterministic across iterations; reset rather than + // accumulate so the reported value matches a single pass. + matches = iter_matches; + rows_scanned = iter_rows; + all_runs.push(wall); + } + + Ok(ScanTiming { + flavor: dataset.flavor, + mean: mean(&all_runs), + median: median(&all_runs), + all_runs, + matches, + rows_scanned, + bytes_scanned, + }) +} + +/// Sum the on-disk sizes of the given files. +fn total_file_size(paths: &[PathBuf]) -> Result { + let mut total = 0u64; + for path in paths { + let meta = + std::fs::metadata(path).with_context(|| format!("stat {} for size", path.display()))?; + total = total.saturating_add(meta.len()); + } + Ok(total) +} + +async fn run_one_iteration( + vortex_files: &[PathBuf], + query: &[f32], + threshold: f32, +) -> Result<(Duration, u64, u64)> { + let mut matches = 0u64; + let mut rows_scanned = 0u64; + + let started = Instant::now(); + for path in vortex_files { + let (m, r) = scan_one_file(path, query, threshold).await?; + matches = matches.saturating_add(m); + rows_scanned = rows_scanned.saturating_add(r); + } + + Ok((started.elapsed(), matches, rows_scanned)) +} + +async fn scan_one_file(path: &Path, query: &[f32], threshold: f32) -> Result<(u64, u64)> { + let file = SESSION + .open_options() + .open_path(path) + .await + .with_context(|| format!("open {}", path.display()))?; + + let total_rows = file.row_count(); + let filter = similarity_filter(query, threshold)?; + let chunks: Vec = file + .scan()? + .with_filter(filter) + .into_array_stream()? + .try_collect() + .await?; + + let matches: u64 = chunks.iter().map(|c| c.len() as u64).sum(); + Ok((matches, total_rows)) +} + +/// Arithmetic mean of a list of [`Duration`]s. Empty lists return [`Duration::ZERO`]. +pub fn mean(runs: &[Duration]) -> Duration { + if runs.is_empty() { + return Duration::ZERO; + } + let total_nanos: u128 = runs.iter().map(|d| d.as_nanos()).sum(); + let avg_nanos = total_nanos / runs.len() as u128; + Duration::from_nanos(u64::try_from(avg_nanos).unwrap_or(u64::MAX)) +} + +/// Median of a list of [`Duration`]s. Empty lists return [`Duration::ZERO`]. +pub fn median(runs: &[Duration]) -> Duration { + if runs.is_empty() { + return Duration::ZERO; + } + let mut sorted = runs.to_vec(); + sorted.sort(); + let mid = sorted.len() / 2; + if sorted.len() % 2 == 1 { + sorted[mid] + } else { + let total_nanos = sorted[mid - 1].as_nanos() + sorted[mid].as_nanos(); + Duration::from_nanos(u64::try_from(total_nanos / 2).unwrap_or(u64::MAX)) + } +} From 413520987c52ad731325879eb279272aa3450b95 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:23:57 -0400 Subject: [PATCH 109/250] Fix reduction/execute cycle (#7522) ## Summary Fixes the regression caused by https://github.com/vortex-data/vortex/pull/7507 The reduction rule would go from `Extension(Constant(..))` into `Constant(Extension(..))`, but the `execute` would just revert that, creating a cycle! ## Testing N/A Signed-off-by: Connor Tsui --- vortex-array/public-api.lock | 12 ++--- .../src/arrays/constant/vtable/canonical.rs | 17 ++++++- .../src/arrays/constant/vtable/mod.rs | 47 ++++++++----------- .../src/arrays/extension/vtable/canonical.rs | 2 - .../src/arrays/extension/vtable/mod.rs | 8 ++-- 5 files changed, 44 insertions(+), 42 deletions(-) delete mode 100644 vortex-array/src/arrays/extension/vtable/canonical.rs diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index b388e7e3a21..8ae349c3da5 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -1494,7 +1494,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -5312,7 +5312,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -19404,7 +19404,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -20376,7 +20376,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -23070,7 +23070,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -24290,7 +24290,7 @@ pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 8d4955c2261..7d004ceb765 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -10,6 +10,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::Canonical; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::BoolArray; @@ -36,7 +37,10 @@ use crate::scalar::Scalar; use crate::validity::Validity; /// Shared implementation for both `canonicalize` and `execute` methods. -pub(crate) fn constant_canonicalize(array: ArrayView<'_, Constant>) -> VortexResult { +pub(crate) fn constant_canonicalize( + array: ArrayView<'_, Constant>, + ctx: &mut ExecutionCtx, +) -> VortexResult { let scalar = array.scalar(); let validity = match array.dtype().nullability() { @@ -163,7 +167,16 @@ pub(crate) fn constant_canonicalize(array: ArrayView<'_, Constant>) -> VortexRes let s = scalar.as_extension(); let storage_scalar = s.to_storage_scalar(); - let storage_self = ConstantArray::new(storage_scalar, array.len()).into_array(); + + // NB: We need to execute the constant array to be canonical because there is a + // reduction rule that turns `Extension(Constant(..))` into `Constant(Extension(..))`, + // and if we don't do this we create an infinite cycle. + // See `ExtensionConstantRule` for more details. + let storage_self = ConstantArray::new(storage_scalar, array.len()) + .into_array() + .execute::(ctx)? + .into_array(); + Canonical::Extension(ExtensionArray::new(ext_dtype.clone(), storage_self)) } DType::Variant(_) => { diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 9b4f9dbb953..7384505d854 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -160,9 +160,10 @@ impl VTable for Constant { PARENT_RULES.evaluate(array, parent, child_idx) } - fn execute(array: Array, _ctx: &mut ExecutionCtx) -> VortexResult { + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { Ok(ExecutionResult::done(constant_canonicalize( array.as_view(), + ctx, )?)) } @@ -268,10 +269,11 @@ fn append_value_or_nulls( #[cfg(test)] mod tests { use rstest::rstest; + use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::ExecutionCtx; use crate::IntoArray; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::constant::vtable::canonical::constant_canonicalize; use crate::assert_arrays_eq; @@ -282,34 +284,29 @@ mod tests { use crate::dtype::StructFields; use crate::scalar::Scalar; - fn ctx() -> ExecutionCtx { - ExecutionCtx::new(VortexSession::empty()) - } - /// Appends `array` into a fresh builder and asserts the result matches `constant_canonicalize`. - fn assert_append_matches_canonical(array: ConstantArray) -> vortex_error::VortexResult<()> { - let expected = constant_canonicalize(array.as_view())?.into_array(); + fn assert_append_matches_canonical(array: ConstantArray) -> VortexResult<()> { + let mut ctx = VortexSession::empty().create_execution_ctx(); + + let expected = constant_canonicalize(array.as_view(), &mut ctx)?.into_array(); let mut builder = builder_with_capacity(array.dtype(), array.len()); array .into_array() - .append_to_builder(builder.as_mut(), &mut ctx())?; + .append_to_builder(builder.as_mut(), &mut ctx)?; let result = builder.finish(); assert_arrays_eq!(&result, &expected); Ok(()) } #[test] - fn test_null_constant_append() -> vortex_error::VortexResult<()> { + fn test_null_constant_append() -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new(Scalar::null(DType::Null), 5)) } #[rstest] #[case::bool_true(true, 5)] #[case::bool_false(false, 3)] - fn test_bool_constant_append( - #[case] value: bool, - #[case] n: usize, - ) -> vortex_error::VortexResult<()> { + fn test_bool_constant_append(#[case] value: bool, #[case] n: usize) -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::bool(value, Nullability::NonNullable), n, @@ -317,7 +314,7 @@ mod tests { } #[test] - fn test_bool_null_constant_append() -> vortex_error::VortexResult<()> { + fn test_bool_null_constant_append() -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::null(DType::Bool(Nullability::Nullable)), 4, @@ -332,7 +329,7 @@ mod tests { fn test_primitive_constant_append( #[case] scalar: Scalar, #[case] n: usize, - ) -> vortex_error::VortexResult<()> { + ) -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new(scalar, n)) } @@ -341,10 +338,7 @@ mod tests { #[case::utf8_noninline("hello world!!", 5)] // >12 bytes: requires buffer block #[case::utf8_empty("", 3)] #[case::utf8_n_zero("hello world!!", 0)] // n=0 with non-inline: must not write orphaned bytes - fn test_utf8_constant_append( - #[case] value: &str, - #[case] n: usize, - ) -> vortex_error::VortexResult<()> { + fn test_utf8_constant_append(#[case] value: &str, #[case] n: usize) -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::utf8(value, Nullability::NonNullable), n, @@ -352,7 +346,7 @@ mod tests { } #[test] - fn test_utf8_null_constant_append() -> vortex_error::VortexResult<()> { + fn test_utf8_null_constant_append() -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::null(DType::Utf8(Nullability::Nullable)), 4, @@ -362,10 +356,7 @@ mod tests { #[rstest] #[case::binary_inline(vec![1u8, 2, 3], 5)] // ≤12 bytes: inlined #[case::binary_noninline(vec![0u8; 13], 5)] // >12 bytes: buffer block - fn test_binary_constant_append( - #[case] value: Vec, - #[case] n: usize, - ) -> vortex_error::VortexResult<()> { + fn test_binary_constant_append(#[case] value: Vec, #[case] n: usize) -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::binary(value, Nullability::NonNullable), n, @@ -373,7 +364,7 @@ mod tests { } #[test] - fn test_binary_null_constant_append() -> vortex_error::VortexResult<()> { + fn test_binary_null_constant_append() -> VortexResult<()> { assert_append_matches_canonical(ConstantArray::new( Scalar::null(DType::Binary(Nullability::Nullable)), 4, @@ -381,7 +372,7 @@ mod tests { } #[test] - fn test_struct_constant_append() -> vortex_error::VortexResult<()> { + fn test_struct_constant_append() -> VortexResult<()> { let fields = StructFields::new( ["x", "y"].into(), vec![ @@ -400,7 +391,7 @@ mod tests { } #[test] - fn test_null_struct_constant_append() -> vortex_error::VortexResult<()> { + fn test_null_struct_constant_append() -> VortexResult<()> { let fields = StructFields::new( ["x"].into(), vec![DType::Primitive(PType::I32, Nullability::Nullable)], diff --git a/vortex-array/src/arrays/extension/vtable/canonical.rs b/vortex-array/src/arrays/extension/vtable/canonical.rs deleted file mode 100644 index 0d735177e5d..00000000000 --- a/vortex-array/src/arrays/extension/vtable/canonical.rs +++ /dev/null @@ -1,2 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index a3fde75f79b..82ad85e701f 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -1,9 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -mod canonical; -mod kernel; -mod operations; -mod validity; use std::hash::Hasher; @@ -36,6 +32,10 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; +mod kernel; +mod operations; +mod validity; + #[derive(Clone, Debug)] pub struct Extension; From ac74f1e917e7c95bd09cf17ceb18d782579a8dda Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:31:58 +0000 Subject: [PATCH 110/250] Update dependency @tanstack/react-virtual to v3.13.24 (#7529) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [@tanstack/react-virtual](https://tanstack.com/virtual) ([source](https://redirect.github.com/TanStack/virtual/tree/HEAD/packages/react-virtual)) | [`3.13.23` → `3.13.24`](https://renovatebot.com/diffs/npm/@tanstack%2freact-virtual/3.13.23/3.13.24) | ![age](https://developer.mend.io/api/mc/badges/age/npm/@tanstack%2freact-virtual/3.13.24?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/@tanstack%2freact-virtual/3.13.23/3.13.24?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
TanStack/virtual (@​tanstack/react-virtual) ### [`v3.13.24`](https://redirect.github.com/TanStack/virtual/blob/HEAD/packages/react-virtual/CHANGELOG.md#31324) [Compare Source](https://redirect.github.com/TanStack/virtual/compare/@tanstack/react-virtual@3.13.23...@tanstack/react-virtual@3.13.24) ##### Patch Changes - Updated dependencies \[[`97a204d`](https://redirect.github.com/TanStack/virtual/commit/97a204dc5526669114458685552b7569b60d2940)]: - [@​tanstack/virtual-core](https://redirect.github.com/tanstack/virtual-core)@​3.14.0
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 0bd6691a913..c8b9823cad5 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -1990,12 +1990,12 @@ } }, "node_modules/@tanstack/react-virtual": { - "version": "3.13.23", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.23.tgz", - "integrity": "sha512-XnMRnHQ23piOVj2bzJqHrRrLg4r+F86fuBcwteKfbIjJrtGxb4z7tIvPVAe4B+4UVwo9G4Giuz5fmapcrnZ0OQ==", + "version": "3.13.24", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.24.tgz", + "integrity": "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg==", "license": "MIT", "dependencies": { - "@tanstack/virtual-core": "3.13.23" + "@tanstack/virtual-core": "3.14.0" }, "funding": { "type": "github", @@ -2020,9 +2020,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.13.23", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.13.23.tgz", - "integrity": "sha512-zSz2Z2HNyLjCplANTDyl3BcdQJc2k1+yyFoKhNRmCr7V7dY8o8q5m8uFTI1/Pg1kL+Hgrz6u3Xo6eFUB7l66cg==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.14.0.tgz", + "integrity": "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q==", "license": "MIT", "funding": { "type": "github", From e42c135024954f89cb13b0009902b8e82c5c6d10 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:32:39 +0000 Subject: [PATCH 111/250] Update storybook monorepo to v10.3.5 (#7530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [@storybook/addon-docs](https://redirect.github.com/storybookjs/storybook/tree/next/code/addons/docs) ([source](https://redirect.github.com/storybookjs/storybook/tree/HEAD/code/addons/docs)) | [`10.3.3` → `10.3.5`](https://renovatebot.com/diffs/npm/@storybook%2faddon-docs/10.3.3/10.3.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/@storybook%2faddon-docs/10.3.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/@storybook%2faddon-docs/10.3.3/10.3.5?slim=true) | | [@storybook/react-vite](https://redirect.github.com/storybookjs/storybook/tree/next/code/frameworks/react-vite) ([source](https://redirect.github.com/storybookjs/storybook/tree/HEAD/code/frameworks/react-vite)) | [`10.3.3` → `10.3.5`](https://renovatebot.com/diffs/npm/@storybook%2freact-vite/10.3.3/10.3.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/@storybook%2freact-vite/10.3.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/@storybook%2freact-vite/10.3.3/10.3.5?slim=true) | | [eslint-plugin-storybook](https://redirect.github.com/storybookjs/storybook/tree/next/code/lib/eslint-plugin#readme) ([source](https://redirect.github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin)) | [`10.3.3` → `10.3.5`](https://renovatebot.com/diffs/npm/eslint-plugin-storybook/10.3.3/10.3.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/eslint-plugin-storybook/10.3.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/eslint-plugin-storybook/10.3.3/10.3.5?slim=true) | | [storybook](https://storybook.js.org) ([source](https://redirect.github.com/storybookjs/storybook/tree/HEAD/code/core)) | [`10.3.3` → `10.3.5`](https://renovatebot.com/diffs/npm/storybook/10.3.3/10.3.5) | ![age](https://developer.mend.io/api/mc/badges/age/npm/storybook/10.3.5?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/storybook/10.3.3/10.3.5?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
storybookjs/storybook (@​storybook/addon-docs) ### [`v10.3.5`](https://redirect.github.com/storybookjs/storybook/blob/HEAD/CHANGELOG.md#1035) [Compare Source](https://redirect.github.com/storybookjs/storybook/compare/v10.3.4...v10.3.5) - Core: Disable component manifest by default - [#​34408](https://redirect.github.com/storybookjs/storybook/pull/34408), thanks [@​yannbf](https://redirect.github.com/yannbf)! > \[!NOTE] > [Version >=0.5.0 of `@storybook/addon-mcp`](https://redirect.github.com/storybookjs/mcp/releases/tag/%40storybook%2Faddon-mcp%400.5.0) enables component manifests again. If you're upgrading Storybook from version >= 10.3.0 to >= 10.3.5 and are using the MCP addon, you should also upgrade `@storybook/addon-mcp` to keep the docs toolset in the MCP server. ### [`v10.3.4`](https://redirect.github.com/storybookjs/storybook/blob/HEAD/CHANGELOG.md#1034) [Compare Source](https://redirect.github.com/storybookjs/storybook/compare/v10.3.3...v10.3.4) - Addon-a11y: Clear status transition timer on unmount to prevent test flake - [#​34203](https://redirect.github.com/storybookjs/storybook/pull/34203), thanks [@​mixelburg](https://redirect.github.com/mixelburg)! - Bug: Skip re-processing already transformed config files for CSF factories - [#​34273](https://redirect.github.com/storybookjs/storybook/pull/34273), thanks [@​huang-julien](https://redirect.github.com/huang-julien)! - Builder-Vite: Use djb2 hash to prevent variable name collisions in builder-vite - [#​34274](https://redirect.github.com/storybookjs/storybook/pull/34274), thanks [@​chida09](https://redirect.github.com/chida09)! - CLI: Prompt for init crash reports - [#​34316](https://redirect.github.com/storybookjs/storybook/pull/34316), thanks [@​JReinhold](https://redirect.github.com/JReinhold)! - CSF4: Fix duplicate preview loading issue in Vitest - [#​34361](https://redirect.github.com/storybookjs/storybook/pull/34361), thanks [@​valentinpalkovic](https://redirect.github.com/valentinpalkovic)! - Core: Fix WebSocket connection for StackBlitz/WebContainers - [#​34281](https://redirect.github.com/storybookjs/storybook/pull/34281), thanks [@​ghengeveld](https://redirect.github.com/ghengeveld)! - React-Docgen: Try .tsx fallback when resolving .js ESM imports in docgen resolvers - [#​34393](https://redirect.github.com/storybookjs/storybook/pull/34393), thanks [@​mixelburg](https://redirect.github.com/mixelburg)! - React-Vite: Upgrade [@​joshwooding/vite-plugin-react-docgen-typescript](https://redirect.github.com/joshwooding/vite-plugin-react-docgen-typescript) to 0.7.0 - [#​34335](https://redirect.github.com/storybookjs/storybook/pull/34335), thanks [@​beeswhacks](https://redirect.github.com/beeswhacks)!
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 131 +++++++++++++++++------------------ 1 file changed, 63 insertions(+), 68 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index c8b9823cad5..dd0d7e06c92 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -990,9 +990,9 @@ } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript": { - "version": "0.6.4", - "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.6.4.tgz", - "integrity": "sha512-6PyZBYKnnVNqOSB0YFly+62R7dmov8segT27A+RVTBVd4iAE6kbW9QBJGlyR2yG4D4ohzhZSTIu7BK1UTtmFFA==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.7.0.tgz", + "integrity": "sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1001,7 +1001,7 @@ }, "peerDependencies": { "typescript": ">= 4.3.x", - "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "typescript": { @@ -1458,16 +1458,16 @@ ] }, "node_modules/@storybook/addon-docs": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/addon-docs/-/addon-docs-10.3.3.tgz", - "integrity": "sha512-trJQTpOtuOEuNv1Rn8X2Sopp5hSPpb0u0soEJ71BZAbxe4d2Y1d/1MYcxBdRKwncum6sCTsnxTpqQ/qvSJKlTQ==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/addon-docs/-/addon-docs-10.3.5.tgz", + "integrity": "sha512-WuHbxia/o5TX4Rg/IFD0641K5qId/Nk0dxhmAUNoFs5L0+yfZUwh65XOBbzXqrkYmYmcVID4v7cgDRmzstQNkA==", "dev": true, "license": "MIT", "dependencies": { "@mdx-js/react": "^3.0.0", - "@storybook/csf-plugin": "10.3.3", + "@storybook/csf-plugin": "10.3.5", "@storybook/icons": "^2.0.1", - "@storybook/react-dom-shim": "10.3.3", + "@storybook/react-dom-shim": "10.3.5", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "ts-dedent": "^2.0.0" @@ -1477,17 +1477,17 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^10.3.3" + "storybook": "^10.3.5" } }, "node_modules/@storybook/builder-vite": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-10.3.3.tgz", - "integrity": "sha512-awspKCTZvXyeV3KabL0id62mFbxR5u/5yyGQultwCiSb2/yVgBfip2MAqLyS850pvTiB6QFVM9deOyd2/G/bEA==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-10.3.5.tgz", + "integrity": "sha512-i4KwCOKbhtlbQIbhm53+Kk7bMnxa0cwTn1pxmtA/x5wm1Qu7FrrBQV0V0DNjkUqzcSKo1CjspASJV/HlY0zYlw==", "dev": true, "license": "MIT", "dependencies": { - "@storybook/csf-plugin": "10.3.3", + "@storybook/csf-plugin": "10.3.5", "ts-dedent": "^2.0.0" }, "funding": { @@ -1495,14 +1495,14 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^10.3.3", + "storybook": "^10.3.5", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/@storybook/csf-plugin": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/csf-plugin/-/csf-plugin-10.3.3.tgz", - "integrity": "sha512-Utlh7zubm+4iOzBBfzLW4F4vD99UBtl2Do4edlzK2F7krQIcFvR2ontjAE8S1FQVLZAC3WHalCOS+Ch8zf3knA==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/csf-plugin/-/csf-plugin-10.3.5.tgz", + "integrity": "sha512-qlEzNKxOjq86pvrbuMwiGD/bylnsXk1dg7ve0j77YFjEEchqtl7qTlrXvFdNaLA89GhW6D/EV6eOCu/eobPDgw==", "dev": true, "license": "MIT", "dependencies": { @@ -1515,7 +1515,7 @@ "peerDependencies": { "esbuild": "*", "rollup": "*", - "storybook": "^10.3.3", + "storybook": "^10.3.5", "vite": "*", "webpack": "*" }, @@ -1553,14 +1553,14 @@ } }, "node_modules/@storybook/react": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/react/-/react-10.3.3.tgz", - "integrity": "sha512-cGG5TbR8Tdx9zwlpsWyBEfWrejm5iWdYF26EwIhwuKq9GFUTAVrQzo0Rs7Tqc3ZyVhRS/YfsRiWSEH+zmq2JiQ==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/react/-/react-10.3.5.tgz", + "integrity": "sha512-tpLTLaVGoA6fLK3ReyGzZUricq7lyPaV2hLPpj5wqdXLV/LpRtAHClUpNoPDYSBjlnSjL81hMZijbkGC3mA+gw==", "dev": true, "license": "MIT", "dependencies": { "@storybook/global": "^5.0.0", - "@storybook/react-dom-shim": "10.3.3", + "@storybook/react-dom-shim": "10.3.5", "react-docgen": "^8.0.2", "react-docgen-typescript": "^2.2.2" }, @@ -1571,7 +1571,7 @@ "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.3.3", + "storybook": "^10.3.5", "typescript": ">= 4.9.x" }, "peerDependenciesMeta": { @@ -1581,9 +1581,9 @@ } }, "node_modules/@storybook/react-dom-shim": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/react-dom-shim/-/react-dom-shim-10.3.3.tgz", - "integrity": "sha512-lkhuh4G3UTreU9M3Iz5Dt32c6U+l/4XuvqLtbe1sDHENZH6aPj7y0b5FwnfHyvuTvYRhtbo29xZrF5Bp9kCC0w==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/react-dom-shim/-/react-dom-shim-10.3.5.tgz", + "integrity": "sha512-Gw8R7XZm0zSUH0XAuxlQJhmizsLzyD6x00KOlP6l7oW9eQHXGfxg3seNDG3WrSAcW07iP1/P422kuiriQlOv7g==", "dev": true, "license": "MIT", "funding": { @@ -1593,20 +1593,20 @@ "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.3.3" + "storybook": "^10.3.5" } }, "node_modules/@storybook/react-vite": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/@storybook/react-vite/-/react-vite-10.3.3.tgz", - "integrity": "sha512-qHdlBe1hjqFAGXa8JL7bWTLbP/gDqXbWDm+SYCB646NHh5yvVDkZLwigP5Y+UL7M2ASfqFtosnroUK9tcCM2dw==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/@storybook/react-vite/-/react-vite-10.3.5.tgz", + "integrity": "sha512-UB5sJHeh26bfd8sNMx2YPGYRYmErIdTRaLOT28m4bykQIa1l9IgVktsYg/geW7KsJU0lXd3oTbnUjLD+enpi3w==", "dev": true, "license": "MIT", "dependencies": { - "@joshwooding/vite-plugin-react-docgen-typescript": "^0.6.4", + "@joshwooding/vite-plugin-react-docgen-typescript": "^0.7.0", "@rollup/pluginutils": "^5.0.2", - "@storybook/builder-vite": "10.3.3", - "@storybook/react": "10.3.3", + "@storybook/builder-vite": "10.3.5", + "@storybook/react": "10.3.5", "empathic": "^2.0.0", "magic-string": "^0.30.0", "react-docgen": "^8.0.0", @@ -1620,7 +1620,7 @@ "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.3.3", + "storybook": "^10.3.5", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, @@ -2591,19 +2591,6 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/@vitest/spy": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyspy": "^4.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/@vitest/expect/node_modules/@vitest/utils": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", @@ -2629,6 +2616,26 @@ "node": ">=14.0.0" } }, + "node_modules/@vitest/spy": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@webcontainer/env": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@webcontainer/env/-/env-1.1.1.tgz", + "integrity": "sha512-6aN99yL695Hi9SuIk1oC88l9o0gmxL1nGWWQ/kNy81HigJ0FoaoTXpytCj6ItzgyCEwA9kF1wixsTuv5cjsgng==", + "dev": true, + "license": "MIT" + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -3357,9 +3364,9 @@ } }, "node_modules/eslint-plugin-storybook": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-10.3.3.tgz", - "integrity": "sha512-jo8wZvKaJlxxrNvf4hCsROJP3CdlpaLiYewAs5Ww+PJxCrLelIi5XVHWOAgBvvr3H9WDKvUw8xuvqPYqAlpkFg==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-storybook/-/eslint-plugin-storybook-10.3.5.tgz", + "integrity": "sha512-rEFkfU3ypF44GpB4tiJ9EFDItueoGvGi3+weLHZax2ON2MB7VIDsxdSUGvIU5tMURg+oWYlpzCyLm4TpDq2deA==", "dev": true, "license": "MIT", "dependencies": { @@ -3367,7 +3374,7 @@ }, "peerDependencies": { "eslint": ">=8", - "storybook": "^10.3.3" + "storybook": "^10.3.5" } }, "node_modules/eslint-scope": { @@ -4934,9 +4941,9 @@ } }, "node_modules/storybook": { - "version": "10.3.3", - "resolved": "https://registry.npmjs.org/storybook/-/storybook-10.3.3.tgz", - "integrity": "sha512-tMoRAts9EVqf+mEMPLC6z1DPyHbcPe+CV1MhLN55IKsl0HxNjvVGK44rVPSePbltPE6vIsn4bdRj6CCUt8SJwQ==", + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/storybook/-/storybook-10.3.5.tgz", + "integrity": "sha512-uBSZu/GZa9aEIW3QMGvdQPMZWhGxSe4dyRWU8B3/Vd47Gy/XLC7tsBxRr13txmmPOEDHZR94uLuq0H50fvuqBw==", "dev": true, "license": "MIT", "dependencies": { @@ -4946,6 +4953,7 @@ "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", + "@webcontainer/env": "^1.1.1", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0", "open": "^10.2.0", "recast": "^0.23.5", @@ -4969,19 +4977,6 @@ } } }, - "node_modules/storybook/node_modules/@vitest/spy": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "tinyspy": "^4.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, "node_modules/storybook/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", From 7f552b165f5d3fba918bdd7cfd869bc1a6494711 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:33:37 +0000 Subject: [PATCH 112/250] Update crate-ci/typos action to v1.45.1 (#7532) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [crate-ci/typos](https://redirect.github.com/crate-ci/typos) | action | minor | `v1.44.0` → `v1.45.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
crate-ci/typos (crate-ci/typos) ### [`v1.45.1`](https://redirect.github.com/crate-ci/typos/blob/HEAD/CHANGELOG.md#014---2019-11-03) [Compare Source](https://redirect.github.com/crate-ci/typos/compare/v1.45.0...v1.45.1) ##### Bug Fixes - Ignore numbers as identifiers ([a00831c8](https://redirect.github.com/crate-ci/typos/commit/a00831c847b7efd81be520ea9b5d02f70555351f)) - Improve the organization of --help ([a48a457c](https://redirect.github.com/crate-ci/typos/commit/a48a457cc3ca817850118e2a2fb8b20fecdd40b8)) ##### Features - Dump files, identifiers, and words ([ce365ae1](https://redirect.github.com/crate-ci/typos/commit/ce365ae12e12fddfb6fc42a7f1e5ea71834d6051), closes [#​41](https://redirect.github.com/crate-ci/typos/issues/41)) - Give control over allowed identifier characters for leading vs rest ([107308a6](https://redirect.github.com/crate-ci/typos/commit/107308a655a425eb593bf5e4928572c16e6a9bdd)) ##### Performance - Use standard identifier rules to avoid doing umber checks ([107308a6](https://redirect.github.com/crate-ci/typos/commit/107308a655a425eb593bf5e4928572c16e6a9bdd)) - Only do hex check if digits are in identifiers ([68cd36d0](https://redirect.github.com/crate-ci/typos/commit/68cd36d0de90226dbc9d31c2ce6d8bf6b69adb5c)) [Unreleased]: https://redirect.github.com/crate-ci/typos/compare/v1.45.1...HEAD [1.45.1]: https://redirect.github.com/crate-ci/typos/compare/v1.45.0...v1.45.1 [1.45.0]: https://redirect.github.com/crate-ci/typos/compare/v1.44.0...v1.45.0 [1.44.0]: https://redirect.github.com/crate-ci/typos/compare/v1.43.5...v1.44.0 [1.43.5]: https://redirect.github.com/crate-ci/typos/compare/v1.43.4...v1.43.5 [1.43.4]: https://redirect.github.com/crate-ci/typos/compare/v1.43.3...v1.43.4 [1.43.3]: https://redirect.github.com/crate-ci/typos/compare/v1.43.2...v1.43.3 [1.43.2]: https://redirect.github.com/crate-ci/typos/compare/v1.43.1...v1.43.2 [1.43.1]: https://redirect.github.com/crate-ci/typos/compare/v1.43.0...v1.43.1 [1.43.0]: https://redirect.github.com/crate-ci/typos/compare/v1.42.3...v1.43.0 [1.42.3]: https://redirect.github.com/crate-ci/typos/compare/v1.42.2...v1.42.3 [1.42.2]: https://redirect.github.com/crate-ci/typos/compare/v1.42.1...v1.42.2 [1.42.1]: https://redirect.github.com/crate-ci/typos/compare/v1.42.0...v1.42.1 [1.42.0]: https://redirect.github.com/crate-ci/typos/compare/v1.41.0...v1.42.0 [1.41.0]: https://redirect.github.com/crate-ci/typos/compare/v1.40.1...v1.41.0 [1.40.1]: https://redirect.github.com/crate-ci/typos/compare/v1.40.0...v1.40.1 [1.40.0]: https://redirect.github.com/crate-ci/typos/compare/v1.39.2...v1.40.0 [1.39.2]: https://redirect.github.com/crate-ci/typos/compare/v1.39.1...v1.39.2 [1.39.1]: https://redirect.github.com/crate-ci/typos/compare/v1.39.0...v1.39.1 [1.39.0]: https://redirect.github.com/crate-ci/typos/compare/v1.38.1...v1.39.0 [1.38.1]: https://redirect.github.com/crate-ci/typos/compare/v1.38.0...v1.38.1 [1.38.0]: https://redirect.github.com/crate-ci/typos/compare/v1.37.3...v1.38.0 [1.37.3]: https://redirect.github.com/crate-ci/typos/compare/v1.37.2...v1.37.3 [1.37.2]: https://redirect.github.com/crate-ci/typos/compare/v1.37.1...v1.37.2 [1.37.1]: https://redirect.github.com/crate-ci/typos/compare/v1.37.0...v1.37.1 [1.37.0]: https://redirect.github.com/crate-ci/typos/compare/v1.36.3...v1.37.0 [1.36.3]: https://redirect.github.com/crate-ci/typos/compare/v1.36.2...v1.36.3 [1.36.2]: https://redirect.github.com/crate-ci/typos/compare/v1.36.1...v1.36.2 [1.36.1]: https://redirect.github.com/crate-ci/typos/compare/v1.36.0...v1.36.1 [1.36.0]: https://redirect.github.com/crate-ci/typos/compare/v1.35.8...v1.36.0 [1.35.8]: https://redirect.github.com/crate-ci/typos/compare/v1.35.7...v1.35.8 [1.35.7]: https://redirect.github.com/crate-ci/typos/compare/v1.35.6...v1.35.7 [1.35.6]: https://redirect.github.com/crate-ci/typos/compare/v1.35.5...v1.35.6 [1.35.5]: https://redirect.github.com/crate-ci/typos/compare/v1.35.4...v1.35.5 [1.35.4]: https://redirect.github.com/crate-ci/typos/compare/v1.35.3...v1.35.4 [1.35.3]: https://redirect.github.com/crate-ci/typos/compare/v1.35.2...v1.35.3 [1.35.2]: https://redirect.github.com/crate-ci/typos/compare/v1.35.1...v1.35.2 [1.35.1]: https://redirect.github.com/crate-ci/typos/compare/v1.35.0...v1.35.1 [1.35.0]: https://redirect.github.com/crate-ci/typos/compare/v1.34.0...v1.35.0 [1.34.0]: https://redirect.github.com/crate-ci/typos/compare/v1.33.1...v1.34.0 [1.33.1]: https://redirect.github.com/crate-ci/typos/compare/v1.33.0...v1.33.1 [1.33.0]: https://redirect.github.com/crate-ci/typos/compare/v1.32.0...v1.33.0 [1.32.0]: https://redirect.github.com/crate-ci/typos/compare/v1.31.2...v1.32.0 [1.31.2]: https://redirect.github.com/crate-ci/typos/compare/v1.31.1...v1.31.2 [1.31.1]: https://redirect.github.com/crate-ci/typos/compare/v1.31.0...v1.31.1 [1.31.0]: https://redirect.github.com/crate-ci/typos/compare/v1.30.3...v1.31.0 [1.30.3]: https://redirect.github.com/crate-ci/typos/compare/v1.30.2...v1.30.3 [1.30.2]: https://redirect.github.com/crate-ci/typos/compare/v1.30.1...v1.30.2 [1.30.1]: https://redirect.github.com/crate-ci/typos/compare/v1.30.0...v1.30.1 [1.30.0]: https://redirect.github.com/crate-ci/typos/compare/v1.29.10...v1.30.0 [1.29.10]: https://redirect.github.com/crate-ci/typos/compare/v1.29.9...v1.29.10 [1.29.9]: https://redirect.github.com/crate-ci/typos/compare/v1.29.8...v1.29.9 [1.29.8]: https://redirect.github.com/crate-ci/typos/compare/v1.29.7...v1.29.8 [1.29.7]: https://redirect.github.com/crate-ci/typos/compare/v1.29.6...v1.29.7 [1.29.6]: https://redirect.github.com/crate-ci/typos/compare/v1.29.5...v1.29.6 [1.29.5]: https://redirect.github.com/crate-ci/typos/compare/v1.29.4...v1.29.5 [1.29.4]: https://redirect.github.com/crate-ci/typos/compare/v1.29.3...v1.29.4 [1.29.3]: https://redirect.github.com/crate-ci/typos/compare/v1.29.2...v1.29.3 [1.29.2]: https://redirect.github.com/crate-ci/typos/compare/v1.29.1...v1.29.2 [1.29.1]: https://redirect.github.com/crate-ci/typos/compare/v1.29.0...v1.29.1 [1.29.0]: https://redirect.github.com/crate-ci/typos/compare/v1.28.4...v1.29.0 [1.28.4]: https://redirect.github.com/crate-ci/typos/compare/v1.28.3...v1.28.4 [1.28.3]: https://redirect.github.com/crate-ci/typos/compare/v1.28.2...v1.28.3 [1.28.2]: https://redirect.github.com/crate-ci/typos/compare/v1.28.1...v1.28.2 [1.28.1]: https://redirect.github.com/crate-ci/typos/compare/v1.28.0...v1.28.1 [1.28.0]: https://redirect.github.com/crate-ci/typos/compare/v1.27.3...v1.28.0 [1.27.3]: https://redirect.github.com/crate-ci/typos/compare/v1.27.2...v1.27.3 [1.27.2]: https://redirect.github.com/crate-ci/typos/compare/v1.27.1...v1.27.2 [1.27.1]: https://redirect.github.com/crate-ci/typos/compare/v1.27.0...v1.27.1 [1.27.0]: https://redirect.github.com/crate-ci/typos/compare/v1.26.8...v1.27.0 [1.26.8]: https://redirect.github.com/crate-ci/typos/compare/v1.26.7...v1.26.8 [1.26.7]: https://redirect.github.com/crate-ci/typos/compare/v1.26.6...v1.26.7 [1.26.6]: https://redirect.github.com/crate-ci/typos/compare/v1.26.5...v1.26.6 [1.26.5]: https://redirect.github.com/crate-ci/typos/compare/v1.26.4...v1.26.5 [1.26.4]: https://redirect.github.com/crate-ci/typos/compare/v1.26.3...v1.26.4 [1.26.3]: https://redirect.github.com/crate-ci/typos/compare/v1.26.2...v1.26.3 [1.26.2]: https://redirect.github.com/crate-ci/typos/compare/v1.26.1...v1.26.2 [1.26.1]: https://redirect.github.com/crate-ci/typos/compare/v1.26.0...v1.26.1 [1.26.0]: https://redirect.github.com/crate-ci/typos/compare/v1.25.0...v1.26.0 [1.25.0]: https://redirect.github.com/crate-ci/typos/compare/v1.24.6...v1.25.0 [1.24.6]: https://redirect.github.com/crate-ci/typos/compare/v1.24.5...v1.24.6 [1.24.5]: https://redirect.github.com/crate-ci/typos/compare/v1.24.4...v1.24.5 [1.24.4]: https://redirect.github.com/crate-ci/typos/compare/v1.24.3...v1.24.4 [1.24.3]: https://redirect.github.com/crate-ci/typos/compare/v1.24.2...v1.24.3 [1.24.2]: https://redirect.github.com/crate-ci/typos/compare/v1.24.1...v1.24.2 [1.24.1]: https://redirect.github.com/crate-ci/typos/compare/v1.24.0...v1.24.1 [1.24.0]: https://redirect.github.com/crate-ci/typos/compare/v1.23.7...v1.24.0 [1.23.7]: https://redirect.github.com/crate-ci/typos/compare/v1.23.6...v1.23.7 [1.23.6]: https://redirect.github.com/crate-ci/typos/compare/v1.23.5...v1.23.6 [1.23.5]: https://redirect.github.com/crate-ci/typos/compare/v1.23.4...v1.23.5 [1.23.4]: https://redirect.github.com/crate-ci/typos/compare/v1.23.3...v1.23.4 [1.23.3]: https://redirect.github.com/crate-ci/typos/compare/v1.23.2...v1.23.3 [1.23.2]: https://redirect.github.com/crate-ci/typos/compare/v1.23.1...v1.23.2 [1.23.1]: https://redirect.github.com/crate-ci/typos/compare/v1.23.0...v1.23.1 [1.23.0]: https://redirect.github.com/crate-ci/typos/compare/v1.22.9...v1.23.0 [1.22.9]: https://redirect.github.com/crate-ci/typos/compare/v1.22.8...v1.22.9 [1.22.8]: https://redirect.github.com/crate-ci/typos/compare/v1.22.7...v1.22.8 [1.22.7]: https://redirect.github.com/crate-ci/typos/compare/v1.22.6...v1.22.7 [1.22.6]: https://redirect.github.com/crate-ci/typos/compare/v1.22.5...v1.22.6 [1.22.5]: https://redirect.github.com/crate-ci/typos/compare/v1.22.4...v1.22.5 [1.22.4]: https://redirect.github.com/crate-ci/typos/compare/v1.22.3...v1.22.4 [1.22.3]: https://redirect.github.com/crate-ci/typos/compare/v1.22.2...v1.22.3 [1.22.2]: https://redirect.github.com/crate-ci/typos/compare/v1.22.1...v1.22.2 [1.22.1]: https://redirect.github.com/crate-ci/typos/compare/v1.22.0...v1.22.1 [1.22.0]: https://redirect.github.com/crate-ci/typos/compare/v1.21.0...v1.22.0 [1.21.0]: https://redirect.github.com/crate-ci/typos/compare/v1.20.10...v1.21.0 [1.20.10]: https://redirect.github.com/crate-ci/typos/compare/v1.20.9...v1.20.10 [1.20.9]: https://redirect.github.com/crate-ci/typos/compare/v1.20.8...v1.20.9 [1.20.8]: https://redirect.github.com/crate-ci/typos/compare/v1.20.7...v1.20.8 [1.20.7]: https://redirect.github.com/crate-ci/typos/compare/v1.20.6...v1.20.7 [1.20.6]: https://redirect.github.com/crate-ci/typos/compare/v1.20.5...v1.20.6 [1.20.5]: https://redirect.github.com/crate-ci/typos/compare/v1.20.4...v1.20.5 [1.20.4]: https://redirect.github.com/crate-ci/typos/compare/v1.20.3...v1.20.4 [1.20.3]: https://redirect.github.com/crate-ci/typos/compare/v1.20.2...v1.20.3 [1.20.2]: https://redirect.github.com/crate-ci/typos/compare/v1.20.1...v1.20.2 [1.20.1]: https://redirect.github.com/crate-ci/typos/compare/v1.20.0...v1.20.1 [1.20.0]: https://redirect.github.com/crate-ci/typos/compare/v1.19.0...v1.20.0 [1.19.0]: https://redirect.github.com/crate-ci/typos/compare/v1.18.2...v1.19.0 [1.18.2]: https://redirect.github.com/crate-ci/typos/compare/v1.18.1...v1.18.2 [1.18.1]: https://redirect.github.com/crate-ci/typos/compare/v1.18.0...v1.18.1 [1.18.0]: https://redirect.github.com/crate-ci/typos/compare/v1.17.2...v1.18.0 [1.17.2]: https://redirect.github.com/crate-ci/typos/compare/v1.17.1...v1.17.2 [1.17.1]: https://redirect.github.com/crate-ci/typos/compare/v1.17.0...v1.17.1 [1.17.0]: https://redirect.github.com/crate-ci/typos/compare/v1.16.26...v1.17.0 [1.16.26]: https://redirect.github.com/crate-ci/typos/compare/v1.16.25...v1.16.26 [1.16.25]: https://redirect.github.com/crate-ci/typos/compare/v1.16.24...v1.16.25 [1.16.24]: https://redirect.github.com/crate-ci/typos/compare/v1.16.23...v1.16.24 [1.16.23]: https://redirect.github.com/crate-ci/typos/compare/v1.16.22...v1.16.23 [1.16.22]: https://redirect.github.com/crate-ci/typos/compare/v1.16.21...v1.16.22 [1.16.21]: https://redirect.github.com/crate-ci/typos/compare/v1.16.20...v1.16.21 [1.16.20]: https://redirect.github.com/crate-ci/typos/compare/v1.16.19...v1.16.20 [1.16.19]: https://redirect.github.com/crate-ci/typos/compare/v1.16.18...v1.16.19 [1.16.18]: https://redirect.github.com/crate-ci/typos/compare/v1.16.17...v1.16.18 [1.16.17]: https://redirect.github.com/crate-ci/typos/compare/v1.16.16...v1.16.17 [1.16.16]: https://redirect.github.com/crate-ci/typos/compare/v1.16.15...v1.16.16 [1.16.15]: https://redirect.github.com/crate-ci/typos/compare/v1.16.14...v1.16.15 [1.16.14]: https://redirect.github.com/crate-ci/typos/compare/v1.16.13...v1.16.14 [1.16.13]: https://redirect.github.com/crate-ci/typos/compare/v1.16.12...v1.16.13 [1.16.12]: https://redirect.github.com/crate-ci/typos/compare/v1.16.11...v1.16.12 [1.16.11]: https://redirect.github.com/crate-ci/typos/compare/v1.16.10...v1.16.11 [1.16.10]: https://redirect.github.com/crate-ci/typos/compare/v1.16.9...v1.16.10 [1.16.9]: https://redirect.github.com/crate-ci/typos/compare/v1.16.8...v1.16.9 [1.16.8]: https://redirect.github.com/crate-ci/typos/compare/v1.16.7...v1.16.8 [1.16.7]: https://redirect.github.com/crate-ci/typos/compare/v1.16.6...v1.16.7 [1.16.6]: https://redirect.github.com/crate-ci/typos/compare/v1.16.5...v1.16.6 [1.16.5]: https://redirect.github.com/crate-ci/typos/compare/v1.16.4...v1.16.5 [1.16.4]: https://redirect.github.com/crate-ci/typos/compare/v1.16.3...v1.16.4 [1.16.3]: https://redirect.github.com/crate-ci/typos/compare/v1.16.2...v1.16.3 [1.16.2]: https://redirect.github.com/crate-ci/typos/compare/v1.16.1...v1.16.2 [1.16.1]: https://redirect.github.com/crate-ci/typos/compare/v1.16.0...v1.16.1 [1.16.0]: https://redirect.github.com/crate-ci/typos/compare/v1.15.10...v1.16.0 [1.15.10]: https://redirect.github.com/crate-ci/typos/compare/v1.15.9...v1.15.10 [1.15.9]: https://redirect.github.com/crate-ci/typos/compare/v1.15.8...v1.15.9 [1.15.8]: https://redirect.github.com/crate-ci/typos/compare/v1.15.7...v1.15.8 [1.15.7]: https://redirect.github.com/crate-ci/typos/compare/v1.15.6...v1.15.7 [1.15.6]: https://redirect.github.com/crate-ci/typos/compare/v1.15.5...v1.15.6 [1.15.5]: https://redirect.github.com/crate-ci/typos/compare/v1.15.4...v1.15.5 [1.15.4]: https://redirect.github.com/crate-ci/typos/compare/v1.15.3...v1.15.4 [1.15.3]: https://redirect.github.com/crate-ci/typos/compare/v1.15.2...v1.15.3 [1.15.2]: https://redirect.github.com/crate-ci/typos/compare/v1.15.1...v1.15.2 [1.15.1]: https://redirect.github.com/crate-ci/typos/compare/v1.15.0...v1.15.1 [1.15.0]: https://redirect.github.com/crate-ci/typos/compare/v1.14.12...v1.15.0 [1.14.12]: https://redirect.github.com/crate-ci/typos/compare/v1.14.11...v1.14.12 [1.14.11]: https://redirect.github.com/crate-ci/typos/compare/v1.14.10...v1.14.11 [1.14.10]: https://redirect.github.com/crate-ci/typos/compare/v1.14.9...v1.14.10 [1.14.9]: https://redirect.github.com/crate-ci/typos/compare/v1.14.8...v1.14.9 [1.14.8]: https://redirect.github.com/crate-ci/typos/compare/v1.14.7...v1.14.8 [1.14.7]: https://redirect.github.com/crate-ci/typos/compare/v1.14.6...v1.14.7 [1.14.6]: https://redirect.github.com/crate-ci/typos/compare/v1.14.5...v1.14.6 [1.14.5]: https://redirect.github.com/crate-ci/typos/compare/v1.14.4...v1.14.5 [1.14.4]: https://redirect.github.com/crate-ci/typos/compare/v1.14.3...v1.14.4 [1.14.3]: https://redirect.github.com/crate-ci/typos/compare/v1.14.2...v1.14.3 [1.14.2]: https://redirect.github.com/crate-ci/typos/compare/v1.14.1...v1.14.2 [1.14.1]: https://redirect.github.com/crate-ci/typos/compare/v1.14.0...v1.14.1 [1.14.0]: https://redirect.github.com/crate-ci/typos/compare/v1.13.26...v1.14.0 [1.13.26]: https://redirect.github.com/crate-ci/typos/compare/v1.13.25...v1.13.26 [1.13.25]: https://redirect.github.com/crate-ci/typos/compare/v1.13.24...v1.13.25 [1.13.24]: https://redirect.github.com/crate-ci/typos/compare/v1.13.23...v1.13.24 [1.13.23]: https://redirect.github.com/crate-ci/typos/compare/v1.13.22...v1.13.23 [1.13.22]: https://redirect.github.com/crate-ci/typos/compare/v1.13.21...v1.13.22 [1.13.21]: https://redirect.github.com/crate-ci/typos/compare/v1.13.20...v1.13.21 [1.13.20]: https://redirect.github.com/crate-ci/typos/compare/v1.13.19...v1.13.20 [1.13.19]: https://redirect.github.com/crate-ci/typos/compare/v1.13.18...v1.13.19 [1.13.18]: https://redirect.github.com/crate-ci/typos/compare/v1.13.17...v1.13.18 [1.13.17]: https://redirect.github.com/crate-ci/typos/compare/v1.13.16...v1.13.17 [1.13.16]: https://redirect.github.com/crate-ci/typos/compare/v1.13.15...v1.13.16 [1.13.15]: https://redirect.github.com/crate-ci/typos/compare/v1.13.14...v1.13.15 [1.13.14]: https://redirect.github.com/crate-ci/typos/compare/v1.13.13...v1.13.14 [1.13.13]: https://redirect.github.com/crate-ci/typos/compare/v1.13.12...v1.13.13 [1.13.12]: https://redirect.github.com/crate-ci/typos/compare/v1.13.11...v1.13.12 [1.13.11]: https://redirect.github.com/crate-ci/typos/compare/v1.13.10...v1.13.11 [1.13.10]: https://redirect.github.com/crate-ci/typos/compare/v1.13.9...v1.13.10 [1.13.9]: https://redirect.github.com/crate-ci/typos/compare/v1.13.8...v1.13.9 [1.13.8]: https://redirect.github.com/crate-ci/typos/compare/v1.13.7...v1.13.8 [1.13.7]: https://redirect.github.com/crate-ci/typos/compare/v1.13.6...v1.13.7 [1.13.6]: https://redirect.github.com/crate-ci/typos/compare/v1.13.5...v1.13.6 [1.13.5]: https://redirect.github.com/crate-ci/typos/compare/v1.13.4...v1.13.5 [1.13.4]: https://redirect.github.com/crate-ci/typos/compare/v1.13.3...v1.13.4 [1.13.3]: https://redirect.github.com/crate-ci/typos/compare/v1.13.2...v1.13.3 [1.13.2]: https://redirect.github.com/crate-ci/typos/compare/v1.13.1...v1.13.2 [1.13.1]: https://redirect.github.com/crate-ci/typos/compare/v1.13.0...v1.13.1 [1.13.0]: https://redirect.github.com/crate-ci/typos/compare/v1.12.14...v1.13.0 [1.12.14]: https://redirect.github.com/crate-ci/typos/compare/v1.12.13...v1.12.14 [1.12.13]: https://redirect.github.com/crate-ci/typos/compare/v1.12.12...v1.12.13 [1.12.12]: https://redirect.github.com/crate-ci/typos/compare/v1.12.11...v1.12.12 [1.12.11]: https://redirect.github.com/crate-ci/typos/compare/v1.12.10...v1.12.11 [1.12.10]: https://redirect.github.com/crate-ci/typos/compare/v1.12.9...v1.12.10 [1.12.9]: https://redirect.github.com/crate-ci/typos/compare/v1.12.8...v1.12.9 [1.12.8]: https://redirect.github.com/crate-ci/typos/compare/v1.12.7...v1.12.8 [1.12.7]: https://redirect.github.com/crate-ci/typos/compare/v1.12.6...v1.12.7 [1.12.6]: https://redirect.github.com/crate-ci/typos/compare/v1.12.5...v1.12.6 [1.12.5]: https://redirect.github.com/crate-ci/typos/compare/v1.12.4...v1.12.5 [1.12.4]: https://redirect.github.com/crate-ci/typos/compare/v1.12.3...v1.12.4 [1.12.3]: https://redirect.github.com/crate-ci/typos/compare/v1.12.2...v1.12.3 [1.12.2]: https://redirect.github.com/crate-ci/typos/compare/v1.12.1...v1.12.2 [1.12.1]: https://redirect.github.com/crate-ci/typos/compare/v1.12.0...v1.12.1 [1.12.0]: https://redirect.github.com/crate-ci/typos/compare/v1.11.5...v1.12.0 [1.11.5]: https://redirect.github.com/crate-ci/typos/compare/v1.11.4...v1.11.5 [1.11.4]: https://redirect.github.com/crate-ci/typos/compare/v1.11.3...v1.11.4 [1.11.3]: https://redirect.github.com/crate-ci/typos/compare/v1.11.2...v1.11.3 [1.11.2]: https://redirect.github.com/crate-ci/typos/compare/v1.11.1...v1.11.2 [1.11.1]: https://redirect.github.com/crate-ci/typos/compare/v1.11.0...v1.11.1 [1.11.0]: https://redirect.github.com/crate-ci/typos/compare/v1.10.3...v1.11.0 [1.10.3]: https://redirect.github.com/crate-ci/typos/compare/v1.10.2...v1.10.3 [1.10.2]: https://redirect.github.com/crate-ci/typos/compare/v1.10.1...v1.10.2 [1.10.1]: https://redirect.github.com/crate-ci/typos/compare/v1.10.0...v1.10.1 [1.10.0]: https://redirect.github.com/crate-ci/typos/compare/v1.9.0...v1.10.0 [1.9.0]: https://redirect.github.com/crate-ci/typos/compare/v1.8.1...v1.9.0 [1.8.1]: https://redirect.github.com/crate-ci/typos/compare/v1.8.0...v1.8.1 [1.8.0]: https://redirect.github.com/crate-ci/typos/compare/v1.7.3...v1.8.0 [1.7.3]: https://redirect.github.com/crate-ci/typos/compare/v1.7.2...v1.7.3 [1.7.2]: https://redirect.github.com/crate-ci/typos/compare/v1.7.1...v1.7.2 [1.7.1]: https://redirect.github.com/crate-ci/typos/compare/v1.7.0...v1.7.1 [1.7.0]: https://redirect.github.com/crate-ci/typos/compare/v1.6.0...v1.7.0 [1.6.0]: https://redirect.github.com/crate-ci/typos/compare/v1.5.0...v1.6.0 [1.5.0]: https://redirect.github.com/crate-ci/typos/compare/v1.4.1...v1.5.0 [1.4.1]: https://redirect.github.com/crate-ci/typos/compare/v1.4.0...v1.4.1 [1.4.0]: https://redirect.github.com/crate-ci/typos/compare/v1.3.9...v1.4.0 [1.3.9]: https://redirect.github.com/crate-ci/typos/compare/v1.3.8...v1.3.9 [1.3.8]: https://redirect.github.com/crate-ci/typos/compare/v1.3.7...v1.3.8 [1.3.7]: https://redirect.github.com/crate-ci/typos/compare/v1.3.6...v1.3.7 [1.3.6]: https://redirect.github.com/crate-ci/typos/compare/v1.3.5...v1.3.6 [1.3.5]: https://redirect.github.com/crate-ci/typos/compare/v1.3.4...v1.3.5 [1.3.4]: https://redirect.github.com/crate-ci/typos/compare/v1.3.3...v1.3.4 [1.3.3]: https://redirect.github.com/crate-ci/typos/compare/v1.3.2...v1.3.3 [1.3.2]: https://redirect.github.com/crate-ci/typos/compare/v1.3.1...v1.3.2 [1.3.1]: https://redirect.github.com/crate-ci/typos/compare/v1.3.0...v1.3.1 [1.3.0]: https://redirect.github.com/crate-ci/typos/compare/v1.2.1...v1.3.0 [1.2.1]: https://redirect.github.com/crate-ci/typos/compare/v1.2.0...v1.2.1 [1.2.0]: https://redirect.github.com/crate-ci/typos/compare/v1.1.9...v1.2.0 [1.1.9]: https://redirect.github.com/crate-ci/typos/compare/v1.1.8...v1.1.9 [1.1.8]: https://redirect.github.com/crate-ci/typos/compare/v1.1.7...v1.1.8 [1.1.7]: https://redirect.github.com/crate-ci/typos/compare/v1.1.6...v1.1.7 [1.1.6]: https://redirect.github.com/crate-ci/typos/compare/v1.1.5...v1.1.6 [1.1.5]: https://redirect.github.com/crate-ci/typos/compare/v1.1.4...v1.1.5 [1.1.4]: https://redirect.github.com/crate-ci/typos/compare/v1.1.3...v1.1.4 [1.1.3]: https://redirect.github.com/crate-ci/typos/compare/v1.1.2...v1.1.3 [1.1.2]: https://redirect.github.com/crate-ci/typos/compare/v1.1.1...v1.1.2 [1.1.1]: https://redirect.github.com/crate-ci/typos/compare/v1.1.0...v1.1.1 [1.1.0]: https://redirect.github.com/crate-ci/typos/compare/v1.0.11...v1.1.0 [1.0.11]: https://redirect.github.com/crate-ci/typos/compare/v1.0.10...v1.0.11 [1.0.10]: https://redirect.github.com/crate-ci/typos/compare/v1.0.9...v1.0.10 [1.0.9]: https://redirect.github.com/crate-ci/typos/compare/v1.0.8...v1.0.9 [1.0.8]: https://redirect.github.com/crate-ci/typos/compare/v1.0.7...v1.0.8 [1.0.7]: https://redirect.github.com/crate-ci/typos/compare/v1.0.6...v1.0.7 [1.0.6]: https://redirect.github.com/crate-ci/typos/compare/v1.0.5...v1.0.6 [1.0.5]: https://redirect.github.com/crate-ci/typos/compare/v1.0.4...v1.0.5 [1.0.4]: https://redirect.github.com/crate-ci/typos/compare/v1.0.3...v1.0.4 [1.0.3]: https://redirect.github.com/crate-ci/typos/compare/v1.0.2...v1.0.3 [1.0.2]: https://redirect.github.com/crate-ci/typos/compare/v1.0.1...v1.0.2 [1.0.1]: https://redirect.github.com/crate-ci/typos/compare/v1.0.0...v1.0.1 [1.0.0]: https://redirect.github.com/crate-ci/typos/compare/v0.4.0...v1.0.0 [0.4.0]: https://redirect.github.com/crate-ci/typos/compare/v0.3.0...v0.4.0 [0.3.0]: https://redirect.github.com/crate-ci/typos/compare/v0.2.0...v0.3.0 [0.2.0]: https://redirect.github.com/crate-ci/typos/compare/v0.1.4...v0.2.0 ### [`v1.45.0`](https://redirect.github.com/crate-ci/typos/blob/HEAD/CHANGELOG.md#014---2019-11-03) [Compare Source](https://redirect.github.com/crate-ci/typos/compare/v1.44.0...v1.45.0) ##### Bug Fixes - Ignore numbers as identifiers ([a00831c8](https://redirect.github.com/crate-ci/typos/commit/a00831c847b7efd81be520ea9b5d02f70555351f)) - Improve the organization of --help ([a48a457c](https://redirect.github.com/crate-ci/typos/commit/a48a457cc3ca817850118e2a2fb8b20fecdd40b8)) ##### Features - Dump files, identifiers, and words ([ce365ae1](https://redirect.github.com/crate-ci/typos/commit/ce365ae12e12fddfb6fc42a7f1e5ea71834d6051), closes [#​41](https://redirect.github.com/crate-ci/typos/issues/41)) - Give control over allowed identifier characters for leading vs rest ([107308a6](https://redirect.github.com/crate-ci/typos/commit/107308a655a425eb593bf5e4928572c16e6a9bdd)) ##### Performance - Use standard identifier rules to avoid doing umber checks ([107308a6](https://redirect.github.com/crate-ci/typos/commit/107308a655a425eb593bf5e4928572c16e6a9bdd)) - Only do hex check if digits are in identifiers ([68cd36d0](https://redirect.github.com/crate-ci/typos/commit/68cd36d0de90226dbc9d31c2ce6d8bf6b69adb5c)) [Unreleased]: https://redirect.github.com/crate-ci/typos/compare/v1.45.0...HEAD [1.45.0]: https://redirect.github.com/crate-ci/typos/compare/v1.44.0...v1.45.0 [1.44.0]: https://redirect.github.com/crate-ci/typos/compare/v1.43.5...v1.44.0 [1.43.5]: https://redirect.github.com/crate-ci/typos/compare/v1.43.4...v1.43.5 [1.43.4]: https://redirect.github.com/crate-ci/typos/compare/v1.43.3...v1.43.4 [1.43.3]: https://redirect.github.com/crate-ci/typos/compare/v1.43.2...v1.43.3 [1.43.2]: https://redirect.github.com/crate-ci/typos/compare/v1.43.1...v1.43.2 [1.43.1]: https://redirect.github.com/crate-ci/typos/compare/v1.43.0...v1.43.1 [1.43.0]: https://redirect.github.com/crate-ci/typos/compare/v1.42.3...v1.43.0 [1.42.3]: https://redirect.github.com/crate-ci/typos/compare/v1.42.2...v1.42.3 [1.42.2]: https://redirect.github.com/crate-ci/typos/compare/v1.42.1...v1.42.2 [1.42.1]: https://redirect.github.com/crate-ci/typos/compare/v1.42.0...v1.42.1 [1.42.0]: https://redirect.github.com/crate-ci/typos/compare/v1.41.0...v1.42.0 [1.41.0]: https://redirect.github.com/crate-ci/typos/compare/v1.40.1...v1.41.0 [1.40.1]: https://redirect.github.com/crate-ci/typos/compare/v1.40.0...v1.40.1 [1.40.0]: https://redirect.github.com/crate-ci/typos/compare/v1.39.2...v1.40.0 [1.39.2]: https://redirect.github.com/crate-ci/typos/compare/v1.39.1...v1.39.2 [1.39.1]: https://redirect.github.com/crate-ci/typos/compare/v1.39.0...v1.39.1 [1.39.0]: https://redirect.github.com/crate-ci/typos/compare/v1.38.1...v1.39.0 [1.38.1]: https://redirect.github.com/crate-ci/typos/compare/v1.38.0...v1.38.1 [1.38.0]: https://redirect.github.com/crate-ci/typos/compare/v1.37.3...v1.38.0 [1.37.3]: https://redirect.github.com/crate-ci/typos/compare/v1.37.2...v1.37.3 [1.37.2]: https://redirect.github.com/crate-ci/typos/compare/v1.37.1...v1.37.2 [1.37.1]: https://redirect.github.com/crate-ci/typos/compare/v1.37.0...v1.37.1 [1.37.0]: https://redirect.github.com/crate-ci/typos/compare/v1.36.3...v1.37.0 [1.36.3]: https://redirect.github.com/crate-ci/typos/compare/v1.36.2...v1.36.3 [1.36.2]: https://redirect.github.com/crate-ci/typos/compare/v1.36.1...v1.36.2 [1.36.1]: https://redirect.github.com/crate-ci/typos/compare/v1.36.0...v1.36.1 [1.36.0]: https://redirect.github.com/crate-ci/typos/compare/v1.35.8...v1.36.0 [1.35.8]: https://redirect.github.com/crate-ci/typos/compare/v1.35.7...v1.35.8 [1.35.7]: https://redirect.github.com/crate-ci/typos/compare/v1.35.6...v1.35.7 [1.35.6]: https://redirect.github.com/crate-ci/typos/compare/v1.35.5...v1.35.6 [1.35.5]: https://redirect.github.com/crate-ci/typos/compare/v1.35.4...v1.35.5 [1.35.4]: https://redirect.github.com/crate-ci/typos/compare/v1.35.3...v1.35.4 [1.35.3]: https://redirect.github.com/crate-ci/typos/compare/v1.35.2...v1.35.3 [1.35.2]: https://redirect.github.com/crate-ci/typos/compare/v1.35.1...v1.35.2 [1.35.1]: https://redirect.github.com/crate-ci/typos/compare/v1.35.0...v1.35.1 [1.35.0]: https://redirect.github.com/crate-ci/typos/compare/v1.34.0...v1.35.0 [1.34.0]: https://redirect.github.com/crate-ci/typos/compare/v1.33.1...v1.34.0 [1.33.1]: https://redirect.github.com/crate-ci/typos/compare/v1.33.0...v1.33.1 [1.33.0]: https://redirect.github.com/crate-ci/typos/compare/v1.32.0...v1.33.0 [1.32.0]: https://redirect.github.com/crate-ci/typos/compare/v1.31.2...v1.32.0 [1.31.2]: https://redirect.github.com/crate-ci/typos/compare/v1.31.1...v1.31.2 [1.31.1]: https://redirect.github.com/crate-ci/typos/compare/v1.31.0...v1.31.1 [1.31.0]: https://redirect.github.com/crate-ci/typos/compare/v1.30.3...v1.31.0 [1.30.3]: https://redirect.github.com/crate-ci/typos/compare/v1.30.2...v1.30.3 [1.30.2]: https://redirect.github.com/crate-ci/typos/compare/v1.30.1...v1.30.2 [1.30.1]: https://redirect.github.com/crate-ci/typos/compare/v1.30.0...v1.30.1 [1.30.0]: https://redirect.github.com/crate-ci/typos/compare/v1.29.10...v1.30.0 [1.29.10]: https://redirect.github.com/crate-ci/typos/compare/v1.29.9...v1.29.10 [1.29.9]: https://redirect.github.com/crate-ci/typos/compare/v1.29.8...v1.29.9 [1.29.8]: https://redirect.github.com/crate-ci/typos/compare/v1.29.7...v1.29.8 [1.29.7]: https://redirect.github.com/crate-ci/typos/compare/v1.29.6...v1.29.7 [1.29.6]: https://redirect.github.com/crate-ci/typos/compare/v1.29.5...v1.29.6 [1.29.5]: https://redirect.github.com/crate-ci/typos/compare/v1.29.4...v1.29.5 [1.29.4]: https://redirect.github.com/crate-ci/typos/compare/v1.29.3...v1.29.4 [1.29.3]: https://redirect.github.com/crate-ci/typos/compare/v1.29.2...v1.29.3 [1.29.2]: https://redirect.github.com/crate-ci/typos/compare/v1.29.1...v1.29.2 [1.29.1]: https://redirect.github.com/crate-ci/typos/compare/v1.29.0...v1.29.1 [1.29.0]: https://redirect.github.com/crate-ci/typos/compare/v1.28.4...v1.29.0 [1.28.4]: https://redirect.github.com/crate-ci/typos/compare/v1.28.3...v1.28.4 [1.28.3]: https://redirect.github.com/crate-ci/typos/compare/v1.28.2...v1.28.3 [1.28.2]: https://redirect.github.com/crate-ci/typos/compare/v1.28.1...v1.28.2 [1.28.1]: https://redirect.github.com/crate-ci/typos/compare/v1.28.0...v1.28.1 [1.28.0]: https://redirect.github.com/crate-ci/typos/compare/v1.27.3...v1.28.0 [1.27.3]: https://redirect.github.com/crate-ci/typos/compare/v1.27.2...v1.27.3 [1.27.2]: https://redirect.github.com/crate-ci/typos/compare/v1.27.1...v1.27.2 [1.27.1]: https://redirect.github.com/crate-ci/typos/compare/v1.27.0...v1.27.1 [1.27.0]: https://redirect.github.com/crate-ci/typos/compare/v1.26.8...v1.27.0 [1.26.8]: https://redirect.github.com/crate-ci/typos/compare/v1.26.7...v1.26.8 [1.26.7]: https://redirect.github.com/crate-ci/typos/compare/v1.26.6...v1.26.7 [1.26.6]: https://redirect.github.com/crate-ci/typos/compare/v1.26.5...v1.26.6 [1.26.5]: https://redirect.github.com/crate-ci/typos/compare/v1.26.4...v1.26.5 [1.26.4]: https://redirect.github.com/crate-ci/typos/compare/v1.26.3...v1.26.4 [1.26.3]: https://redirect.github.com/crate-ci/typos/compare/v1.26.2...v1.26.3 [1.26.2]: https://redirect.github.com/crate-ci/typos/compare/v1.26.1...v1.26.2 [1.26.1]: https://redirect.github.com/crate-ci/typos/compare/v1.26.0...v1.26.1 [1.26.0]: https://redirect.github.com/crate-ci/typos/compare/v1.25.0...v1.26.0 [1.25.0]: https://redirect.github.com/crate-ci/typos/compare/v1.24.6...v1.25.0 [1.24.6]: https://redirect.github.com/crate-ci/typos/compare/v1.24.5...v1.24.6 [1.24.5]: https://redirect.github.com/crate-ci/typos/compare/v1.24.4...v1.24.5 [1.24.4]: https://redirect.github.com/crate-ci/typos/compare/v1.24.3...v1.24.4 [1.24.3]: https://redirect.github.com/crate-ci/typos/compare/v1.24.2...v1.24.3 [1.24.2]: https://redirect.github.com/crate-ci/typos/compare/v1.24.1...v1.24.2 [1.24.1]: https://redirect.github.com/crate-ci/typos/compare/v1.24.0...v1.24.1 [1.24.0]: https://redirect.github.com/crate-ci/typos/compare/v1.23.7...v1.24.0 [1.23.7]: https://redirect.github.com/crate-ci/typos/compare/v1.23.6...v1.23.7 [1.23.6]: https://redirect.github.com/crate-ci/typos/compare/v1.23.5...v1.23.6 [1.23.5]: https://redirect.github.com/crate-ci/typos/compare/v1.23.4...v1.23.5 [1.23.4]: https://redirect.github.com/crate-ci/typos/compare/v1.23.3...v1.23.4 [1.23.3]: https://redirect.github.com/crate-ci/typos/compare/v1.23.2...v1.23.3 [1.23.2]: https://redirect.github.com/crate-ci/typos/compare/v1.23.1...v1.23.2 [1.23.1]: https://redirect.github.com/crate-ci/typos/compare/v1.23.0...v1.23.1 [1.23.0]: https://redirect.github.com/crate-ci/typos/compare/v1.22.9...v1.23.0 [1.22.9]: https://redirect.github.com/crate-ci/typos/compare/v1.22.8...v1.22.9 [1.22.8]: https://redirect.github.com/crate-ci/typos/compare/v1.22.7...v1.22.8 [1.22.7]: https://redirect.github.com/crate-ci/typos/compare/v1.22.6...v1.22.7 [1.22.6]: https://redirect.github.com/crate-ci/typos/compare/v1.22.5...v1.22.6 [1.22.5]: https://redirect.github.com/crate-ci/typos/compare/v1.22.4...v1.22.5 [1.22.4]: https://redirect.github.com/crate-ci/typos/compare/v1.22.3...v1.22.4 [1.22.3]: https://redirect.github.com/crate-ci/typos/compare/v1.22.2...v1.22.3 [1.22.2]: https://redirect.github.com/crate-ci/typos/compare/v1.22.1...v1.22.2 [1.22.1]: https://redirect.github.com/crate-ci/typos/compare/v1.22.0...v1.22.1 [1.22.0]: https://redirect.github.com/crate-ci/typos/compare/v1.21.0...v1.22.0 [1.21.0]: https://redirect.github.com/crate-ci/typos/compare/v1.20.10...v1.21.0 [1.20.10]: https://redirect.github.com/crate-ci/typos/compare/v1.20.9...v1.20.10 [1.20.9]: https://redirect.github.com/crate-ci/typos/compare/v1.20.8...v1.20.9 [1.20.8]: https://redirect.github.com/crate-ci/typos/compare/v1.20.7...v1.20.8 [1.20.7]: https://redirect.github.com/crate-ci/typos/compare/v1.20.6...v1.20.7 [1.20.6]: https://redirect.github.com/crate-ci/typos/compare/v1.20.5...v1.20.6 [1.20.5]: https://redirect.github.com/crate-ci/typos/compare/v1.20.4...v1.20.5 [1.20.4]: https://redirect.github.com/crate-ci/typos/compare/v1.20.3...v1.20.4 [1.20.3]: https://redirect.github.com/crate-ci/typos/compare/v1.20.2...v1.20.3 [1.20.2]: https://redirect.github.com/crate-ci/typos/compare/v1.20.1...v1.20.2 [1.20.1]: https://redirect.github.com/crate-ci/typos/compare/v1.20.0...v1.20.1 [1.20.0]: https://redirect.github.com/crate-ci/typos/compare/v1.19.0...v1.20.0 [1.19.0]: https://redirect.github.com/crate-ci/typos/compare/v1.18.2...v1.19.0 [1.18.2]: https://redirect.github.com/crate-ci/typos/compare/v1.18.1...v1.18.2 [1.18.1]: https://redirect.github.com/crate-ci/typos/compare/v1.18.0...v1.18.1 [1.18.0]: https://redirect.github.com/crate-ci/typos/compare/v1.17.2...v1.18.0 [1.17.2]: https://redirect.github.com/crate-ci/typos/compare/v1.17.1...v1.17.2 [1.17.1]: https://redirect.github.com/crate-ci/typos/compare/v1.17.0...v1.17.1 [1.17.0]: https://redirect.github.com/crate-ci/typos/compare/v1.16.26...v1.17.0 [1.16.26]: https://redirect.github.com/crate-ci/typos/compare/v1.16.25...v1.16.26 [1.16.25]: https://redirect.github.com/crate-ci/typos/compare/v1.16.24...v1.16.25 [1.16.24]: https://redirect.github.com/crate-ci/typos/compare/v1.16.23...v1.16.24 [1.16.23]: https://redirect.github.com/crate-ci/typos/compare/v1.16.22...v1.16.23 [1.16.22]: https://redirect.github.com/crate-ci/typos/compare/v1.16.21...v1.16.22 [1.16.21]: https://redirect.github.com/crate-ci/typos/compare/v1.16.20...v1.16.21 [1.16.20]: https://redirect.github.com/crate-ci/typos/compare/v1.16.19...v1.16.20 [1.16.19]: https://redirect.github.com/crate-ci/typos/compare/v1.16.18...v1.16.19 [1.16.18]: https://redirect.github.com/crate-ci/typos/compare/v1.16.17...v1.16.18 [1.16.17]: https://redirect.github.com/crate-ci/typos/compare/v1.16.16...v1.16.17 [1.16.16]: https://redirect.github.com/crate-ci/typos/compare/v1.16.15...v1.16.16 [1.16.15]: https://redirect.github.com/crate-ci/typos/compare/v1.16.14...v1.16.15 [1.16.14]: https://redirect.github.com/crate-ci/typos/compare/v1.16.13...v1.16.14 [1.16.13]: https://redirect.github.com/crate-ci/typos/compare/v1.16.12...v1.16.13 [1.16.12]: https://redirect.github.com/crate-ci/typos/compare/v1.16.11...v1.16.12 [1.16.11]: https://redirect.github.com/crate-ci/typos/compare/v1.16.10...v1.16.11 [1.16.10]: https://redirect.github.com/crate-ci/typos/compare/v1.16.9...v1.16.10 [1.16.9]: https://redirect.github.com/crate-ci/typos/compare/v1.16.8...v1.16.9 [1.16.8]: https://redirect.github.com/crate-ci/typos/compare/v1.16.7...v1.16.8 [1.16.7]: https://redirect.github.com/crate-ci/typos/compare/v1.16.6...v1.16.7 [1.16.6]: https://redirect.github.com/crate-ci/typos/compare/v1.16.5...v1.16.6 [1.16.5]: https://redirect.github.com/crate-ci/typos/compare/v1.16.4...v1.16.5 [1.16.4]: https://redirect.github.com/crate-ci/typos/compare/v1.16.3...v1.16.4 [1.16.3]: https://redirect.github.com/crate-ci/typos/compare/v1.16.2...v1.16.3 [1.16.2]: https://redirect.github.com/crate-ci/typos/compare/v1.16.1...v1.16.2 [1.16.1]: https://redirect.github.com/crate-ci/typos/compare/v1.16.0...v1.16.1 [1.16.0]: https://redirect.github.com/crate-ci/typos/compare/v1.15.10...v1.16.0 [1.15.10]: https://redirect.github.com/crate-ci/typos/compare/v1.15.9...v1.15.10 [1.15.9]: https://redirect.github.com/crate-ci/typos/compare/v1.15.8...v1.15.9 [1.15.8]: https://redirect.github.com/crate-ci/typos/compare/v1.15.7...v1.15.8 [1.15.7]: https://redirect.github.com/crate-ci/typos/compare/v1.15.6...v1.15.7 [1.15.6]: https://redirect.github.com/crate-ci/typos/compare/v1.15.5...v1.15.6 [1.15.5]: https://redirect.github.com/crate-ci/typos/compare/v1.15.4...v1.15.5 [1.15.4]: https://redirect.github.com/crate-ci/typos/compare/v1.15.3...v1.15.4 [1.15.3]: https://redirect.github.com/crate-ci/typos/compare/v1.15.2...v1.15.3 [1.15.2]: https://redirect.github.com/crate-ci/typos/compare/v1.15.1...v1.15.2 [1.15.1]: https://redirect.github.com/crate-ci/typos/compare/v1.15.0...v1.15.1 [1.15.0]: https://redirect.github.com/crate-ci/typos/compare/v1.14.12...v1.15.0 [1.14.12]: https://redirect.github.com/crate-ci/typos/compare/v1.14.11...v1.14.12 [1.14.11]: https://redirect.github.com/crate-ci/typos/compare/v1.14.10...v1.14.11 [1.14.10]: https://redirect.github.com/crate-ci/typos/compare/v1.14.9...v1.14.10 [1.14.9]: https://redirect.github.com/crate-ci/typos/compare/v1.14.8...v1.14.9 [1.14.8]: https://redirect.github.com/crate-ci/typos/compare/v1.14.7...v1.14.8 [1.14.7]: https://redirect.github.com/crate-ci/typos/compare/v1.14.6...v1.14.7 [1.14.6]: https://redirect.github.com/crate-ci/typos/compare/v1.14.5...v1.14.6 [1.14.5]: https://redirect.github.com/crate-ci/typos/compare/v1.14.4...v1.14.5 [1.14.4]: https://redirect.github.com/crate-ci/typos/compare/v1.14.3...v1.14.4 [1.14.3]: https://redirect.github.com/crate-ci/typos/compare/v1.14.2...v1.14.3 [1.14.2]: https://redirect.github.com/crate-ci/typos/compare/v1.14.1...v1.14.2 [1.14.1]: https://redirect.github.com/crate-ci/typos/compare/v1.14.0...v1.14.1 [1.14.0]: https://redirect.github.com/crate-ci/typos/compare/v1.13.26...v1.14.0 [1.13.26]: https://redirect.github.com/crate-ci/typos/compare/v1.13.25...v1.13.26 [1.13.25]: https://redirect.github.com/crate-ci/typos/compare/v1.13.24...v1.13.25 [1.13.24]: https://redirect.github.com/crate-ci/typos/compare/v1.13.23...v1.13.24 [1.13.23]: https://redirect.github.com/crate-ci/typos/compare/v1.13.22...v1.13.23 [1.13.22]: https://redirect.github.com/crate-ci/typos/compare/v1.13.21...v1.13.22 [1.13.21]: https://redirect.github.com/crate-ci/typos/compare/v1.13.20...v1.13.21 [1.13.20]: https://redirect.github.com/crate-ci/typos/compare/v1.13.19...v1.13.20 [1.13.19]: https://redirect.github.com/crate-ci/typos/compare/v1.13.18...v1.13.19 [1.13.18]: https://redirect.github.com/crate-ci/typos/compare/v1.13.17...v1.13.18 [1.13.17]: https://redirect.github.com/crate-ci/typos/compare/v1.13.16...v1.13.17 [1.13.16]: https://redirect.github.com/crate-ci/typos/compare/v1.13.15...v1.13.16 [1.13.15]: https://redirect.github.com/crate-ci/typos/compare/v1.13.14...v1.13.15 [1.13.14]: https://redirect.github.com/crate-ci/typos/compare/v1.13.13...v1.13.14 [1.13.13]: https://redirect.github.com/crate-ci/typos/compare/v1.13.12...v1.13.13 [1.13.12]: https://redirect.github.com/crate-ci/typos/compare/v1.13.11...v1.13.12 [1.13.11]: https://redirect.github.com/crate-ci/typos/compare/v1.13.10...v1.13.11 [1.13.10]: https://redirect.github.com/crate-ci/typos/compare/v1.13.9...v1.13.10 [1.13.9]: https://redirect.github.com/crate-ci/typos/compare/v1.13.8...v1.13.9 [1.13.8]: https://redirect.github.com/crate-ci/typos/compare/v1.13.7...v1.13.8 [1.13.7]: https://redirect.github.com/crate-ci/typos/compare/v1.13.6...v1.13.7 [1.13.6]: https://redirect.github.com/crate-ci/typos/compare/v1.13.5...v1.13.6 [1.13.5]: https://redirect.github.com/crate-ci/typos/compare/v1.13.4...v1.13.5 [1.13.4]: https://redirect.github.com/crate-ci/typos/compare/v1.13.3...v1.13.4 [1.13.3]: https://redirect.github.com/crate-ci/typos/compare/v1.13.2...v1.13.3 [1.13.2]: https://redirect.github.com/crate-ci/typos/compare/v1.13.1...v1.13.2 [1.13.1]: https://redirect.github.com/crate-ci/typos/compare/v1.13.0...v1.13.1 [1.13.0]: https://redirect.github.com/crate-ci/typos/compare/v1.12.14...v1.13.0 [1.12.14]: https://redirect.github.com/crate-ci/typos/compare/v1.12.13...v1.12.14 [1.12.13]: https://redirect.github.com/crate-ci/typos/compare/v1.12.12...v1.12.13 [1.12.12]: https://redirect.github.com/crate-ci/typos/compare/v1.12.11...v1.12.12 [1.12.11]: https://redirect.github.com/crate-ci/typos/compare/v1.12.10...v1.12.11 [1.12.10]: https://redirect.github.com/crate-ci/typos/compare/v1.12.9...v1.12.10 [1.12.9]: https://redirect.github.com/crate-ci/typos/compare/v1.12.8...v1.12.9 [1.12.8]: https://redirect.github.com/crate-ci/typos/compare/v1.12.7...v1.12.8 [1.12.7]: https://redirect.github.com/crate-ci/typos/compare/v1.12.6...v1.12.7 [1.12.6]: https://redirect.github.com/crate-ci/typos/compare/v1.12.5...v1.12.6 [1.12.5]: https://redirect.github.com/crate-ci/typos/compare/v1.12.4...v1.12.5 [1.12.4]: https://redirect.github.com/crate-ci/typos/compare/v1.12.3...v1.12.4 [1.12.3]: https://redirect.github.com/crate-ci/typos/compare/v1.12.2...v1.12.3 [1.12.2]: https://redirect.github.com/crate-ci/typos/compare/v1.12.1...v1.12.2 [1.12.1]: https://redirect.github.com/crate-ci/typos/compare/v1.12.0...v1.12.1 [1.12.0]: https://redirect.github.com/crate-ci/typos/compare/v1.11.5...v1.12.0 [1.11.5]: https://redirect.github.com/crate-ci/typos/compare/v1.11.4...v1.11.5 [1.11.4]: https://redirect.github.com/crate-ci/typos/compare/v1.11.3...v1.11.4 [1.11.3]: https://redirect.github.com/crate-ci/typos/compare/v1.11.2...v1.11.3 [1.11.2]: https://redirect.github.com/crate-ci/typos/compare/v1.11.1...v1.11.2 [1.11.1]: https://redirect.github.com/crate-ci/typos/compare/v1.11.0...v1.11.1 [1.11.0]: https://redirect.github.com/crate-ci/typos/compare/v1.10.3...v1.11.0 [1.10.3]: https://redirect.github.com/crate-ci/typos/compare/v1.10.2...v1.10.3 [1.10.2]: https://redirect.github.com/crate-ci/typos/compare/v1.10.1...v1.10.2 [1.10.1]: https://redirect.github.com/crate-ci/typos/compare/v1.10.0...v1.10.1 [1.10.0]: https://redirect.github.com/crate-ci/typos/compare/v1.9.0...v1.10.0 [1.9.0]: https://redirect.github.com/crate-ci/typos/compare/v1.8.1...v1.9.0 [1.8.1]: https://redirect.github.com/crate-ci/typos/compare/v1.8.0...v1.8.1 [1.8.0]: https://redirect.github.com/crate-ci/typos/compare/v1.7.3...v1.8.0 [1.7.3]: https://redirect.github.com/crate-ci/typos/compare/v1.7.2...v1.7.3 [1.7.2]: https://redirect.github.com/crate-ci/typos/compare/v1.7.1...v1.7.2 [1.7.1]: https://redirect.github.com/crate-ci/typos/compare/v1.7.0...v1.7.1 [1.7.0]: https://redirect.github.com/crate-ci/typos/compare/v1.6.0...v1.7.0 [1.6.0]: https://redirect.github.com/crate-ci/typos/compare/v1.5.0...v1.6.0 [1.5.0]: https://redirect.github.com/crate-ci/typos/compare/v1.4.1...v1.5.0 [1.4.1]: https://redirect.github.com/crate-ci/typos/compare/v1.4.0...v1.4.1 [1.4.0]: https://redirect.github.com/crate-ci/typos/compare/v1.3.9...v1.4.0 [1.3.9]: https://redirect.github.com/crate-ci/typos/compare/v1.3.8...v1.3.9 [1.3.8]: https://redirect.github.com/crate-ci/typos/compare/v1.3.7...v1.3.8 [1.3.7]: https://redirect.github.com/crate-ci/typos/compare/v1.3.6...v1.3.7 [1.3.6]: https://redirect.github.com/crate-ci/typos/compare/v1.3.5...v1.3.6 [1.3.5]: https://redirect.github.com/crate-ci/typos/compare/v1.3.4...v1.3.5 [1.3.4]: https://redirect.github.com/crate-ci/typos/compare/v1.3.3...v1.3.4 [1.3.3]: https://redirect.github.com/crate-ci/typos/compare/v1.3.2...v1.3.3 [1.3.2]: https://redirect.github.com/crate-ci/typos/compare/v1.3.1...v1.3.2 [1.3.1]: https://redirect.github.com/crate-ci/typos/compare/v1.3.0...v1.3.1 [1.3.0]: https://redirect.github.com/crate-ci/typos/compare/v1.2.1...v1.3.0 [1.2.1]: https://redirect.github.com/crate-ci/typos/compare/v1.2.0...v1.2.1 [1.2.0]: https://redirect.github.com/crate-ci/typos/compare/v1.1.9...v1.2.0 [1.1.9]: https://redirect.github.com/crate-ci/typos/compare/v1.1.8...v1.1.9 [1.1.8]: https://redirect.github.com/crate-ci/typos/compare/v1.1.7...v1.1.8 [1.1.7]: https://redirect.github.com/crate-ci/typos/compare/v1.1.6...v1.1.7 [1.1.6]: https://redirect.github.com/crate-ci/typos/compare/v1.1.5...v1.1.6 [1.1.5]: https://redirect.github.com/crate-ci/typos/compare/v1.1.4...v1.1.5 [1.1.4]: https://redirect.github.com/crate-ci/typos/compare/v1.1.3...v1.1.4 [1.1.3]: https://redirect.github.com/crate-ci/typos/compare/v1.1.2...v1.1.3 [1.1.2]: https://redirect.github.com/crate-ci/typos/compare/v1.1.1...v1.1.2 [1.1.1]: https://redirect.github.com/crate-ci/typos/compare/v1.1.0...v1.1.1 [1.1.0]: https://redirect.github.com/crate-ci/typos/compare/v1.0.11...v1.1.0 [1.0.11]: https://redirect.github.com/crate-ci/typos/compare/v1.0.10...v1.0.11 [1.0.10]: https://redirect.github.com/crate-ci/typos/compare/v1.0.9...v1.0.10 [1.0.9]: https://redirect.github.com/crate-ci/typos/compare/v1.0.8...v1.0.9 [1.0.8]: https://redirect.github.com/crate-ci/typos/compare/v1.0.7...v1.0.8 [1.0.7]: https://redirect.github.com/crate-ci/typos/compare/v1.0.6...v1.0.7 [1.0.6]: https://redirect.github.com/crate-ci/typos/compare/v1.0.5...v1.0.6 [1.0.5]: https://redirect.github.com/crate-ci/typos/compare/v1.0.4...v1.0.5 [1.0.4]: https://redirect.github.com/crate-ci/typos/compare/v1.0.3...v1.0.4 [1.0.3]: https://redirect.github.com/crate-ci/typos/compare/v1.0.2...v1.0.3 [1.0.2]: https://redirect.github.com/crate-ci/typos/compare/v1.0.1...v1.0.2 [1.0.1]: https://redirect.github.com/crate-ci/typos/compare/v1.0.0...v1.0.1 [1.0.0]: https://redirect.github.com/crate-ci/typos/compare/v0.4.0...v1.0.0 [0.4.0]: https://redirect.github.com/crate-ci/typos/compare/v0.3.0...v0.4.0 [0.3.0]: https://redirect.github.com/crate-ci/typos/compare/v0.2.0...v0.3.0 [0.2.0]: https://redirect.github.com/crate-ci/typos/compare/v0.1.4...v0.2.0
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/typos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 41f96cd6853..eeabe3c7388 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -21,4 +21,4 @@ jobs: - name: Checkout Actions Repository uses: actions/checkout@v6 - name: Spell Check Repo - uses: crate-ci/typos@v1.44.0 + uses: crate-ci/typos@v1.45.1 From c65422391681bc2c2e2bf78a53118c691cd2e8b6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:34:41 +0000 Subject: [PATCH 113/250] Update dependency com.adobe.testing:s3mock-testcontainers to v4.12.4 (#7534) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [com.adobe.testing:s3mock-testcontainers](https://redirect.github.com/adobe/S3Mock) | `4.11.0` → `4.12.4` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.adobe.testing:s3mock-testcontainers/4.12.4?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.adobe.testing:s3mock-testcontainers/4.11.0/4.12.4?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
adobe/S3Mock (com.adobe.testing:s3mock-testcontainers) ### [`v4.12.4`](https://redirect.github.com/adobe/S3Mock/blob/HEAD/CHANGELOG.md#4124) [Compare Source](https://redirect.github.com/adobe/S3Mock/compare/4.11.0...4.12.4) Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration. **This is the last planned minor release of 4.x.** - Features and fixes - none - Version updates (deliverable dependencies) - Bump testcontainers.version from 1.21.3 to 1.21.4 - Version updates (build dependencies) - none
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index 6e93cb5e96c..a4aa501d6e7 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -14,7 +14,7 @@ protobuf = "4.33.5" slf4j = "2.0.17" spark3 = "3.5.8" spark4 = "4.1.1" -s3mock = "4.11.0" +s3mock = "4.12.4" testcontainers-jupiter = "1.20.4" [libraries] From 045d3d78654802488130f0d7b4d19004e488e5b1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:35:53 +0000 Subject: [PATCH 114/250] Update dependency eslint-plugin-react-refresh to ^0.5.0 (#7537) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [eslint-plugin-react-refresh](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh) | [`^0.4.20` → `^0.5.0`](https://renovatebot.com/diffs/npm/eslint-plugin-react-refresh/0.4.26/0.5.2) | ![age](https://developer.mend.io/api/mc/badges/age/npm/eslint-plugin-react-refresh/0.5.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/eslint-plugin-react-refresh/0.4.26/0.5.2?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
ArnaudBarre/eslint-plugin-react-refresh (eslint-plugin-react-refresh) ### [`v0.5.2`](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/blob/HEAD/CHANGELOG.md#052) [Compare Source](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/compare/v0.5.1...v0.5.2) - Support nested function calls for extraHOCs (actually fixes [#​104](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/issues/104)) ### [`v0.5.1`](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/blob/HEAD/CHANGELOG.md#051) [Compare Source](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/compare/v0.5.0...v0.5.1) - Mark ESLint v10 as supported - Support false positives with TypeScript function overloading (fixes [#​105](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/issues/105)) - Support nested function calls for extraHOCs (fixes [#​104](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/issues/104)) ### [`v0.5.0`](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/blob/HEAD/CHANGELOG.md#050) [Compare Source](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/compare/v0.4.26...v0.5.0) ##### Breaking changes - The package now ships as ESM and requires ESLint 9 + node 20. Because legacy config doesn't support ESM, this requires to use [flat config](https://eslint.org/docs/latest/use/configure/migration-guide) - A new `reactRefresh` export is available and prefered over the default export. It's an object with two properties: - `plugin`: The plugin object with the rules - `configs`: An object containing configuration presets, each exposed as a function. These functions accept your custom options, merge them with sensible defaults for that config, and return the final config object. - `customHOCs` option was renamed to `extraHOCs` - Validation of HOCs calls is now more strict, you may need to add some HOCs to the `extraHOCs` option Config example: ```js import { defineConfig } from "eslint/config"; import { reactRefresh } from "eslint-plugin-react-refresh"; export default defineConfig( /* Main config */ reactRefresh.configs.vite({ extraHOCs: ["someLibHOC"] }), ); ``` Config example without config: ```js import { defineConfig } from "eslint/config"; import { reactRefresh } from "eslint-plugin-react-refresh"; export default defineConfig({ files: ["**/*.ts", "**/*.tsx"], plugins: { // other plugins "react-refresh": reactRefresh.plugin, }, rules: { // other rules "react-refresh/only-export-components": [ "warn", { extraHOCs: ["someLibHOC"] }, ], }, }); ``` ##### Why This version follows a revamp of the internal logic to better make the difference between random call expressions like `export const Enum = Object.keys(Record)` and actual React HOC calls like `export const MemoComponent = memo(Component)`. (fixes [#​93](https://redirect.github.com/ArnaudBarre/eslint-plugin-react-refresh/issues/93)) The rule now handles ternaries and patterns like `export default customHOC(props)(Component)` which makes it able to correctly support files like [this one](https://redirect.github.com/eclipse-apoapsis/ort-server/blob/ddfc624ce71b9f2ca6bad9b8c82d4c3249dd9c8b/ui/src/routes/__root.tsx) given this config: ```json { "react-refresh/only-export-components": [ "warn", { "extraHOCs": ["createRootRouteWithContext"] } ] } ``` > \[!NOTE] > Actually createRoute functions from TanStack Router are not React HOCs, they return route objects that [fake to be a memoized component](https://redirect.github.com/TanStack/router/blob/8628d0189412ccb8d3a01840aa18bac8295e18c8/packages/react-router/src/route.tsx#L263) but are not. When only doing `createRootRoute({ component: Foo })`, HMR will work fine, but as soon as you add a prop to the options that is not a React component, HMR will not work. I would recommend to avoid adding any TanStack function to `extraHOCs` it you want to preserve good HMR in the long term. [Bluesky thread](https://bsky.app/profile/arnaud-barre.bsky.social/post/3ma5h5tf2sk2e). Because I'm not 100% sure this new logic doesn't introduce any false positive, this is done in a major-like version. This also give me the occasion to remove the hardcoded `connect` from the rule. If you are using `connect` from `react-redux`, you should now add it to `extraHOCs` like this: ```json { "react-refresh/only-export-components": ["warn", { "extraHOCs": ["connect"] }] } ```
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 10 +++++----- vortex-web/package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index dd0d7e06c92..b3381a9e3ae 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -27,7 +27,7 @@ "eslint": "^9.25.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^5.2.0", - "eslint-plugin-react-refresh": "^0.4.20", + "eslint-plugin-react-refresh": "^0.5.0", "eslint-plugin-storybook": "^10.3.3", "globals": "^16.0.0", "prettier": "^3.8.1", @@ -3354,13 +3354,13 @@ } }, "node_modules/eslint-plugin-react-refresh": { - "version": "0.4.26", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.26.tgz", - "integrity": "sha512-1RETEylht2O6FM/MvgnyvT+8K21wLqDNg4qD51Zj3guhjt433XbnnkVttHMyaVyAFD03QSV4LPS5iE3VQmO7XQ==", + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.5.2.tgz", + "integrity": "sha512-hmgTH57GfzoTFjVN0yBwTggnsVUF2tcqi7RJZHqi9lIezSs4eFyAMktA68YD4r5kNw1mxyY4dmkyoFDb3FIqrA==", "dev": true, "license": "MIT", "peerDependencies": { - "eslint": ">=8.40" + "eslint": "^9 || ^10" } }, "node_modules/eslint-plugin-storybook": { diff --git a/vortex-web/package.json b/vortex-web/package.json index ae1ea81bd5a..76de26d444d 100644 --- a/vortex-web/package.json +++ b/vortex-web/package.json @@ -38,7 +38,7 @@ "eslint": "^9.25.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^5.2.0", - "eslint-plugin-react-refresh": "^0.4.20", + "eslint-plugin-react-refresh": "^0.5.0", "eslint-plugin-storybook": "^10.3.3", "globals": "^16.0.0", "prettier": "^3.8.1", From 7162a10531e28c4b27d6a592a25bb2074c5fabd2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:37:02 +0000 Subject: [PATCH 115/250] Update dependency com.google.protobuf:protobuf-java to v4.34.1 (#7536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [com.google.protobuf:protobuf-java](https://developers.google.com/protocol-buffers/) ([source](https://redirect.github.com/protocolbuffers/protobuf)) | `4.33.5` → `4.34.1` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.google.protobuf:protobuf-java/4.34.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.google.protobuf:protobuf-java/4.33.5/4.34.1?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index a4aa501d6e7..1dc730f7b1d 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -10,7 +10,7 @@ junit-jupiter = "6.0.3" logback = "1.5.32" netty = "4.2.12.Final" nopen = "1.0.1" -protobuf = "4.33.5" +protobuf = "4.34.1" slf4j = "2.0.17" spark3 = "3.5.8" spark4 = "4.1.1" From a6dd7762612ad55edbf32c3c0e29f3aa741f41e4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:37:20 +0000 Subject: [PATCH 116/250] Update datafusion monorepo to v53.1.0 (#7533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [datafusion](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-catalog](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-common](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-common-runtime](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-datasource](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-execution](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-expr](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-functions](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-physical-expr](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-physical-expr-adapter](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-physical-expr-common](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-physical-plan](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-pruning](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | | [datafusion-sqllogictest](https://datafusion.apache.org) ([source](https://redirect.github.com/apache/datafusion)) | workspace.dependencies | minor | `53.0.0` → `53.1.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
apache/datafusion (datafusion) ### [`v53.1.0`](https://redirect.github.com/apache/datafusion/compare/53.0.0...53.1.0) [Compare Source](https://redirect.github.com/apache/datafusion/compare/53.0.0...53.1.0)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 580 ++++++++++++++++++++++++++--------------------------- 1 file changed, 290 insertions(+), 290 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7fc01c5ebb..fb4803cecf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,7 +961,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.11.0", "log", "prettyplease", "proc-macro2", @@ -2114,9 +2114,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ "arrow 58.0.0", "arrow-schema 58.0.0", @@ -2124,32 +2124,32 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-catalog 53.0.0", - "datafusion-catalog-listing 53.0.0", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-datasource-arrow 53.0.0", + "datafusion-catalog 53.1.0", + "datafusion-catalog-listing 53.1.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-datasource-arrow 53.1.0", "datafusion-datasource-avro", - "datafusion-datasource-csv 53.0.0", - "datafusion-datasource-json 53.0.0", + "datafusion-datasource-csv 53.1.0", + "datafusion-datasource-json 53.1.0", "datafusion-datasource-parquet", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-functions-aggregate 53.0.0", - "datafusion-functions-nested 53.0.0", - "datafusion-functions-table 53.0.0", - "datafusion-functions-window 53.0.0", - "datafusion-optimizer 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-adapter 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-optimizer 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", - "datafusion-sql 53.0.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-nested 53.1.0", + "datafusion-functions-table 53.1.0", + "datafusion-functions-window 53.1.0", + "datafusion-optimizer 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-optimizer 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", + "datafusion-sql 53.1.0", "flate2", "futures", "itertools 0.14.0", @@ -2175,9 +2175,9 @@ dependencies = [ "anyhow", "clap", "custom-labels", - "datafusion 53.0.0", - "datafusion-common 53.0.0", - "datafusion-physical-plan 53.0.0", + "datafusion 53.1.0", + "datafusion-common 53.1.0", + "datafusion-physical-plan 53.1.0", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2221,21 +2221,21 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ "arrow 58.0.0", "async-trait", "dashmap", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "itertools 0.14.0", "log", @@ -2270,21 +2270,21 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ "arrow 58.0.0", "async-trait", - "datafusion-catalog 53.0.0", - "datafusion-common 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-adapter 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", "futures", "itertools 0.14.0", "log", @@ -2315,9 +2315,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2352,9 +2352,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" dependencies = [ "futures", "log", @@ -2392,9 +2392,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ "arrow 58.0.0", "async-compression", @@ -2402,15 +2402,15 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-adapter 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "flate2", "futures", "glob", @@ -2451,22 +2451,22 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ "arrow 58.0.0", "arrow-ipc 58.0.0", "async-trait", "bytes", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "itertools 0.14.0", "object_store 0.13.2", @@ -2475,19 +2475,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49dda81c79b6ba57b1853a9158abc66eb85a3aa1cede0c517dabec6d8a4ed3aa" +checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" dependencies = [ "apache-avro", "arrow 58.0.0", "async-trait", "bytes", - "datafusion-common 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "num-traits", "object_store 0.13.2", @@ -2518,21 +2518,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ "arrow 58.0.0", "async-trait", "bytes", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "object_store 0.13.2", "regex", @@ -2563,21 +2563,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ "arrow 58.0.0", "async-trait", "bytes", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-session 53.1.0", "futures", "object_store 0.13.2", "serde_json", @@ -2587,25 +2587,25 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ "arrow 58.0.0", "async-trait", "bytes", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-adapter 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-pruning 53.0.0", - "datafusion-session 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", + "datafusion-session 53.1.0", "futures", "itertools 0.14.0", "log", @@ -2623,9 +2623,9 @@ checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" [[package]] name = "datafusion-execution" @@ -2649,18 +2649,18 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ "arrow 58.0.0", "arrow-buffer 58.0.0", "async-trait", "chrono", "dashmap", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "log", "object_store 0.13.2", @@ -2694,19 +2694,19 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ "arrow 58.0.0", "async-trait", "chrono", - "datafusion-common 53.0.0", - "datafusion-doc 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-functions-window-common 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "indexmap", "itertools 0.14.0", "paste", @@ -2730,12 +2730,12 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ "arrow 58.0.0", - "datafusion-common 53.0.0", + "datafusion-common 53.1.0", "indexmap", "itertools 0.14.0", "paste", @@ -2773,9 +2773,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ "arrow 58.0.0", "arrow-buffer 58.0.0", @@ -2784,12 +2784,12 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common 53.0.0", - "datafusion-doc 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-macros 53.0.0", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-macros 53.1.0", "hex", "itertools 0.14.0", "log", @@ -2826,20 +2826,20 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-doc 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-macros 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "log", "num-traits", @@ -2861,15 +2861,15 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", ] [[package]] @@ -2897,22 +2897,22 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ "arrow 58.0.0", "arrow-ord 58.0.0", - "datafusion-common 53.0.0", - "datafusion-doc 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-functions-aggregate 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-macros 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr-common 53.1.0", "hashbrown 0.16.1", "itertools 0.14.0", "itoa", @@ -2938,16 +2938,16 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ "arrow 58.0.0", "async-trait", - "datafusion-catalog 53.0.0", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-plan 53.0.0", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "parking_lot", "paste", ] @@ -2972,18 +2972,18 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-doc 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions-window-common 53.0.0", - "datafusion-macros 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-doc 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-macros 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "log", "paste", ] @@ -3000,12 +3000,12 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" dependencies = [ - "datafusion-common 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", ] [[package]] @@ -3021,11 +3021,11 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" dependencies = [ - "datafusion-doc 53.0.0", + "datafusion-doc 53.1.0", "quote", "syn 2.0.117", ] @@ -3051,16 +3051,16 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ "arrow 58.0.0", "chrono", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-physical-expr 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", "indexmap", "itertools 0.14.0", "log", @@ -3093,17 +3093,17 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-physical-expr-common 53.1.0", "half", "hashbrown 0.16.1", "indexmap", @@ -3132,16 +3132,16 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "itertools 0.14.0", ] @@ -3161,15 +3161,15 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash 0.8.12", "arrow 58.0.0", "chrono", - "datafusion-common 53.0.0", - "datafusion-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr-common 53.1.0", "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", @@ -3196,19 +3196,19 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-pruning 53.0.0", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", "itertools 0.14.0", "recursive", ] @@ -3246,24 +3246,24 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash 0.8.12", "arrow 58.0.0", "arrow-ord 58.0.0", "arrow-schema 58.0.0", "async-trait", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-functions-aggregate-common 53.0.0", - "datafusion-functions-window-common 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate-common 53.1.0", + "datafusion-functions-window-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", "futures", "half", "hashbrown 0.16.1", @@ -3295,17 +3295,17 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ "arrow 58.0.0", - "datafusion-common 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-expr-common 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", + "datafusion-common 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-expr-common 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", "itertools 0.14.0", "log", ] @@ -3326,36 +3326,36 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" dependencies = [ "async-trait", - "datafusion-common 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-physical-plan 53.0.0", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-physical-plan 53.1.0", "parking_lot", ] [[package]] name = "datafusion-spark" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923a8b871962a9d860f036f743a20af50ff04729f1da2468ed220dab4f61c97d" +checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" dependencies = [ "arrow 58.0.0", "bigdecimal", "chrono", "crc32fast", - "datafusion 53.0.0", - "datafusion-catalog 53.0.0", - "datafusion-common 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-functions-aggregate 53.0.0", - "datafusion-functions-nested 53.0.0", + "datafusion 53.1.0", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-functions-aggregate 53.1.0", + "datafusion-functions-nested 53.1.0", "log", "percent-encoding", "rand 0.9.2", @@ -3384,16 +3384,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ "arrow 58.0.0", "bigdecimal", "chrono", - "datafusion-common 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions-nested 53.0.0", + "datafusion-common 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions-nested 53.1.0", "indexmap", "log", "recursive", @@ -3403,15 +3403,15 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a43746bd59e7f2655be4c5553ede4a1ceb1cd34005932fa9e2bd0641c714c46e" +checksum = "04e5a4a7a49143a68936992b6dbb0db44121c635e9992b2482817278f1e69c56" dependencies = [ "arrow 58.0.0", "async-trait", "bigdecimal", "clap", - "datafusion 53.0.0", + "datafusion 53.1.0", "datafusion-spark", "datafusion-substrait", "futures", @@ -3429,14 +3429,14 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5e5656a7e63d51dd3e5af3dbd347ea83bbe993a77c66b854b74961570d16490" +checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" dependencies = [ "async-recursion", "async-trait", "chrono", - "datafusion 53.0.0", + "datafusion 53.1.0", "half", "itertools 0.14.0", "object_store 0.13.2", @@ -6970,7 +6970,7 @@ checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ "rand 0.9.2", "socket2", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -7177,7 +7177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -7209,7 +7209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.117", @@ -10438,19 +10438,19 @@ dependencies = [ "anyhow", "arrow-schema 58.0.0", "async-trait", - "datafusion 53.0.0", - "datafusion-catalog 53.0.0", - "datafusion-common 53.0.0", - "datafusion-common-runtime 53.0.0", - "datafusion-datasource 53.0.0", - "datafusion-execution 53.0.0", - "datafusion-expr 53.0.0", - "datafusion-functions 53.0.0", - "datafusion-physical-expr 53.0.0", - "datafusion-physical-expr-adapter 53.0.0", - "datafusion-physical-expr-common 53.0.0", - "datafusion-physical-plan 53.0.0", - "datafusion-pruning 53.0.0", + "datafusion 53.1.0", + "datafusion-catalog 53.1.0", + "datafusion-common 53.1.0", + "datafusion-common-runtime 53.1.0", + "datafusion-datasource 53.1.0", + "datafusion-execution 53.1.0", + "datafusion-expr 53.1.0", + "datafusion-functions 53.1.0", + "datafusion-physical-expr 53.1.0", + "datafusion-physical-expr-adapter 53.1.0", + "datafusion-physical-expr-common 53.1.0", + "datafusion-physical-plan 53.1.0", + "datafusion-pruning 53.1.0", "futures", "insta", "itertools 0.14.0", @@ -10974,7 +10974,7 @@ dependencies = [ "async-trait", "bigdecimal", "clap", - "datafusion 53.0.0", + "datafusion 53.1.0", "datafusion-sqllogictest", "futures", "indicatif", @@ -11031,7 +11031,7 @@ dependencies = [ "clap", "console_error_panic_hook", "crossterm", - "datafusion 53.0.0", + "datafusion 53.1.0", "env_logger", "flatbuffers", "futures", From 618b688b9c2b5637af15789abf71d18885feb86f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:45:09 +0000 Subject: [PATCH 117/250] Update all patch updates (#7528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---|---|---| | [clap](https://redirect.github.com/clap-rs/clap) | workspace.dependencies | patch | `4.6.0` → `4.6.1` | ![age](https://developer.mend.io/api/mc/badges/age/crate/clap/4.6.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/clap/4.6.0/4.6.1?slim=true) | | [mimalloc](https://redirect.github.com/purpleprotocol/mimalloc_rust) | workspace.dependencies | patch | `0.1.48` → `0.1.49` | ![age](https://developer.mend.io/api/mc/badges/age/crate/mimalloc/0.1.49?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/crate/mimalloc/0.1.48/0.1.49?slim=true) | | [prettier](https://prettier.io) ([source](https://redirect.github.com/prettier/prettier)) | devDependencies | patch | [`3.8.2` → `3.8.3`](https://renovatebot.com/diffs/npm/prettier/3.8.2/3.8.3) | ![age](https://developer.mend.io/api/mc/badges/age/npm/prettier/3.8.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/prettier/3.8.2/3.8.3?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
clap-rs/clap (clap) ### [`v4.6.1`](https://redirect.github.com/clap-rs/clap/blob/HEAD/CHANGELOG.md#461---2026-04-15) [Compare Source](https://redirect.github.com/clap-rs/clap/compare/v4.6.0...v4.6.1) ##### Fixes - *(derive)* Ensure rebuilds happen when an read env variable is changed
purpleprotocol/mimalloc_rust (mimalloc) ### [`v0.1.49`](https://redirect.github.com/purpleprotocol/mimalloc_rust/releases/tag/v0.1.49): Version 0.1.49 [Compare Source](https://redirect.github.com/purpleprotocol/mimalloc_rust/compare/v0.1.48...v0.1.49) ##### Changes - Update to mimalloc v2.3.0 and v3.3.0 - Use mimalloc v3 by default.
prettier/prettier (prettier) ### [`v3.8.3`](https://redirect.github.com/prettier/prettier/compare/3.8.2...3.8.3) [Compare Source](https://redirect.github.com/prettier/prettier/compare/3.8.2...3.8.3)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 37 ++++++++++++++++++------------------ vortex-web/package-lock.json | 6 +++--- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb4803cecf7..26b0aa741d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1410,9 +1410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1433,9 +1433,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -3534,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4898,7 +4898,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4979,7 +4979,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5783,12 +5783,11 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libmimalloc-sys" -version = "0.1.44" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe" dependencies = [ "cc", - "libc", ] [[package]] @@ -6026,9 +6025,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.48" +version = "0.1.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a" dependencies = [ "libmimalloc-sys", ] @@ -6297,7 +6296,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8164,7 +8163,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8222,7 +8221,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9301,7 +9300,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9320,7 +9319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11325,7 +11324,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index b3381a9e3ae..c610f592e9f 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -4622,9 +4622,9 @@ } }, "node_modules/prettier": { - "version": "3.8.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.2.tgz", - "integrity": "sha512-8c3mgTe0ASwWAJK+78dpviD+A8EqhndQPUBpNUIPt6+xWlIigCwfN01lWr9MAede4uqXGTEKeQWTvzb3vjia0Q==", + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", + "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", "dev": true, "license": "MIT", "bin": { From 8dacc51c6b796a6cc6c6787c7f89cc90edfcb512 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:48:34 +0000 Subject: [PATCH 118/250] Update dependency lucide-react to ^0.577.0 (#7538) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [lucide-react](https://lucide.dev) ([source](https://redirect.github.com/lucide-icons/lucide/tree/HEAD/packages/lucide-react)) | [`^0.563.0` → `^0.577.0`](https://renovatebot.com/diffs/npm/lucide-react/0.563.0/0.577.0) | ![age](https://developer.mend.io/api/mc/badges/age/npm/lucide-react/0.577.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/lucide-react/0.563.0/0.577.0?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
lucide-icons/lucide (lucide-react) ### [`v0.577.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.577.0): Version 0.577.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.576.0...0.577.0) #### What's Changed - chore(deps): bump rollup from 4.53.3 to 4.59.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​4106](https://redirect.github.com/lucide-icons/lucide/pull/4106) - fix(repo): correctly ignore docs/releaseMetadata via .gitignore by [@​bhavberi](https://redirect.github.com/bhavberi) in [#​4100](https://redirect.github.com/lucide-icons/lucide/pull/4100) - feat(icons): added `ellipse` icon by [@​KISHORE-KUMAR-S](https://redirect.github.com/KISHORE-KUMAR-S) in [#​3749](https://redirect.github.com/lucide-icons/lucide/pull/3749) #### New Contributors - [@​bhavberi](https://redirect.github.com/bhavberi) made their first contribution in [#​4100](https://redirect.github.com/lucide-icons/lucide/pull/4100) - [@​KISHORE-KUMAR-S](https://redirect.github.com/KISHORE-KUMAR-S) made their first contribution in [#​3749](https://redirect.github.com/lucide-icons/lucide/pull/3749) **Full Changelog**: ### [`v0.576.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.576.0): Version 0.576.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.575.0...0.576.0) #### What's Changed - Added zodiac signs by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​712](https://redirect.github.com/lucide-icons/lucide/pull/712) - fix(icons): fixes guideline violations in `package-*` icons. by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​4074](https://redirect.github.com/lucide-icons/lucide/pull/4074) - fix(icons): changed `receipt` icon by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​4075](https://redirect.github.com/lucide-icons/lucide/pull/4075) - fix(icons): updated `cuboid` icon tags and categories by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​4095](https://redirect.github.com/lucide-icons/lucide/pull/4095) - fix(icons): changed `cuboid` icon by [@​jamiemlaw](https://redirect.github.com/jamiemlaw) in [#​4098](https://redirect.github.com/lucide-icons/lucide/pull/4098) - fix(lucide-font, lucide-static): Fixing stable code points by [@​ericfennis](https://redirect.github.com/ericfennis) in [#​3894](https://redirect.github.com/lucide-icons/lucide/pull/3894) - feat(icons): added `fishing-rod` icon by [@​7ender](https://redirect.github.com/7ender) in [#​3839](https://redirect.github.com/lucide-icons/lucide/pull/3839) **Full Changelog**: ### [`v0.575.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.575.0): Version 0.575.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.574.0...0.575.0) #### What's Changed - feat(icons): added `message-square-check` icon by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​4076](https://redirect.github.com/lucide-icons/lucide/pull/4076) - fix(lucide): Fix ESM Module output path in build by [@​ericfennis](https://redirect.github.com/ericfennis) in [#​4084](https://redirect.github.com/lucide-icons/lucide/pull/4084) - feat(icons): added `metronome` icon by [@​edwloef](https://redirect.github.com/edwloef) in [#​4063](https://redirect.github.com/lucide-icons/lucide/pull/4063) - fix(icons): remove execution permission of SVG files by [@​duckafire](https://redirect.github.com/duckafire) in [#​4053](https://redirect.github.com/lucide-icons/lucide/pull/4053) - fix(icons): changed `file-pen-line` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3970](https://redirect.github.com/lucide-icons/lucide/pull/3970) - feat(icons): added `square-arrow-right-exit` and `square-arrow-right-enter` icons by [@​EthanHazel](https://redirect.github.com/EthanHazel) in [#​3958](https://redirect.github.com/lucide-icons/lucide/pull/3958) - fix(icons): renamed `flip-*` to `square-centerline-dashed-*` by [@​jguddas](https://redirect.github.com/jguddas) in [#​3945](https://redirect.github.com/lucide-icons/lucide/pull/3945) #### New Contributors - [@​edwloef](https://redirect.github.com/edwloef) made their first contribution in [#​4063](https://redirect.github.com/lucide-icons/lucide/pull/4063) - [@​duckafire](https://redirect.github.com/duckafire) made their first contribution in [#​4053](https://redirect.github.com/lucide-icons/lucide/pull/4053) **Full Changelog**: ### [`v0.574.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.574.0): Version 0.574.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.573.0...0.574.0) #### What's Changed - fix(icons): changed `rocking-chair` icon by [@​jamiemlaw](https://redirect.github.com/jamiemlaw) in [#​3445](https://redirect.github.com/lucide-icons/lucide/pull/3445) - fix(icons): flipped `coins` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3158](https://redirect.github.com/lucide-icons/lucide/pull/3158) - feat(icons): added `x-line-top` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​2838](https://redirect.github.com/lucide-icons/lucide/pull/2838) - feat(icons): added `mouse-left` icon by [@​marvfash](https://redirect.github.com/marvfash) in [#​2788](https://redirect.github.com/lucide-icons/lucide/pull/2788) - feat(icons): added `mouse-right` icon by [@​marvfash](https://redirect.github.com/marvfash) in [#​2787](https://redirect.github.com/lucide-icons/lucide/pull/2787) #### New Contributors - [@​marvfash](https://redirect.github.com/marvfash) made their first contribution in [#​2788](https://redirect.github.com/lucide-icons/lucide/pull/2788) **Full Changelog**: ### [`v0.573.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.573.0): Version 0.573.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.572.0...0.573.0) #### What's Changed - fix(icons): changed `rocking-chair` icon by [@​jamiemlaw](https://redirect.github.com/jamiemlaw) in [#​3445](https://redirect.github.com/lucide-icons/lucide/pull/3445) - fix(icons): flipped `coins` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3158](https://redirect.github.com/lucide-icons/lucide/pull/3158) - feat(icons): added `x-line-top` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​2838](https://redirect.github.com/lucide-icons/lucide/pull/2838) - feat(icons): added `mouse-left` icon by [@​marvfash](https://redirect.github.com/marvfash) in [#​2788](https://redirect.github.com/lucide-icons/lucide/pull/2788) - feat(icons): added `mouse-right` icon by [@​marvfash](https://redirect.github.com/marvfash) in [#​2787](https://redirect.github.com/lucide-icons/lucide/pull/2787) #### New Contributors - [@​marvfash](https://redirect.github.com/marvfash) made their first contribution in [#​2788](https://redirect.github.com/lucide-icons/lucide/pull/2788) **Full Changelog**: ### [`v0.572.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.572.0): Version 0.572.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.571.0...0.572.0) #### What's Changed - feat(icons): added `message-circle-check` icon by [@​Shrinks99](https://redirect.github.com/Shrinks99) in [#​3770](https://redirect.github.com/lucide-icons/lucide/pull/3770) #### New Contributors - [@​Shrinks99](https://redirect.github.com/Shrinks99) made their first contribution in [#​3770](https://redirect.github.com/lucide-icons/lucide/pull/3770) **Full Changelog**: ### [`v0.571.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.571.0): Version 0.571.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.570.0...0.571.0) #### What's Changed - fix(icons): rearange `circle`-icons path and circle order by [@​adamlindqvist](https://redirect.github.com/adamlindqvist) in [#​3746](https://redirect.github.com/lucide-icons/lucide/pull/3746) - feat(icons): added `shelving-unit` icon by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​3041](https://redirect.github.com/lucide-icons/lucide/pull/3041) #### New Contributors - [@​adamlindqvist](https://redirect.github.com/adamlindqvist) made their first contribution in [#​3746](https://redirect.github.com/lucide-icons/lucide/pull/3746) **Full Changelog**: ### [`v0.570.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.570.0): Version 0.570.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.569.0...0.570.0) #### What's Changed - feat(icons): added `towel-rack` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3350](https://redirect.github.com/lucide-icons/lucide/pull/3350) **Full Changelog**: ### [`v0.569.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.569.0): Version 0.569.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.568.0...0.569.0) #### What's Changed - fix(icons): changed `clipboard-pen` icon by [@​Spleefies](https://redirect.github.com/Spleefies) in [#​4006](https://redirect.github.com/lucide-icons/lucide/pull/4006) - feat(icons): add `mirror-round` and `mirror-rectangular` by [@​Muhammad-Aqib-Bashir](https://redirect.github.com/Muhammad-Aqib-Bashir) in [#​3832](https://redirect.github.com/lucide-icons/lucide/pull/3832) **Full Changelog**: ### [`v0.568.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.568.0): Version 0.568.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.567.0...0.568.0) #### What's Changed - fix(icons): adjusted `clapperboard` so slash is no longer protruding by [@​torfmuer](https://redirect.github.com/torfmuer) in [#​3764](https://redirect.github.com/lucide-icons/lucide/pull/3764) - feat(icons): Add `git-merge-conflict` icon by [@​timmy471](https://redirect.github.com/timmy471) in [#​3008](https://redirect.github.com/lucide-icons/lucide/pull/3008) #### New Contributors - [@​torfmuer](https://redirect.github.com/torfmuer) made their first contribution in [#​3764](https://redirect.github.com/lucide-icons/lucide/pull/3764) - [@​timmy471](https://redirect.github.com/timmy471) made their first contribution in [#​3008](https://redirect.github.com/lucide-icons/lucide/pull/3008) **Full Changelog**: ### [`v0.567.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.567.0): Version 0.567.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.566.0...0.567.0) #### What's Changed - chore(tags): added tags to `info` by [@​jamiemlaw](https://redirect.github.com/jamiemlaw) in [#​4047](https://redirect.github.com/lucide-icons/lucide/pull/4047) - fix(icons): changed `gift` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3977](https://redirect.github.com/lucide-icons/lucide/pull/3977) - feat(icons): added `line-dot-right-horizontal` icon by [@​nathan-de-pachtere](https://redirect.github.com/nathan-de-pachtere) in [#​3742](https://redirect.github.com/lucide-icons/lucide/pull/3742) **Full Changelog**: ### [`v0.566.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.566.0): Version 0.566.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.565.0...0.566.0) #### What's Changed - fix(icons): changed `forklift` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​4069](https://redirect.github.com/lucide-icons/lucide/pull/4069) - fix(icons): changed `rocket` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​4067](https://redirect.github.com/lucide-icons/lucide/pull/4067) - feat(icons): added `globe-off` icon by [@​TimNekk](https://redirect.github.com/TimNekk) in [#​4051](https://redirect.github.com/lucide-icons/lucide/pull/4051) #### New Contributors - [@​TimNekk](https://redirect.github.com/TimNekk) made their first contribution in [#​4051](https://redirect.github.com/lucide-icons/lucide/pull/4051) **Full Changelog**: ### [`v0.565.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.565.0): Version 0.565.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.564.0...0.565.0) #### What's Changed - feat(icons): add `lens-concave` and `lens-convex` by [@​Muhammad-Aqib-Bashir](https://redirect.github.com/Muhammad-Aqib-Bashir) in [#​3831](https://redirect.github.com/lucide-icons/lucide/pull/3831) **Full Changelog**: ### [`v0.564.0`](https://redirect.github.com/lucide-icons/lucide/releases/tag/0.564.0): Version 0.564.0 [Compare Source](https://redirect.github.com/lucide-icons/lucide/compare/0.563.0...0.564.0) #### What's Changed - chore(docs): Improve SEO icon detail pages by [@​ericfennis](https://redirect.github.com/ericfennis) in [#​4040](https://redirect.github.com/lucide-icons/lucide/pull/4040) - feat(icons): added `database-search` icon by [@​Spleefies](https://redirect.github.com/Spleefies) in [#​4003](https://redirect.github.com/lucide-icons/lucide/pull/4003) - fix(icons): changed `user-lock` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3971](https://redirect.github.com/lucide-icons/lucide/pull/3971) - fix(icons): changed `bug-off` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3972](https://redirect.github.com/lucide-icons/lucide/pull/3972) - fix(icons): changed `bell-dot` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3973](https://redirect.github.com/lucide-icons/lucide/pull/3973) - fix(icons): changed `bandage` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3967](https://redirect.github.com/lucide-icons/lucide/pull/3967) - fix(icons): changed `hard-drive` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3622](https://redirect.github.com/lucide-icons/lucide/pull/3622) - fix(icons): changed `git-branch` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3938](https://redirect.github.com/lucide-icons/lucide/pull/3938) - fix(icons): changed `file-cog` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3965](https://redirect.github.com/lucide-icons/lucide/pull/3965) - fix(icons): changed `cloud-alert` and `cloud-check` icon by [@​jguddas](https://redirect.github.com/jguddas) in [#​3976](https://redirect.github.com/lucide-icons/lucide/pull/3976) - feat(icons): adds `user-key` and `user-round-key`, updates other `-key` icons to match by [@​karsa-mistmere](https://redirect.github.com/karsa-mistmere) in [#​4044](https://redirect.github.com/lucide-icons/lucide/pull/4044) #### New Contributors - [@​Spleefies](https://redirect.github.com/Spleefies) made their first contribution in [#​4003](https://redirect.github.com/lucide-icons/lucide/pull/4003) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- benchmarks-website/package-lock.json | 8 ++++---- benchmarks-website/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks-website/package-lock.json b/benchmarks-website/package-lock.json index 9fbe9f01769..0c826ef1d73 100644 --- a/benchmarks-website/package-lock.json +++ b/benchmarks-website/package-lock.json @@ -12,7 +12,7 @@ "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", "hammerjs": "^2.0.8", - "lucide-react": "^0.563.0", + "lucide-react": "^0.577.0", "react": "^18.3.1", "react-chartjs-2": "^5.2.0", "react-dom": "^18.3.1" @@ -1759,9 +1759,9 @@ } }, "node_modules/lucide-react": { - "version": "0.563.0", - "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.563.0.tgz", - "integrity": "sha512-8dXPB2GI4dI8jV4MgUDGBeLdGk8ekfqVZ0BdLcrRzocGgG75ltNEmWS+gE7uokKF/0oSUuczNDT+g9hFJ23FkA==", + "version": "0.577.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.577.0.tgz", + "integrity": "sha512-4LjoFv2eEPwYDPg/CUdBJQSDfPyzXCRrVW1X7jrx/trgxnxkHFjnVZINbzvzxjN70dxychOfg+FTYwBiS3pQ5A==", "license": "ISC", "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" diff --git a/benchmarks-website/package.json b/benchmarks-website/package.json index 9c1bcc623d2..6cda687b838 100644 --- a/benchmarks-website/package.json +++ b/benchmarks-website/package.json @@ -17,7 +17,7 @@ "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", "hammerjs": "^2.0.8", - "lucide-react": "^0.563.0", + "lucide-react": "^0.577.0", "react": "^18.3.1", "react-chartjs-2": "^5.2.0", "react-dom": "^18.3.1" From ea16b26e9e80ff64ea3924f4f136f10261ec4fcd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:49:27 +0000 Subject: [PATCH 119/250] Update dependency typescript to ~5.9.0 (#7541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [typescript](https://www.typescriptlang.org/) ([source](https://redirect.github.com/microsoft/TypeScript)) | [`~5.7.0` → `~5.9.0`](https://renovatebot.com/diffs/npm/typescript/5.7.3/5.9.3) | ![age](https://developer.mend.io/api/mc/badges/age/npm/typescript/5.9.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/typescript/5.7.3/5.9.3?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
microsoft/TypeScript (typescript) ### [`v5.9.3`](https://redirect.github.com/microsoft/TypeScript/releases/tag/v5.9.3): TypeScript 5.9.3 [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v5.9.2...v5.9.3) Note: this tag was recreated to point at the correct commit. The npm package contained the correct content. For release notes, check out the [release announcement](https://devblogs.microsoft.com/typescript/announcing-typescript-5-9/) - [fixed issues query for Typescript 5.9.0 (Beta)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.9.0%22+is%3Aclosed+). - [fixed issues query for Typescript 5.9.1 (RC)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.9.1%22+is%3Aclosed+). - *No specific changes for TypeScript 5.9.2 (Stable)* - [fixed issues query for Typescript 5.9.3 (Stable)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.9.3%22+is%3Aclosed+). Downloads are available on: - [npm](https://www.npmjs.com/package/typescript) ### [`v5.9.2`](https://redirect.github.com/microsoft/TypeScript/releases/tag/v5.9.2): TypeScript 5.9 [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v5.8.3...v5.9.2) Note: this tag was recreated to point at the correct commit. The npm package contained the correct content. For release notes, check out the [release announcement](https://devblogs.microsoft.com/typescript/announcing-typescript-5-9/) - [fixed issues query for Typescript 5.9.0 (Beta)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.9.0%22+is%3Aclosed+). - [fixed issues query for Typescript 5.9.1 (RC)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.9.1%22+is%3Aclosed+). - *No specific changes for TypeScript 5.9.2 (Stable)* Downloads are available on: - [npm](https://www.npmjs.com/package/typescript) ### [`v5.8.3`](https://redirect.github.com/microsoft/TypeScript/releases/tag/v5.8.3): TypeScript 5.8.3 [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v5.8.2...v5.8.3) Note: this tag was recreated to point at the correct commit. The npm package contained the correct content. For release notes, check out the [release announcement](https://devblogs.microsoft.com/typescript/announcing-typescript-5-8/). - [fixed issues query for Typescript 5.8.0 (Beta)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.0%22+is%3Aclosed+). - [fixed issues query for Typescript 5.8.1 (RC)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.1%22+is%3Aclosed+). - [fixed issues query for Typescript 5.8.2 (Stable)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.2%22+is%3Aclosed+). - [fixed issues query for Typescript 5.8.3 (Stable)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.3%22+is%3Aclosed+). Downloads are available on: - [npm](https://www.npmjs.com/package/typescript) ### [`v5.8.2`](https://redirect.github.com/microsoft/TypeScript/releases/tag/v5.8.2): TypeScript 5.8 [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v5.7.3...v5.8.2) For release notes, check out the [release announcement](https://devblogs.microsoft.com/typescript/announcing-typescript-5-8/). - [fixed issues query for Typescript 5.8.0 (Beta)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.0%22+is%3Aclosed+). - [fixed issues query for Typescript 5.8.1 (RC)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.1%22+is%3Aclosed+). - [fixed issues query for Typescript 5.8.2 (Stable)](https://redirect.github.com/Microsoft/TypeScript/issues?utf8=%E2%9C%93\&q=milestone%3A%22TypeScript+5.8.2%22+is%3Aclosed+). Downloads are available on: - [npm](https://www.npmjs.com/package/typescript)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 8 ++++---- vortex-web/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index c610f592e9f..2ed4bf845ee 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -33,7 +33,7 @@ "prettier": "^3.8.1", "storybook": "^10.3.3", "tailwindcss": "^4.1.4", - "typescript": "~5.7.0", + "typescript": "~5.9.0", "typescript-eslint": "^8.30.1", "vite": "^6.3.2" } @@ -5177,9 +5177,9 @@ } }, "node_modules/typescript": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", - "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/vortex-web/package.json b/vortex-web/package.json index 76de26d444d..55740faee04 100644 --- a/vortex-web/package.json +++ b/vortex-web/package.json @@ -44,7 +44,7 @@ "prettier": "^3.8.1", "storybook": "^10.3.3", "tailwindcss": "^4.1.4", - "typescript": "~5.7.0", + "typescript": "~5.9.0", "typescript-eslint": "^8.30.1", "vite": "^6.3.2" } From c1f634530b2a541e28819a8241a524542ec42246 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:49:48 +0000 Subject: [PATCH 120/250] Update dependency typescript-eslint to v8.58.2 (#7542) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [typescript-eslint](https://typescript-eslint.io/packages/typescript-eslint) ([source](https://redirect.github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/typescript-eslint)) | [`8.57.2` → `8.58.2`](https://renovatebot.com/diffs/npm/typescript-eslint/8.57.2/8.58.2) | ![age](https://developer.mend.io/api/mc/badges/age/npm/typescript-eslint/8.58.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/typescript-eslint/8.57.2/8.58.2?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
typescript-eslint/typescript-eslint (typescript-eslint) ### [`v8.58.2`](https://redirect.github.com/typescript-eslint/typescript-eslint/blob/HEAD/packages/typescript-eslint/CHANGELOG.md#8582-2026-04-13) [Compare Source](https://redirect.github.com/typescript-eslint/typescript-eslint/compare/v8.58.1...v8.58.2) ##### 🩹 Fixes - remove tsbuildinfo cache file from published packages ([#​12187](https://redirect.github.com/typescript-eslint/typescript-eslint/pull/12187)) ##### ❤️ Thank You - Abhijeet Singh [@​cseas](https://redirect.github.com/cseas) See [GitHub Releases](https://redirect.github.com/typescript-eslint/typescript-eslint/releases/tag/v8.58.2) for more information. You can read about our [versioning strategy](https://typescript-eslint.io/users/versioning) and [releases](https://typescript-eslint.io/users/releases) on our website. ### [`v8.58.1`](https://redirect.github.com/typescript-eslint/typescript-eslint/blob/HEAD/packages/typescript-eslint/CHANGELOG.md#8581-2026-04-08) [Compare Source](https://redirect.github.com/typescript-eslint/typescript-eslint/compare/v8.58.0...v8.58.1) This was a version bump only for typescript-eslint to align it with other projects, there were no code changes. See [GitHub Releases](https://redirect.github.com/typescript-eslint/typescript-eslint/releases/tag/v8.58.1) for more information. You can read about our [versioning strategy](https://typescript-eslint.io/users/versioning) and [releases](https://typescript-eslint.io/users/releases) on our website. ### [`v8.58.0`](https://redirect.github.com/typescript-eslint/typescript-eslint/blob/HEAD/packages/typescript-eslint/CHANGELOG.md#8580-2026-03-30) [Compare Source](https://redirect.github.com/typescript-eslint/typescript-eslint/compare/v8.57.2...v8.58.0) ##### 🚀 Features - support TypeScript 6 ([#​12124](https://redirect.github.com/typescript-eslint/typescript-eslint/pull/12124)) ##### ❤️ Thank You - Evyatar Daud [@​StyleShit](https://redirect.github.com/StyleShit) See [GitHub Releases](https://redirect.github.com/typescript-eslint/typescript-eslint/releases/tag/v8.58.0) for more information. You can read about our [versioning strategy](https://typescript-eslint.io/users/versioning) and [releases](https://typescript-eslint.io/users/releases) on our website.
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 152 +++++++++++++++++------------------ 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 2ed4bf845ee..762acb6af94 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -2246,20 +2246,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.2.tgz", - "integrity": "sha512-NZZgp0Fm2IkD+La5PR81sd+g+8oS6JwJje+aRWsDocxHkjyRw0J5L5ZTlN3LI1LlOcGL7ph3eaIUmTXMIjLk0w==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.2.tgz", + "integrity": "sha512-aC2qc5thQahutKjP+cl8cgN9DWe3ZUqVko30CMSZHnFEHyhOYoZSzkGtAI2mcwZ38xeImDucI4dnqsHiOYuuCw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/type-utils": "8.57.2", - "@typescript-eslint/utils": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/type-utils": "8.58.2", + "@typescript-eslint/utils": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2269,9 +2269,9 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.2", + "@typescript-eslint/parser": "^8.58.2", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { @@ -2285,16 +2285,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.2.tgz", - "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.2.tgz", + "integrity": "sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "debug": "^4.4.3" }, "engines": { @@ -2306,18 +2306,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.2.tgz", - "integrity": "sha512-FuH0wipFywXRTHf+bTTjNyuNQQsQC3qh/dYzaM4I4W0jrCqjCVuUh99+xd9KamUfmCGPvbO8NDngo/vsnNVqgw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.2.tgz", + "integrity": "sha512-Cq6UfpZZk15+r87BkIh5rDpi38W4b+Sjnb8wQCPPDDweS/LRCFjCyViEbzHk5Ck3f2QDfgmlxqSa7S7clDtlfg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.2", - "@typescript-eslint/types": "^8.57.2", + "@typescript-eslint/tsconfig-utils": "^8.58.2", + "@typescript-eslint/types": "^8.58.2", "debug": "^4.4.3" }, "engines": { @@ -2328,18 +2328,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.2.tgz", - "integrity": "sha512-snZKH+W4WbWkrBqj4gUNRIGb/jipDW3qMqVJ4C9rzdFc+wLwruxk+2a5D+uoFcKPAqyqEnSb4l2ULuZf95eSkw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.2.tgz", + "integrity": "sha512-SgmyvDPexWETQek+qzZnrG6844IaO02UVyOLhI4wpo82dpZJY9+6YZCKAMFzXb7qhx37mFK1QcPQ18tud+vo6Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2" + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2350,9 +2350,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.2.tgz", - "integrity": "sha512-3Lm5DSM+DCowsUOJC+YqHHnKEfFh5CoGkj5Z31NQSNF4l5wdOwqGn99wmwN/LImhfY3KJnmordBq/4+VDe2eKw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.2.tgz", + "integrity": "sha512-3SR+RukipDvkkKp/d0jP0dyzuls3DbGmwDpVEc5wqk5f38KFThakqAAO0XMirWAE+kT00oTauTbzMFGPoAzB0A==", "dev": true, "license": "MIT", "engines": { @@ -2363,21 +2363,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.2.tgz", - "integrity": "sha512-Co6ZCShm6kIbAM/s+oYVpKFfW7LBc6FXoPXjTRQ449PPNBY8U0KZXuevz5IFuuUj2H9ss40atTaf9dlGLzbWZg==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.2.tgz", + "integrity": "sha512-Z7EloNR/B389FvabdGeTo2XMs4W9TjtPiO9DAsmT0yom0bwlPyRjkJ1uCdW1DvrrrYP50AJZ9Xc3sByZA9+dcg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2", + "@typescript-eslint/utils": "8.58.2", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2388,13 +2388,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.2.tgz", - "integrity": "sha512-/iZM6FnM4tnx9csuTxspMW4BOSegshwX5oBDznJ7S4WggL7Vczz5d2W11ecc4vRrQMQHXRSxzrCsyG5EsPPTbA==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.2.tgz", + "integrity": "sha512-9TukXyATBQf/Jq9AMQXfvurk+G5R2MwfqQGDR2GzGz28HvY/lXNKGhkY+6IOubwcquikWk5cjlgPvD2uAA7htQ==", "dev": true, "license": "MIT", "engines": { @@ -2406,21 +2406,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.2.tgz", - "integrity": "sha512-2MKM+I6g8tJxfSmFKOnHv2t8Sk3T6rF20A1Puk0svLK+uVapDZB/4pfAeB7nE83uAZrU6OxW+HmOd5wHVdXwXA==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.2.tgz", + "integrity": "sha512-ELGuoofuhhoCvNbQjFFiobFcGgcDCEm0ThWdmO4Z0UzLqPXS3KFvnEZ+SHewwOYHjM09tkzOWXNTv9u6Gqtyuw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.2", - "@typescript-eslint/tsconfig-utils": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/project-service": "8.58.2", + "@typescript-eslint/tsconfig-utils": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/visitor-keys": "8.58.2", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2430,7 +2430,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { @@ -2457,13 +2457,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "brace-expansion": "^5.0.5" }, "engines": { "node": "18 || 20 || >=22" @@ -2486,16 +2486,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.2.tgz", - "integrity": "sha512-krRIbvPK1ju1WBKIefiX+bngPs+odIQUtR7kymzPfo1POVw3jlF+nLkmexdSSd4UCbDcQn+wMBATOOmpBbqgKg==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.2.tgz", + "integrity": "sha512-QZfjHNEzPY8+l0+fIXMvuQ2sJlplB4zgDZvA+NmvZsZv3EQwOcc1DuIU1VJUTWZ/RKouBMhDyNaBMx4sWvrzRA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2" + "@typescript-eslint/scope-manager": "8.58.2", + "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2506,17 +2506,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.2.tgz", - "integrity": "sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.2.tgz", + "integrity": "sha512-f1WO2Lx8a9t8DARmcWAUPJbu0G20bJlj8L4z72K00TMeJAoyLr/tHhI/pzYBLrR4dXWkcxO1cWYZEOX8DKHTqA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", + "@typescript-eslint/types": "8.58.2", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -5191,16 +5191,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.2.tgz", - "integrity": "sha512-VEPQ0iPgWO/sBaZOU1xo4nuNdODVOajPnTIbog2GKYr31nIlZ0fWPoCQgGfF3ETyBl1vn63F/p50Um9Z4J8O8A==", + "version": "8.58.2", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.58.2.tgz", + "integrity": "sha512-V8iSng9mRbdZjl54VJ9NKr6ZB+dW0J3TzRXRGcSbLIej9jV86ZRtlYeTKDR/QLxXykocJ5icNzbsl2+5TzIvcQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.57.2", - "@typescript-eslint/parser": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2" + "@typescript-eslint/eslint-plugin": "8.58.2", + "@typescript-eslint/parser": "8.58.2", + "@typescript-eslint/typescript-estree": "8.58.2", + "@typescript-eslint/utils": "8.58.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -5211,7 +5211,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/typical": { From 8f8688171440d841f589328f7a2248241f2e949c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:49:59 +0000 Subject: [PATCH 121/250] Update release-drafter/release-drafter action to v7.2.0 (#7544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [release-drafter/release-drafter](https://redirect.github.com/release-drafter/release-drafter) | action | minor | `v7.1.1` → `v7.2.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
release-drafter/release-drafter (release-drafter/release-drafter) ### [`v7.2.0`](https://redirect.github.com/release-drafter/release-drafter/releases/tag/v7.2.0) [Compare Source](https://redirect.github.com/release-drafter/release-drafter/compare/v7.1.1...v7.2.0) ### What's Changed #### New - feat: allow always collapsing a category ([#​1444](https://redirect.github.com/release-drafter/release-drafter/issues/1444)) [@​mhanberg](https://redirect.github.com/mhanberg) #### Bug Fixes - fix: improve advanced substitutions in replacers ([#​1555](https://redirect.github.com/release-drafter/release-drafter/issues/1555)) [@​jetersen](https://redirect.github.com/jetersen) - fix: support repo-only \_extends and prevent .github/ path doubling ([#​1577](https://redirect.github.com/release-drafter/release-drafter/issues/1577)) [@​jetersen](https://redirect.github.com/jetersen) #### Maintenance - chore(deps): update dependency typescript to 6.0.2 ([#​1587](https://redirect.github.com/release-drafter/release-drafter/issues/1587)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update vitest to 4.1.4 ([#​1585](https://redirect.github.com/release-drafter/release-drafter/issues/1585)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - ci(deps): update peter-evans/create-pull-request action to v8 ([#​1588](https://redirect.github.com/release-drafter/release-drafter/issues/1588)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency vite to 8.0.5 ([#​1579](https://redirect.github.com/release-drafter/release-drafter/issues/1579)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency nock to 14.0.12 ([#​1583](https://redirect.github.com/release-drafter/release-drafter/issues/1583)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency [@​types/node](https://redirect.github.com/types/node) to 24.12.2 ([#​1582](https://redirect.github.com/release-drafter/release-drafter/issues/1582)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency [@​biomejs/biome](https://redirect.github.com/biomejs/biome) to 2.4.10 ([#​1581](https://redirect.github.com/release-drafter/release-drafter/issues/1581)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore: move codegen to monthly scheduled workflow ([#​1578](https://redirect.github.com/release-drafter/release-drafter/issues/1578)) [@​jetersen](https://redirect.github.com/jetersen) - chore: replace vite-tsconfig-paths plugin with native resolve.tsconfigPaths ([#​1571](https://redirect.github.com/release-drafter/release-drafter/issues/1571)) [@​jetersen](https://redirect.github.com/jetersen) #### Documentation - docs: fix autolabeler example tag ([#​1568](https://redirect.github.com/release-drafter/release-drafter/issues/1568)) [@​cchanche](https://redirect.github.com/cchanche) #### Dependency Updates - build(deps): bump lodash and [@​graphql-codegen/plugin-helpers](https://redirect.github.com/graphql-codegen/plugin-helpers) ([#​1589](https://redirect.github.com/release-drafter/release-drafter/issues/1589)) @​[dependabot\[bot\]](https://redirect.github.com/apps/dependabot) - fix(deps): update dependency [@​actions/github](https://redirect.github.com/actions/github) to 9.1.0 ([#​1586](https://redirect.github.com/release-drafter/release-drafter/issues/1586)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency yaml to 2.8.3 ([#​1580](https://redirect.github.com/release-drafter/release-drafter/issues/1580)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update node.js to v24.14.1 ([#​1584](https://redirect.github.com/release-drafter/release-drafter/issues/1584)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) - chore(deps): update dependency [@​biomejs/biome](https://redirect.github.com/biomejs/biome) to 2.4.10 ([#​1581](https://redirect.github.com/release-drafter/release-drafter/issues/1581)) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/release-drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index f87f34483b2..e9e0af00017 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: release-drafter/release-drafter@v7.1.1 + - uses: release-drafter/release-drafter@v7.2.0 with: commitish: ${{ github.ref_name }} prerelease: ${{ github.event_name == 'workflow_dispatch' }} From b66f9e46d958321768477ab47f0137d87360a01a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 02:51:46 +0000 Subject: [PATCH 122/250] Update dependency com.google.guava:guava to v33.6.0-jre (#7535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [com.google.guava:guava](https://redirect.github.com/google/guava) | `33.5.0-jre` → `33.6.0-jre` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.google.guava:guava/33.6.0-jre?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.google.guava:guava/33.5.0-jre/33.6.0-jre?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index 1dc730f7b1d..e5b726aaa45 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -4,7 +4,7 @@ [versions] arrow = "18.3.0" errorprone = "2.36.0" -guava = "33.5.0-jre" +guava = "33.6.0-jre" immutables = "2.12.1" junit-jupiter = "6.0.3" logback = "1.5.32" From 92c573b8371f6f897a5291e7e6070fcd48a7ef8f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 03:06:11 +0000 Subject: [PATCH 123/250] Update arrow-rs to v58.1.0 (#7531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [arrow-arith](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-array](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-buffer](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-cast](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-data](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-ipc](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-ord](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-schema](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-select](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [arrow-string](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [parquet](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [parquet-variant](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | | [parquet-variant-compute](https://redirect.github.com/apache/arrow-rs) | workspace.dependencies | minor | `58.0.0` → `58.1.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
apache/arrow-rs (arrow-arith) ### [`v58.1.0`](https://redirect.github.com/apache/arrow-rs/blob/HEAD/CHANGELOG.md#5810-2026-03-20) [Compare Source](https://redirect.github.com/apache/arrow-rs/compare/58.0.0...58.1.0) [Full Changelog](https://redirect.github.com/apache/arrow-rs/compare/58.0.0...58.1.0) **Implemented enhancements:** - Reuse compression dict lz4\_block [#​9566](https://redirect.github.com/apache/arrow-rs/issues/9566) - \[Variant] Add `variant_to_arrow` `Struct` type support [#​9529](https://redirect.github.com/apache/arrow-rs/issues/9529) - \[Variant] Add `unshred_variant` support for `Binary` and `LargeBinary` types [#​9526](https://redirect.github.com/apache/arrow-rs/issues/9526) - \[Variant] Add `shred_variant` support for `LargeUtf8` and `LargeBinary` types [#​9525](https://redirect.github.com/apache/arrow-rs/issues/9525) - \[Variant] `variant_get` tests clean up [#​9517](https://redirect.github.com/apache/arrow-rs/issues/9517) - parquet\_variant: Support LargeUtf8 typed value in `unshred_variant` [#​9513](https://redirect.github.com/apache/arrow-rs/issues/9513) - parquet-variant: Support string view typed value in `unshred_variant` [#​9512](https://redirect.github.com/apache/arrow-rs/issues/9512) - Deprecate ArrowTimestampType::make\_value in favor of from\_naive\_datetime [#​9490](https://redirect.github.com/apache/arrow-rs/issues/9490) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Followup for support \['fieldName'] in VariantPath [#​9478](https://redirect.github.com/apache/arrow-rs/issues/9478) - Speedup DELTA\_BINARY\_PACKED decoding when bitwidth is 0 [#​9476](https://redirect.github.com/apache/arrow-rs/issues/9476) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] - Support CSV files encoded with charsets other than UTF-8 [#​9465](https://redirect.github.com/apache/arrow-rs/issues/9465) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Expose Avro writer schema when building the reader [#​9460](https://redirect.github.com/apache/arrow-rs/issues/9460) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Python: avoid importing pyarrow classes ever time [#​9438](https://redirect.github.com/apache/arrow-rs/issues/9438) - Add `append_nulls` to `MapBuilder` [#​9431](https://redirect.github.com/apache/arrow-rs/issues/9431) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Add `append_non_nulls` to `StructBuilder` [#​9429](https://redirect.github.com/apache/arrow-rs/issues/9429) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Add `append_value_n` to GenericByteBuilder [#​9425](https://redirect.github.com/apache/arrow-rs/issues/9425) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Optimize `from_bitwise_binary_op` [#​9378](https://redirect.github.com/apache/arrow-rs/issues/9378) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Configurable Arrow representation of UTC timestamps for Avro reader [#​9279](https://redirect.github.com/apache/arrow-rs/issues/9279) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] **Fixed bugs:** - MutableArrayData::extend does not copy child values for ListView arrays [#​9561](https://redirect.github.com/apache/arrow-rs/issues/9561) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - ListView interleave bug [#​9559](https://redirect.github.com/apache/arrow-rs/issues/9559) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Flight encoding panics with "no dict id for field" with nested dict arrays [#​9555](https://redirect.github.com/apache/arrow-rs/issues/9555) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] \[[arrow-flight](https://redirect.github.com/apache/arrow-rs/labels/arrow-flight)] - "DeltaBitPackDecoder only supports Int32Type and Int64Type" but unsigned types are supported too [#​9551](https://redirect.github.com/apache/arrow-rs/issues/9551) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] - Potential overflow when calling `util::bit_mask::set_bits` (soundness issue) [#​9543](https://redirect.github.com/apache/arrow-rs/issues/9543) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - handle Null type in try\_merge for Struct, List, LargeList, and Union [#​9523](https://redirect.github.com/apache/arrow-rs/issues/9523) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Invalid offset in sparse column chunk data for multiple predicates [#​9516](https://redirect.github.com/apache/arrow-rs/issues/9516) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] - debug\_assert\_eq! in BatchCoalescer panics in debug mode when batch\_size < 4 [#​9506](https://redirect.github.com/apache/arrow-rs/issues/9506) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - Parquet Statistics::null\_count\_opt wrongly returns Some(0) when stats are missing [#​9451](https://redirect.github.com/apache/arrow-rs/issues/9451) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] - Error "Not all children array length are the same!" when decoding rows spanning across page boundaries in parquet file when using `RowSelection` [#​9370](https://redirect.github.com/apache/arrow-rs/issues/9370) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] - Avro schema resolution not properly supported for complex types [#​9336](https://redirect.github.com/apache/arrow-rs/issues/9336) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] **Documentation updates:** - Update planned release schedule in README.md [#​9466](https://redirect.github.com/apache/arrow-rs/pull/9466) ([alamb](https://redirect.github.com/alamb)) **Performance improvements:** - Introduce `NullBuffer::try_from_unsliced` to simplify array construction [#​9385](https://redirect.github.com/apache/arrow-rs/issues/9385) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] - perf: Coalesce page fetches when RowSelection selects all rows [#​9578](https://redirect.github.com/apache/arrow-rs/pull/9578) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://redirect.github.com/Dandandan)) - Use chunks\_exact for has\_true/has\_false to enable compiler unrolling [#​9570](https://redirect.github.com/apache/arrow-rs/pull/9570) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([adriangb](https://redirect.github.com/adriangb)) - pyarrow: Cache the imported classes to avoid importing them each time [#​9439](https://redirect.github.com/apache/arrow-rs/pull/9439) ([Tpt](https://redirect.github.com/Tpt)) **Closed issues:** - Duplicate macro definition: `partially_shredded_variant_array_gen` [#​9492](https://redirect.github.com/apache/arrow-rs/issues/9492) - Enable `LargeList` / `ListView` / `LargeListView` for `VariantArray::try_new` [#​9455](https://redirect.github.com/apache/arrow-rs/issues/9455) - Support variables/expressions in record\_batch! macro [#​9245](https://redirect.github.com/apache/arrow-rs/issues/9245) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] **Merged pull requests:** - \[Variant] Add unshred\_variant support for Binary and LargeBinary types [#​9576](https://redirect.github.com/apache/arrow-rs/pull/9576) ([kunalsinghdadhwal](https://redirect.github.com/kunalsinghdadhwal)) - \[Variant] Add `variant_to_arrow` `Struct` type support [#​9572](https://redirect.github.com/apache/arrow-rs/pull/9572) ([sdf-jkl](https://redirect.github.com/sdf-jkl)) - Make Sbbf Constructers Public [#​9569](https://redirect.github.com/apache/arrow-rs/pull/9569) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([cetra3](https://redirect.github.com/cetra3)) - fix: Used `checked_add` for bounds checks to avoid UB [#​9568](https://redirect.github.com/apache/arrow-rs/pull/9568) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([etseidl](https://redirect.github.com/etseidl)) - Add mutable operations to BooleanBuffer (Bit\*Assign) [#​9567](https://redirect.github.com/apache/arrow-rs/pull/9567) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://redirect.github.com/Dandandan)) - chore(deps): update lz4\_flex requirement from 0.12 to 0.13 [#​9565](https://redirect.github.com/apache/arrow-rs/pull/9565) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([dependabot\[bot\]](https://redirect.github.com/apps/dependabot)) - arrow-select: fix MutableArrayData interleave for ListView [#​9560](https://redirect.github.com/apache/arrow-rs/pull/9560) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://redirect.github.com/asubiotto)) - Move `ValueIter` into own module, and add public `record_count` function [#​9557](https://redirect.github.com/apache/arrow-rs/pull/9557) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://redirect.github.com/Rafferty97)) - arrow-flight: generate dict\_ids for dicts nested inside complex types [#​9556](https://redirect.github.com/apache/arrow-rs/pull/9556) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] \[[arrow-flight](https://redirect.github.com/apache/arrow-rs/labels/arrow-flight)] ([asubiotto](https://redirect.github.com/asubiotto)) - add `shred_variant` support for `LargeUtf8` and `LargeBinary` [#​9554](https://redirect.github.com/apache/arrow-rs/pull/9554) ([sdf-jkl](https://redirect.github.com/sdf-jkl)) - \[minor] Download clickbench file when missing [#​9553](https://redirect.github.com/apache/arrow-rs/pull/9553) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://redirect.github.com/Dandandan)) - DeltaBitPackEncoderConversion: Fix panic message on invalid type [#​9552](https://redirect.github.com/apache/arrow-rs/pull/9552) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([progval](https://redirect.github.com/progval)) - Replace interleave overflow panic with error [#​9549](https://redirect.github.com/apache/arrow-rs/pull/9549) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([xudong963](https://redirect.github.com/xudong963)) - feat(arrow-avro): `HeaderInfo` to expose OCF header [#​9548](https://redirect.github.com/apache/arrow-rs/pull/9548) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://redirect.github.com/mzabaluev)) - chore: Protect `main` branch with required reviews [#​9547](https://redirect.github.com/apache/arrow-rs/pull/9547) ([comphead](https://redirect.github.com/comphead)) - Add benchmark for `infer_json_schema` [#​9546](https://redirect.github.com/apache/arrow-rs/pull/9546) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://redirect.github.com/Rafferty97)) - chore(deps): bump black from 24.3.0 to 26.3.1 in /parquet/pytest [#​9545](https://redirect.github.com/apache/arrow-rs/pull/9545) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([dependabot\[bot\]](https://redirect.github.com/apps/dependabot)) - Unroll interleave -25-30% [#​9542](https://redirect.github.com/apache/arrow-rs/pull/9542) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://redirect.github.com/Dandandan)) - Optimize `take_fixed_size_binary` For Predefined Value Lengths [#​9535](https://redirect.github.com/apache/arrow-rs/pull/9535) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([tobixdev](https://redirect.github.com/tobixdev)) - feat: expose arrow schema on async avro reader [#​9534](https://redirect.github.com/apache/arrow-rs/pull/9534) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://redirect.github.com/mzabaluev)) - Make with\_file\_decryption\_properties pub instead of pub(crate) [#​9532](https://redirect.github.com/apache/arrow-rs/pull/9532) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://redirect.github.com/Dandandan)) - fix: handle Null type in try\_merge for Struct, List, LargeList, and Union [#​9524](https://redirect.github.com/apache/arrow-rs/pull/9524) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([zhuqi-lucas](https://redirect.github.com/zhuqi-lucas)) - chore: extend record\_batch macro to support variables and expressions [#​9522](https://redirect.github.com/apache/arrow-rs/pull/9522) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([buraksenn](https://redirect.github.com/buraksenn)) - \[Variant] clean up `variant_get` tests [#​9518](https://redirect.github.com/apache/arrow-rs/pull/9518) ([sdf-jkl](https://redirect.github.com/sdf-jkl)) - support large string for unshred variant [#​9515](https://redirect.github.com/apache/arrow-rs/pull/9515) ([friendlymatthew](https://redirect.github.com/friendlymatthew)) - support string view unshred variant [#​9514](https://redirect.github.com/apache/arrow-rs/pull/9514) ([friendlymatthew](https://redirect.github.com/friendlymatthew)) - Add has\_true() and has\_false() to BooleanArray [#​9511](https://redirect.github.com/apache/arrow-rs/pull/9511) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([adriangb](https://redirect.github.com/adriangb)) - Fix Invalid offset in sparse column chunk data error for multiple predicates [#​9509](https://redirect.github.com/apache/arrow-rs/pull/9509) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([cetra3](https://redirect.github.com/cetra3)) - fix: remove incorrect debug assertion in BatchCoalescer [#​9508](https://redirect.github.com/apache/arrow-rs/pull/9508) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Tim-53](https://redirect.github.com/Tim-53)) - \[Json] Add benchmarks for list json reader [#​9507](https://redirect.github.com/apache/arrow-rs/pull/9507) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://redirect.github.com/liamzwbao)) - fix: first next\_back() on new RowsIter panics [#​9505](https://redirect.github.com/apache/arrow-rs/pull/9505) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://redirect.github.com/rluvaton)) - Add some benchmarks for decoding delta encoded Parquet [#​9500](https://redirect.github.com/apache/arrow-rs/pull/9500) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://redirect.github.com/etseidl)) - chore: remove duplicate macro `partially_shredded_variant_array_gen` [#​9498](https://redirect.github.com/apache/arrow-rs/pull/9498) ([codephage2020](https://redirect.github.com/codephage2020)) - Deprecate ArrowTimestampType::make\_value in favor of from\_naive\_datetime [#​9491](https://redirect.github.com/apache/arrow-rs/pull/9491) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://redirect.github.com/codephage2020)) - fix: Do not assume missing nullcount stat means zero nullcount [#​9481](https://redirect.github.com/apache/arrow-rs/pull/9481) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([scovich](https://redirect.github.com/scovich)) - \[Variant] Enahcne bracket access for VariantPath [#​9479](https://redirect.github.com/apache/arrow-rs/pull/9479) ([klion26](https://redirect.github.com/klion26)) - Optimize delta binary decoder in the case where bitwidth=0 [#​9477](https://redirect.github.com/apache/arrow-rs/pull/9477) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://redirect.github.com/etseidl)) - Add PrimitiveRunBuilder::with\_data\_type() to customize the values' DataType [#​9473](https://redirect.github.com/apache/arrow-rs/pull/9473) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://redirect.github.com/brunal)) - Convert `prettyprint` tests in `arrow-cast` to `insta` inline snapshots [#​9472](https://redirect.github.com/apache/arrow-rs/pull/9472) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([grtlr](https://redirect.github.com/grtlr)) - Update strum\_macros requirement from 0.27 to 0.28 [#​9471](https://redirect.github.com/apache/arrow-rs/pull/9471) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([dependabot\[bot\]](https://redirect.github.com/apps/dependabot)) - docs(parquet): Fix broken links in README [#​9467](https://redirect.github.com/apache/arrow-rs/pull/9467) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([SYaoJun](https://redirect.github.com/SYaoJun)) - Add list-like types support to VariantArray::try\_new [#​9457](https://redirect.github.com/apache/arrow-rs/pull/9457) ([sdf-jkl](https://redirect.github.com/sdf-jkl)) - Simplify downcast\_...!() macro definitions [#​9454](https://redirect.github.com/apache/arrow-rs/pull/9454) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://redirect.github.com/brunal)) - feat(parquet): add content defined chunking for arrow writer [#​9450](https://redirect.github.com/apache/arrow-rs/pull/9450) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://redirect.github.com/kszucs)) - refactor: simplify iterator using cloned().map(Some) [#​9449](https://redirect.github.com/apache/arrow-rs/pull/9449) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([SYaoJun](https://redirect.github.com/SYaoJun)) - feat: Optimize from\_bitwise\_binary\_op with 64-bit alignment [#​9441](https://redirect.github.com/apache/arrow-rs/pull/9441) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([kunalsinghdadhwal](https://redirect.github.com/kunalsinghdadhwal)) - docs: fix markdown link syntax in README [#​9440](https://redirect.github.com/apache/arrow-rs/pull/9440) ([SYaoJun](https://redirect.github.com/SYaoJun)) - Move `ListLikeArray` to arrow-array to be shared with json writer and parquet unshredding [#​9437](https://redirect.github.com/apache/arrow-rs/pull/9437) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://redirect.github.com/liamzwbao)) - Add `claim` method to recordbatch for memory accounting [#​9433](https://redirect.github.com/apache/arrow-rs/pull/9433) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://redirect.github.com/cetra3)) - Add `append_nulls` to `MapBuilder` [#​9432](https://redirect.github.com/apache/arrow-rs/pull/9432) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://redirect.github.com/Fokko)) - Add `append_non_nulls` to `StructBuilder` [#​9430](https://redirect.github.com/apache/arrow-rs/pull/9430) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://redirect.github.com/Fokko)) - Add `append_value_n` to GenericByteBuilder [#​9426](https://redirect.github.com/apache/arrow-rs/pull/9426) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://redirect.github.com/Fokko)) - refactor: simplify dynamic state for Avro record projection [#​9419](https://redirect.github.com/apache/arrow-rs/pull/9419) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://redirect.github.com/mzabaluev)) - Add `NullBuffer::from_unsliced_buffer` helper and refactor call sites [#​9411](https://redirect.github.com/apache/arrow-rs/pull/9411) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([Eyad3skr](https://redirect.github.com/Eyad3skr)) - Implement min, max, sum for run-end-encoded arrays. [#​9409](https://redirect.github.com/apache/arrow-rs/pull/9409) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://redirect.github.com/brunal)) - feat: add `RunArray::new_unchecked` and `RunArray::into_parts` [#​9376](https://redirect.github.com/apache/arrow-rs/pull/9376) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://redirect.github.com/rluvaton)) - Fix skip\_records over-counting when partial record precedes num\_rows page skip [#​9374](https://redirect.github.com/apache/arrow-rs/pull/9374) \[[parquet](https://redirect.github.com/apache/arrow-rs/labels/parquet)] ([jonded94](https://redirect.github.com/jonded94)) - fix: resolution of complex type variants in Avro unions [#​9328](https://redirect.github.com/apache/arrow-rs/pull/9328) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://redirect.github.com/mzabaluev)) - feat(arrow-avro): Configurable Arrow timezone ID for Avro timestamps [#​9280](https://redirect.github.com/apache/arrow-rs/pull/9280) \[[arrow](https://redirect.github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://redirect.github.com/mzabaluev)) \* *This Changelog was automatically generated by [github\_changelog\_generator](https://redirect.github.com/github-changelog-generator/github-changelog-generator)*
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 437 +++++++++++++++++++++++++++-------------------------- 1 file changed, 223 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26b0aa741d7..c3bb56183f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,23 +248,23 @@ dependencies = [ [[package]] name = "arrow" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ - "arrow-arith 58.0.0", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-cast 58.0.0", - "arrow-csv 58.0.0", - "arrow-data 58.0.0", - "arrow-ipc 58.0.0", - "arrow-json 58.0.0", - "arrow-ord 58.0.0", - "arrow-row 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", - "arrow-string 58.0.0", + "arrow-arith 58.1.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-csv 58.1.0", + "arrow-data 58.1.0", + "arrow-ipc 58.1.0", + "arrow-json 58.1.0", + "arrow-ord 58.1.0", + "arrow-row 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "arrow-string 58.1.0", ] [[package]] @@ -283,14 +283,14 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "chrono", "num-traits", ] @@ -316,14 +316,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash 0.8.12", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "chrono", "chrono-tz", "half", @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c1a85bb2e94ee10b76531d8bc3ce9b7b4c0d508cabfb17d477f63f2617bd20" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -381,16 +381,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-ord 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-ord 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", "atoi", "base64", "chrono", @@ -418,13 +418,13 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ - "arrow-array 58.0.0", - "arrow-cast 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-cast 58.1.0", + "arrow-schema 58.1.0", "chrono", "csv", "csv-core", @@ -446,12 +446,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ - "arrow-buffer 58.0.0", - "arrow-schema 58.0.0", + "arrow-buffer 58.1.0", + "arrow-schema 58.1.0", "half", "num-integer", "num-traits", @@ -475,17 +475,17 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", "flatbuffers", - "lz4_flex 0.12.1", + "lz4_flex 0.13.0", "zstd", ] @@ -515,15 +515,15 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-cast 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "chrono", "half", "indexmap", @@ -552,15 +552,15 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", ] [[package]] @@ -578,14 +578,14 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "half", ] @@ -602,9 +602,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b47e0ca91cc438d2c7879fe95e0bca5329fff28649e30a88c6f760b1faeddcb" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "bitflags", "serde_core", @@ -627,15 +627,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash 0.8.12", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "num-traits", ] @@ -658,15 +658,15 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", "memchr", "num-traits", "regex", @@ -1466,7 +1466,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.2", + "unicode-width 0.1.14", ] [[package]] @@ -1611,8 +1611,8 @@ name = "compress-bench" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "async-trait", "bytes", "clap", @@ -1620,7 +1620,7 @@ dependencies = [ "indicatif", "itertools 0.14.0", "lance-bench", - "parquet 58.0.0", + "parquet 58.1.0", "regex", "tokio", "tracing", @@ -2118,8 +2118,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" dependencies = [ - "arrow 58.0.0", - "arrow-schema 58.0.0", + "arrow 58.1.0", + "arrow-schema 58.1.0", "async-trait", "bytes", "bzip2", @@ -2157,7 +2157,7 @@ dependencies = [ "log", "object_store 0.13.2", "parking_lot", - "parquet 58.0.0", + "parquet 58.1.0", "rand 0.9.2", "regex", "sqlparser 0.61.0", @@ -2225,7 +2225,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "dashmap", "datafusion-common 53.1.0", @@ -2274,7 +2274,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "datafusion-catalog 53.1.0", "datafusion-common 53.1.0", @@ -2321,8 +2321,8 @@ checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" dependencies = [ "ahash 0.8.12", "apache-avro", - "arrow 58.0.0", - "arrow-ipc 58.0.0", + "arrow 58.1.0", + "arrow-ipc 58.1.0", "chrono", "half", "hashbrown 0.16.1", @@ -2331,7 +2331,7 @@ dependencies = [ "libc", "log", "object_store 0.13.2", - "parquet 58.0.0", + "parquet 58.1.0", "paste", "recursive", "sqlparser 0.61.0", @@ -2396,7 +2396,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-compression", "async-trait", "bytes", @@ -2455,8 +2455,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" dependencies = [ - "arrow 58.0.0", - "arrow-ipc 58.0.0", + "arrow 58.1.0", + "arrow-ipc 58.1.0", "async-trait", "bytes", "datafusion-common 53.1.0", @@ -2480,7 +2480,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" dependencies = [ "apache-avro", - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "bytes", "datafusion-common 53.1.0", @@ -2522,7 +2522,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "bytes", "datafusion-common 53.1.0", @@ -2567,7 +2567,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "bytes", "datafusion-common 53.1.0", @@ -2591,7 +2591,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "bytes", "datafusion-common 53.1.0", @@ -2611,7 +2611,7 @@ dependencies = [ "log", "object_store 0.13.2", "parking_lot", - "parquet 58.0.0", + "parquet 58.1.0", "tokio", ] @@ -2653,8 +2653,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" dependencies = [ - "arrow 58.0.0", - "arrow-buffer 58.0.0", + "arrow 58.1.0", + "arrow-buffer 58.1.0", "async-trait", "chrono", "dashmap", @@ -2698,7 +2698,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "chrono", "datafusion-common 53.1.0", @@ -2734,7 +2734,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "indexmap", "itertools 0.14.0", @@ -2777,8 +2777,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ - "arrow 58.0.0", - "arrow-buffer 58.0.0", + "arrow 58.1.0", + "arrow-buffer 58.1.0", "base64", "blake2", "blake3", @@ -2831,7 +2831,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" dependencies = [ "ahash 0.8.12", - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-doc 53.1.0", "datafusion-execution 53.1.0", @@ -2866,7 +2866,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" dependencies = [ "ahash 0.8.12", - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-expr-common 53.1.0", "datafusion-physical-expr-common 53.1.0", @@ -2901,8 +2901,8 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" dependencies = [ - "arrow 58.0.0", - "arrow-ord 58.0.0", + "arrow 58.1.0", + "arrow-ord 58.1.0", "datafusion-common 53.1.0", "datafusion-doc 53.1.0", "datafusion-execution 53.1.0", @@ -2942,7 +2942,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "datafusion-catalog 53.1.0", "datafusion-common 53.1.0", @@ -2976,7 +2976,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-doc 53.1.0", "datafusion-expr 53.1.0", @@ -3055,7 +3055,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "chrono", "datafusion-common 53.1.0", "datafusion-expr 53.1.0", @@ -3098,7 +3098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" dependencies = [ "ahash 0.8.12", - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-expr 53.1.0", "datafusion-expr-common 53.1.0", @@ -3136,7 +3136,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-expr 53.1.0", "datafusion-functions 53.1.0", @@ -3166,7 +3166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" dependencies = [ "ahash 0.8.12", - "arrow 58.0.0", + "arrow 58.1.0", "chrono", "datafusion-common 53.1.0", "datafusion-expr-common 53.1.0", @@ -3200,7 +3200,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-execution 53.1.0", "datafusion-expr 53.1.0", @@ -3251,9 +3251,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" dependencies = [ "ahash 0.8.12", - "arrow 58.0.0", - "arrow-ord 58.0.0", - "arrow-schema 58.0.0", + "arrow 58.1.0", + "arrow-ord 58.1.0", + "arrow-schema 58.1.0", "async-trait", "datafusion-common 53.1.0", "datafusion-common-runtime 53.1.0", @@ -3299,7 +3299,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "datafusion-common 53.1.0", "datafusion-datasource 53.1.0", "datafusion-expr-common 53.1.0", @@ -3344,7 +3344,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "bigdecimal", "chrono", "crc32fast", @@ -3388,7 +3388,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "bigdecimal", "chrono", "datafusion-common 53.1.0", @@ -3407,7 +3407,7 @@ version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e5a4a7a49143a68936992b6dbb0db44121c635e9992b2482817278f1e69c56" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "async-trait", "bigdecimal", "clap", @@ -3534,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4898,7 +4898,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4979,7 +4979,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5942,6 +5942,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rust2" version = "0.16.2" @@ -6296,7 +6305,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6731,17 +6740,17 @@ dependencies = [ [[package]] name = "parquet" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" +checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" dependencies = [ "ahash 0.8.12", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-data 58.0.0", - "arrow-ipc 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", "base64", "brotli", "bytes", @@ -6750,7 +6759,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "lz4_flex 0.12.1", + "lz4_flex 0.13.0", "num-bigint", "num-integer", "num-traits", @@ -6767,11 +6776,11 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ba4e5dcbc8ad65882b7337a95c12a0f9cbb6add237c53d93b803b7d7f70f02" +checksum = "2bf493f3c9ddd984d0efb019f67343e4aa4bab893931f6a14b82083065dc3d28" dependencies = [ - "arrow-schema 58.0.0", + "arrow-schema 58.1.0", "chrono", "half", "indexmap", @@ -6781,12 +6790,12 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec4cfb8da15565c8d211b6bc51e8eb481ea65d19132462af3f948b150ac8efe" +checksum = "6ac038d46a503a7d563b4f5df5802c4315d5343d009feab195d15ac512b4cb27" dependencies = [ - "arrow 58.0.0", - "arrow-schema 58.0.0", + "arrow 58.1.0", + "arrow-schema 58.1.0", "chrono", "half", "indexmap", @@ -6798,11 +6807,11 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.0.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3668ff00a6aeb29d172ba15f9d8fedf1675d79bff7d1916daa333efdeaa13e46" +checksum = "015a09c2ffe5108766c7c1235c307b8a3c2ea64eca38455ba1a7f3a7f32f16e2" dependencies = [ - "arrow-schema 58.0.0", + "arrow-schema 58.1.0", "base64", "chrono", "parquet-variant", @@ -6967,7 +6976,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.9.2", + "rand 0.8.5", "socket2", "thiserror 1.0.69", ] @@ -8163,7 +8172,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8221,7 +8230,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9300,7 +9309,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9319,7 +9328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9730,7 +9739,7 @@ name = "tpchgen-arrow" version = "2.0.2" source = "git+https://github.com/clflushopt/tpchgen-rs.git?rev=438e9c2dbc25b2fff82c0efc08b3f13b5707874f#438e9c2dbc25b2fff82c0efc08b3f13b5707874f" dependencies = [ - "arrow 58.0.0", + "arrow 58.1.0", "tpchgen", ] @@ -10046,13 +10055,13 @@ name = "vector-search-bench" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-schema 58.1.0", "clap", "futures", "indicatif", - "parquet 58.0.0", + "parquet 58.1.0", "rand 0.10.1", "serde", "tabled", @@ -10076,12 +10085,12 @@ name = "vortex" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", + "arrow-array 58.1.0", "codspeed-divan-compat", "fastlanes", "futures", "mimalloc", - "parquet 58.0.0", + "parquet 58.1.0", "paste", "rand 0.10.1", "rand_distr 0.6.0", @@ -10147,15 +10156,15 @@ version = "0.1.0" dependencies = [ "arbitrary", "arcref", - "arrow-arith 58.0.0", - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-cast 58.0.0", - "arrow-data 58.0.0", - "arrow-ord 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", - "arrow-string 58.0.0", + "arrow-arith 58.1.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-data 58.1.0", + "arrow-ord 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "arrow-string 58.1.0", "async-lock", "bytes", "cfg-if", @@ -10218,9 +10227,9 @@ name = "vortex-bench" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", "async-trait", "bytes", "bzip2", @@ -10235,7 +10244,7 @@ dependencies = [ "noodles-bgzf", "noodles-vcf", "parking_lot", - "parquet 58.0.0", + "parquet 58.1.0", "rand 0.10.1", "regex", "reqwest 0.12.28", @@ -10295,7 +10304,7 @@ dependencies = [ name = "vortex-buffer" version = "0.1.0" dependencies = [ - "arrow-buffer 58.0.0", + "arrow-buffer 58.1.0", "bitvec", "bytes", "codspeed-divan-compat", @@ -10326,12 +10335,12 @@ dependencies = [ name = "vortex-compat" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-select 58.0.0", + "arrow-array 58.1.0", + "arrow-select 58.1.0", "bytes", "clap", "futures", - "parquet 58.0.0", + "parquet 58.1.0", "reqwest 0.12.28", "serde", "serde_json", @@ -10420,8 +10429,8 @@ name = "vortex-cxx" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "async-fs", "cxx", "futures", @@ -10435,7 +10444,7 @@ name = "vortex-datafusion" version = "0.1.0" dependencies = [ "anyhow", - "arrow-schema 58.0.0", + "arrow-schema 58.1.0", "async-trait", "datafusion 53.1.0", "datafusion-catalog 53.1.0", @@ -10529,7 +10538,7 @@ dependencies = [ name = "vortex-error" version = "0.1.0" dependencies = [ - "arrow-schema 58.0.0", + "arrow-schema 58.1.0", "flatbuffers", "jiff", "object_store 0.13.2", @@ -10564,8 +10573,8 @@ dependencies = [ name = "vortex-ffi" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "async-fs", "cbindgen", "futures", @@ -10731,9 +10740,9 @@ dependencies = [ name = "vortex-jni" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-ipc 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", "futures", "jni", "object_store 0.13.2", @@ -10752,8 +10761,8 @@ name = "vortex-layout" version = "0.1.0" dependencies = [ "arcref", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "async-stream", "async-trait", "bit-vec", @@ -10793,7 +10802,7 @@ dependencies = [ name = "vortex-mask" version = "0.1.0" dependencies = [ - "arrow-buffer 58.0.0", + "arrow-buffer 58.1.0", "codspeed-divan-compat", "itertools 0.14.0", "rstest", @@ -10826,9 +10835,9 @@ dependencies = [ name = "vortex-parquet-variant" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-buffer 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-schema 58.1.0", "chrono", "parquet-variant", "parquet-variant-compute", @@ -10868,9 +10877,9 @@ dependencies = [ name = "vortex-python" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-data 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "bytes", "itertools 0.14.0", "log", @@ -10892,8 +10901,8 @@ name = "vortex-runend" version = "0.1.0" dependencies = [ "arbitrary", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "codspeed-divan-compat", "itertools 0.14.0", "num-traits", @@ -11013,8 +11022,8 @@ dependencies = [ name = "vortex-test-e2e-cuda" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "futures", "vortex", "vortex-cuda", @@ -11025,8 +11034,8 @@ name = "vortex-tui" version = "0.1.0" dependencies = [ "anyhow", - "arrow-array 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-schema 58.1.0", "clap", "console_error_panic_hook", "crossterm", @@ -11039,7 +11048,7 @@ dependencies = [ "indicatif", "itertools 0.14.0", "js-sys", - "parquet 58.0.0", + "parquet 58.1.0", "ratatui", "ratzilla", "serde", @@ -11067,9 +11076,9 @@ dependencies = [ name = "vortex-web-wasm" version = "0.1.0" dependencies = [ - "arrow-array 58.0.0", - "arrow-ipc 58.0.0", - "arrow-schema 58.0.0", + "arrow-array 58.1.0", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", "console_error_panic_hook", "futures", "js-sys", @@ -11324,7 +11333,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From 24fcc566acbf06c7288d9a4007006baec2623616 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 03:06:57 +0000 Subject: [PATCH 124/250] Update dependency org.testcontainers:junit-jupiter to v1.21.4 (#7539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [org.testcontainers:junit-jupiter](https://java.testcontainers.org) ([source](https://redirect.github.com/testcontainers/testcontainers-java)) | `1.20.4` → `1.21.4` | ![age](https://developer.mend.io/api/mc/badges/age/maven/org.testcontainers:junit-jupiter/1.21.4?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/org.testcontainers:junit-jupiter/1.20.4/1.21.4?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
testcontainers/testcontainers-java (org.testcontainers:junit-jupiter) ### [`v1.21.4`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.21.4) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.21.3...1.21.4) This release makes version 1.21.x works with recent Docker Engine changes. ### [`v1.21.3`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.21.3) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.21.2...1.21.3) ### What's Changed - Update testcontainers/sshd version to 1.3.0 ([#​10377](https://redirect.github.com/testcontainers/testcontainers-java/issues/10377)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Support docker/mcp-gateway image ([#​10378](https://redirect.github.com/testcontainers/testcontainers-java/issues/10378)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Update testcontainers version to 1.21.2 ([#​10369](https://redirect.github.com/testcontainers/testcontainers-java/issues/10369)) [@​github-actions](https://redirect.github.com/github-actions) - Update docs version to 1.21.2 ([#​10368](https://redirect.github.com/testcontainers/testcontainers-java/issues/10368)) [@​github-actions](https://redirect.github.com/github-actions) ### [`v1.21.2`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.21.2) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.21.1...1.21.2) ### What's Changed - Update ryuk version to 0.12.0 ([#​10357](https://redirect.github.com/testcontainers/testcontainers-java/issues/10357)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Update docs version to 1.21.1 ([#​10281](https://redirect.github.com/testcontainers/testcontainers-java/issues/10281)) [@​github-actions](https://redirect.github.com/github-actions) - Update testcontainers version to 1.21.1 ([#​10282](https://redirect.github.com/testcontainers/testcontainers-java/issues/10282)) [@​github-actions](https://redirect.github.com/github-actions) #### 📖 Documentation - Add DockerMcpGatewayContainer ([#​10364](https://redirect.github.com/testcontainers/testcontainers-java/issues/10364)) [@​eddumelendez](https://redirect.github.com/eddumelendez) #### 📦 Dependency updates - Update Gradle Wrapper from undefined to 8.14.2 ([#​10352](https://redirect.github.com/testcontainers/testcontainers-java/issues/10352)) [@​github-actions](https://redirect.github.com/github-actions) ### [`v1.21.1`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.21.1) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.21.0...1.21.1) ### What's Changed - Update docs version to 1.21.0 ([#​10193](https://redirect.github.com/testcontainers/testcontainers-java/issues/10193)) [@​github-actions](https://redirect.github.com/github-actions) - Update testcontainers version to 1.21.0 ([#​10194](https://redirect.github.com/testcontainers/testcontainers-java/issues/10194)) [@​github-actions](https://redirect.github.com/github-actions) #### 🚀 Features & Enhancements - Expose Loki in LgtmContainer ([#​10256](https://redirect.github.com/testcontainers/testcontainers-java/issues/10256)) [@​jaydeluca](https://redirect.github.com/jaydeluca) - Add support to pull model for DockerModelRunnerContainer ([#​10253](https://redirect.github.com/testcontainers/testcontainers-java/issues/10253)) [@​eddumelendez](https://redirect.github.com/eddumelendez) #### 🐛 Bug Fixes - Use generic init script filename when copying it into a Cassandra container ([#​9606](https://redirect.github.com/testcontainers/testcontainers-java/issues/9606)) [@​maximevw](https://redirect.github.com/maximevw) #### 📖 Documentation - Add support to clickhouse JDBC V2 ([#​10280](https://redirect.github.com/testcontainers/testcontainers-java/issues/10280)) [@​thiagohora](https://redirect.github.com/thiagohora) - Fix register listener for kafka docs ([#​10268](https://redirect.github.com/testcontainers/testcontainers-java/issues/10268)) [@​julianladisch](https://redirect.github.com/julianladisch) #### 📦 Dependency updates - Update checkstyle version to 10.23.0 ([#​10196](https://redirect.github.com/testcontainers/testcontainers-java/issues/10196)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ### [`v1.21.0`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.21.0) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.20.6...1.21.0) ##### What's Changed - Fix typo in LGTM container method ([#​10189](https://redirect.github.com/testcontainers/testcontainers-java/issues/10189)) [@​jaydeluca](https://redirect.github.com/jaydeluca) - Pass `start` command required in Solr 10 ([#​10174](https://redirect.github.com/testcontainers/testcontainers-java/issues/10174)) [@​epugh](https://redirect.github.com/epugh) - \[solr] Replace "create\_core" with "create" command ([#​10172](https://redirect.github.com/testcontainers/testcontainers-java/issues/10172)) [@​epugh](https://redirect.github.com/epugh) - Update docs version to ${GITHUB\_REF##\*/} ([#​10063](https://redirect.github.com/testcontainers/testcontainers-java/issues/10063)) [@​github-actions](https://redirect.github.com/github-actions) - Update testcontainers version to ${GITHUB\_REF##\*/} ([#​10062](https://redirect.github.com/testcontainers/testcontainers-java/issues/10062)) [@​github-actions](https://redirect.github.com/github-actions) ##### ⚠️ Breaking API changes - Remove spock-core from spock module ([#​10069](https://redirect.github.com/testcontainers/testcontainers-java/issues/10069)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Uses `clickhouse/clickhouse-server` as Docker Image in ClickHouseProvider ([#​8738](https://redirect.github.com/testcontainers/testcontainers-java/issues/8738)) [@​linghengqian](https://redirect.github.com/linghengqian) ##### 🚀 Features & Enhancements - Expose Tempo in LgtmContainer ([#​10192](https://redirect.github.com/testcontainers/testcontainers-java/issues/10192)) [@​jaydeluca](https://redirect.github.com/jaydeluca) - Allow spock tests to be skipped when Docker is unavailable ([#​10180](https://redirect.github.com/testcontainers/testcontainers-java/issues/10180)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Support new chromadb api version ([#​10170](https://redirect.github.com/testcontainers/testcontainers-java/issues/10170)) [@​dev-jonghoonpark](https://redirect.github.com/dev-jonghoonpark) - Add default database name to MongoDB Atlas ([#​10034](https://redirect.github.com/testcontainers/testcontainers-java/issues/10034)) [@​blancqua](https://redirect.github.com/blancqua) - \[servicebus] Skip waiting for sql to be ready ([#​10092](https://redirect.github.com/testcontainers/testcontainers-java/issues/10092)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Support additional flags in FirestoreEmulatorContainer ([#​10067](https://redirect.github.com/testcontainers/testcontainers-java/issues/10067)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### ☠️ Deprecations - Deprecate getUserPass and add getPassword ([#​10064](https://redirect.github.com/testcontainers/testcontainers-java/issues/10064)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### 🐛 Bug Fixes - Fix connection leak in JdbcDatabaseDelegate ([#​9662](https://redirect.github.com/testcontainers/testcontainers-java/issues/9662)) [@​froque](https://redirect.github.com/froque) - Allow configuring the AlwaysPullPolicy ([#​10188](https://redirect.github.com/testcontainers/testcontainers-java/issues/10188)) [@​sebastian-steiner](https://redirect.github.com/sebastian-steiner) ##### 📖 Documentation - Add DockerModelRunnerContainer to core ([#​10183](https://redirect.github.com/testcontainers/testcontainers-java/issues/10183)) [@​kiview](https://redirect.github.com/kiview) - Allow configuring the AlwaysPullPolicy ([#​10188](https://redirect.github.com/testcontainers/testcontainers-java/issues/10188)) [@​sebastian-steiner](https://redirect.github.com/sebastian-steiner) - Fix Apache Solr link ([#​10171](https://redirect.github.com/testcontainers/testcontainers-java/issues/10171)) [@​epugh](https://redirect.github.com/epugh) - Remove incubator note from Solr docs ([#​10173](https://redirect.github.com/testcontainers/testcontainers-java/issues/10173)) [@​epugh](https://redirect.github.com/epugh) - Remove linked-container ([#​10065](https://redirect.github.com/testcontainers/testcontainers-java/issues/10065)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### 🧹 Housekeeping - Add SFTP host key check example ([#​10127](https://redirect.github.com/testcontainers/testcontainers-java/issues/10127)) [@​julianladisch](https://redirect.github.com/julianladisch) - Remove linked-container ([#​10065](https://redirect.github.com/testcontainers/testcontainers-java/issues/10065)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### 📦 Dependency updates - Update Gradle Wrapper from undefined to 8.13 ([#​10033](https://redirect.github.com/testcontainers/testcontainers-java/issues/10033)) [@​github-actions](https://redirect.github.com/github-actions) - Update docker-java version to 3.4.2 ([#​10071](https://redirect.github.com/testcontainers/testcontainers-java/issues/10071)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ### [`v1.20.6`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.20.6) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.20.5...1.20.6) ##### What's Changed - Bump confluentinc/cp-kcat from 7.4.1 to 7.9.0 ([#​10000](https://redirect.github.com/testcontainers/testcontainers-java/issues/10000)) [@​julianladisch](https://redirect.github.com/julianladisch) - Set sourceCompatibility and targetCompatibility to 1.8 in `spock` module ### [`v1.20.5`](https://redirect.github.com/testcontainers/testcontainers-java/releases/tag/1.20.5) [Compare Source](https://redirect.github.com/testcontainers/testcontainers-java/compare/1.20.4...1.20.5) ##### What's Changed - Add `ServiceBusEmulatorContainer` to Azure module ([#​9795](https://redirect.github.com/testcontainers/testcontainers-java/issues/9795)) [@​nagyesta](https://redirect.github.com/nagyesta) - Add `EventHubsEmulatorContainer` to Azure module ([#​9665](https://redirect.github.com/testcontainers/testcontainers-java/issues/9665)) [@​nagyesta](https://redirect.github.com/nagyesta) - Add `AzuriteContainer` to Azure module ([#​9661](https://redirect.github.com/testcontainers/testcontainers-java/issues/9661)) [@​nagyesta](https://redirect.github.com/nagyesta) - Add `ldap` module ([#​9987](https://redirect.github.com/testcontainers/testcontainers-java/issues/9987)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Add `scylladb` module ([#​8002](https://redirect.github.com/testcontainers/testcontainers-java/issues/8002)) [@​mkorolyov](https://redirect.github.com/mkorolyov) - Add `pinecone` module ([#​9911](https://redirect.github.com/testcontainers/testcontainers-java/issues/9911)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### 🚀 Features & Enhancements - Set `RABBITMQ_DEFAULT_USER` env var with `withAdminUser` ([#​9571](https://redirect.github.com/testcontainers/testcontainers-java/issues/9571)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Move ollama port to a constant and provide new `getPort` method ([#​9623](https://redirect.github.com/testcontainers/testcontainers-java/issues/9623)) [@​edeandrea](https://redirect.github.com/edeandrea) - ##### 🐛 Bug Fixes - Fix reuse support for `CouchbaseContainer` ([#​9957](https://redirect.github.com/testcontainers/testcontainers-java/issues/9957)) [@​albihnf](https://redirect.github.com/albihnf) - Fix `SolrContainer` start parameters for version >= 9.7.0 ([#​9926](https://redirect.github.com/testcontainers/testcontainers-java/issues/9926)) [@​mkr](https://redirect.github.com/mkr) - Fix clickhouse authentication ([#​9942](https://redirect.github.com/testcontainers/testcontainers-java/issues/9942)) [@​livk-cloud](https://redirect.github.com/livk-cloud) - Fix cluster creation with `ConfluentKafkaContainer` and `KafkaContainer` ([#​9910](https://redirect.github.com/testcontainers/testcontainers-java/issues/9910)) [@​eddumelendez](https://redirect.github.com/eddumelendez) ##### 📖 Documentation - Fix typos ([#​9783](https://redirect.github.com/testcontainers/testcontainers-java/issues/9783)) [@​NathanBaulch](https://redirect.github.com/NathanBaulch) - Added Dash0 as Adoptor ([#​9630](https://redirect.github.com/testcontainers/testcontainers-java/issues/9630)) [@​CodingFabian](https://redirect.github.com/CodingFabian) - Improve Docker Compose docs ([#​9461](https://redirect.github.com/testcontainers/testcontainers-java/issues/9461)) [@​etrandafir93](https://redirect.github.com/etrandafir93) ##### 🧹 Housekeeping - Use docker/setup-docker-action ([#​9625](https://redirect.github.com/testcontainers/testcontainers-java/issues/9625)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Declare Java action in windows workflow ([#​9604](https://redirect.github.com/testcontainers/testcontainers-java/issues/9604)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Test against multiple Java versions ([#​8988](https://redirect.github.com/testcontainers/testcontainers-java/issues/8988)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Don't extend configuration compileOnly and testCompile from shaded ([#​9579](https://redirect.github.com/testcontainers/testcontainers-java/issues/9579)) [@​patrick-dedication](https://redirect.github.com/patrick-dedication) - Remove specific Java version for testing in gradle files ([#​9626](https://redirect.github.com/testcontainers/testcontainers-java/issues/9626)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Drop references to `vectorized/redpanda` image ([#​9624](https://redirect.github.com/testcontainers/testcontainers-java/issues/9624)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Polish AbstractPulsar test ([#​9600](https://redirect.github.com/testcontainers/testcontainers-java/issues/9600)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Polish BigtableEmulatorContainer test ([#​9599](https://redirect.github.com/testcontainers/testcontainers-java/issues/9599)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Fix typo in SolrContainer ([#​9572](https://redirect.github.com/testcontainers/testcontainers-java/issues/9572)) [@​dajudge](https://redirect.github.com/dajudge) ##### 📦 Dependency updates - Update docker-java version to 3.4.1 ([#​9627](https://redirect.github.com/testcontainers/testcontainers-java/issues/9627)) [@​eddumelendez](https://redirect.github.com/eddumelendez) - Adjust shadow gradle plugin coordinates ([#​9577](https://redirect.github.com/testcontainers/testcontainers-java/issues/9577)) [@​patrick-dedication](https://redirect.github.com/patrick-dedication)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index e5b726aaa45..3762b901617 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -15,7 +15,7 @@ slf4j = "2.0.17" spark3 = "3.5.8" spark4 = "4.1.1" s3mock = "4.12.4" -testcontainers-jupiter = "1.20.4" +testcontainers-jupiter = "1.21.4" [libraries] arrow-c-data = { module = "org.apache.arrow:arrow-c-data", version.ref = "arrow" } From 2c48e5710cd05f518196363e7375c586fa953d7f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 03:55:56 +0000 Subject: [PATCH 125/250] Update plugin com.google.protobuf to v0.10.0 (#7546) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | com.google.protobuf | `0.9.6` → `0.10.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.google.protobuf:com.google.protobuf.gradle.plugin/0.10.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.google.protobuf:com.google.protobuf.gradle.plugin/0.9.6/0.10.0?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/build.gradle.kts b/java/build.gradle.kts index 48c1ed3349f..128ec72e963 100644 --- a/java/build.gradle.kts +++ b/java/build.gradle.kts @@ -7,7 +7,7 @@ plugins { id("com.diffplug.spotless") version "8.4.0" id("com.palantir.git-version") version "5.0.0" id("net.ltgt.errorprone") version "5.1.0" apply false - id("com.google.protobuf") version "0.9.6" apply false + id("com.google.protobuf") version "0.10.0" apply false id("com.vanniktech.maven.publish") version "0.36.0" apply false } From ec439b06b7b2d7f66454636d263cc672d13c38bf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 03:59:10 +0000 Subject: [PATCH 126/250] Update Rust crate noodles-vcf to 0.87.0 (#7550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [noodles-vcf](https://redirect.github.com/zaeleus/noodles) | workspace.dependencies | minor | `0.86.0` → `0.87.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
zaeleus/noodles (noodles-vcf) ### [`v0.87.0`](https://redirect.github.com/zaeleus/noodles/compare/noodles-vcf-0.86.0...noodles-vcf-0.87.0) [Compare Source](https://redirect.github.com/zaeleus/noodles/compare/noodles-vcf-0.86.0...noodles-vcf-0.87.0)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3bb56183f7..9afab5122f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,7 +961,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -1466,7 +1466,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.1.14", + "unicode-width 0.2.2", ] [[package]] @@ -6253,9 +6253,9 @@ dependencies = [ [[package]] name = "noodles-vcf" -version = "0.86.0" +version = "0.87.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0131147080a5aeadcd6c2ab43e7ca1f9e8018cdd482eed797d081c40e8f0a3c7" +checksum = "12b339e85269ab7689661c690f9108dceb606bf3d230e1768799ae107d505e76" dependencies = [ "futures", "indexmap", @@ -6976,9 +6976,9 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.8.5", + "rand 0.9.2", "socket2", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -7185,7 +7185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.11.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -7217,7 +7217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", diff --git a/Cargo.toml b/Cargo.toml index 1ed524d1c6a..dd90cf3c28a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -177,7 +177,7 @@ mimalloc = "0.1.42" moka = { version = "0.12.10", default-features = false } multiversion = "0.8.0" noodles-bgzf = "0.46.0" -noodles-vcf = { version = "0.86.0", features = ["async"] } +noodles-vcf = { version = "0.87.0", features = ["async"] } num-traits = "0.2.19" num_enum = { version = "0.7.3", default-features = false } object_store = { version = "0.13.1", default-features = false } From 6f8f84d20eb1ce8212256e35dcca31b2180fe03a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 03:59:48 +0000 Subject: [PATCH 127/250] Update Rust crate taffy to 0.10.0 (#7554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [taffy](https://redirect.github.com/DioxusLabs/taffy) | workspace.dependencies | minor | `0.9.0` → `0.10.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
DioxusLabs/taffy (taffy) ### [`v0.10.1`](https://redirect.github.com/DioxusLabs/taffy/blob/HEAD/CHANGELOG.md#0101) [Compare Source](https://redirect.github.com/DioxusLabs/taffy/compare/v0.10.0...v0.10.1) ##### Fixed - CSS Grid auto-repeat and minimum-size handling ([#​946](https://redirect.github.com/DioxusLabs/taffy/issues/946)) ### [`v0.10.0`](https://redirect.github.com/DioxusLabs/taffy/blob/HEAD/CHANGELOG.md#0100) [Compare Source](https://redirect.github.com/DioxusLabs/taffy/compare/v0.9.2...v0.10.0) The MSRV for this release is 1.71. ##### Support for `direction` The `direction` property is now supported, allowing for RTL layout of boxes in Block, Flexbox, and CSS Grid layout modes. ##### Support for floats The `float` and `clear` properties are now supported. Support consists of a general-purpose `FloatContext` in the `compute` module, and integration of float layout into Block layout. Block layout now also has a `BlockContext` that allows a `FloatContext` to be shared across an entire Block formatting context. Float support is feature flagged by the `float_layout` feature. ##### Support for parsing styles from CSS string ([#​929](https://redirect.github.com/DioxusLabs/taffy/issues/929)) All of Taffy's style types (except the top-level `Style` struct) now have `FromStr` implementations that parses the type from the CSS representation of that value (e.g. `30px` or `50%` for `LengthPercentage`. A future version of Taffy will likely add support for parsing `Style` from `;`-seperated CSS. CSS parsing is feature flagged by the `parse` feature. Additionally the `parse_faster` feature enables optimizations for faster parsing at the cost of pulling in proc-macro dependencies such as `syn`. ##### Changed - Make DetailedGridTracksInfo accessible from a public module ([#​899](https://redirect.github.com/DioxusLabs/taffy/issues/899)) - Add `TaffyTree::write_tree` method to debug print the tree into an arbitrary writer ([#​925](https://redirect.github.com/DioxusLabs/taffy/issues/925)) - The cache `set` and `set` APIs now take `&LayoutInput` rather than individual values ([#​933](https://redirect.github.com/DioxusLabs/taffy/issues/933)) ##### Fixed - Flexbox: apply gap even when there are auto margins ([#​938](https://redirect.github.com/DioxusLabs/taffy/issues/938))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9afab5122f7..707a6d1696d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9093,9 +9093,9 @@ dependencies = [ [[package]] name = "taffy" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ba83ebaf2954d31d05d67340fd46cebe99da2b7133b0dd68d70c65473a437b" +checksum = "aea22054047c16c3f34d3ac473a2170be1424b1115b2a3adcf28cfb067c88859" dependencies = [ "arrayvec", "grid", diff --git a/Cargo.toml b/Cargo.toml index dd90cf3c28a..a46fefb6f7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -233,7 +233,7 @@ strum = "0.28" syn = { version = "2.0.117", features = ["full"] } sysinfo = "0.38.0" tabled = { version = "0.20.0", default-features = false } -taffy = "0.9.0" +taffy = "0.10.0" take_mut = "0.2.2" tar = "0.4" target-lexicon = "0.13" From 4c41a02e3c44daa262524284e3e08dd4cde27d6f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 04:00:40 +0000 Subject: [PATCH 128/250] Update Rust crate test-with to 0.16 (#7555) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [test-with](https://redirect.github.com/yanganto/test-with) | workspace.dependencies | minor | `0.15` → `0.16` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
yanganto/test-with (test-with) ### [`v0.16.1`](https://redirect.github.com/yanganto/test-with/compare/v0.16.0...v0.16.1) [Compare Source](https://redirect.github.com/yanganto/test-with/compare/v0.16.0...v0.16.1) ### [`v0.16.0`](https://redirect.github.com/yanganto/test-with/compare/v0.15.8...v0.16.0) [Compare Source](https://redirect.github.com/yanganto/test-with/compare/v0.15.8...v0.16.0)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 21 +++++++++++++++++++-- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 707a6d1696d..7c4d18c4ebd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9339,9 +9339,26 @@ checksum = "d4d1330fe7f7f872cd05165130b10602d667b205fd85be09be2814b115d4ced9" [[package]] name = "test-with" -version = "0.15.8" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c292571fb159e65c78c7c88a96e5ddfc326d6819e49b3a040d7b0aa3497925b" +dependencies = [ + "byte-unit", + "chrono", + "num_cpus", + "ping", + "reqwest 0.13.2", + "sysinfo", + "test-with-derive", + "uzers", + "which", +] + +[[package]] +name = "test-with-derive" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27838d769fa9bf364bf4a352ec88862b6e6cb96a8e6705cc78fbb16ff26ee8b9" +checksum = "01b95b557b54c0d50b04688ab86aaeaf16f201a0341936f3bfa19497dee081bd" dependencies = [ "byte-unit", "chrono", diff --git a/Cargo.toml b/Cargo.toml index a46fefb6f7c..5edb453219e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -240,7 +240,7 @@ target-lexicon = "0.13" temp-env = "0.3" tempfile = "3" termtree = { version = "1.0" } -test-with = "0.15" +test-with = "0.16" thiserror = "2.0.3" tokio = { version = "1.48" } tokio-stream = "0.1.17" From b1bf4abb714a4e707a3a66d2bc36332e0018f031 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 04:02:46 +0000 Subject: [PATCH 129/250] Update Rust crate hashbrown to 0.17.0 (#7547) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [hashbrown](https://redirect.github.com/rust-lang/hashbrown) | workspace.dependencies | minor | `0.16.1` → `0.17.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
rust-lang/hashbrown (hashbrown) ### [`v0.17.0`](https://redirect.github.com/rust-lang/hashbrown/blob/HEAD/CHANGELOG.md#0170---2026-04-06) [Compare Source](https://redirect.github.com/rust-lang/hashbrown/compare/v0.16.1...v0.17.0) ##### Added - Added `hash_table::OccupiedEntry::replace_entry_with` ([#​669](https://redirect.github.com/rust-lang/hashbrown/issues/669)) - Added `hash_map::{OccupiedEntry::into_entry, VacantEntryRef::insert_entry_with_key}` ([#​670](https://redirect.github.com/rust-lang/hashbrown/issues/670)) - Added `hash_table::UnsafeIter` ([#​667](https://redirect.github.com/rust-lang/hashbrown/issues/667)) - Added `iter` methods to various `HashTable` iterators ([#​667](https://redirect.github.com/rust-lang/hashbrown/issues/667)) - Added `HashMap::{replace_key,replace_key_unchecked,insert_with_key_unchecked}` ([#​681](https://redirect.github.com/rust-lang/hashbrown/issues/681)) - Added `into_map` methods to all `HashMap` entry types ([#​686](https://redirect.github.com/rust-lang/hashbrown/issues/686)) - Added `into_table` methods to all `HashTable` entry types ([#​686](https://redirect.github.com/rust-lang/hashbrown/issues/686)) - Added `#[must_use]` to constructors ([#​697](https://redirect.github.com/rust-lang/hashbrown/issues/697)) - `TryReserveError` now implements `Error` ([#​698](https://redirect.github.com/rust-lang/hashbrown/issues/698)) ##### Changed - Changed `EntryRef` to use `ToOwned` ([#​670](https://redirect.github.com/rust-lang/hashbrown/issues/670)) - Bumped MSRV to 1.85 (2024 edition) ([#​676](https://redirect.github.com/rust-lang/hashbrown/issues/676)) ##### Fixed - `HashTable:clone_from` now forwards to `RawTable::clone_from` instead of using the default implementation ([#​668](https://redirect.github.com/rust-lang/hashbrown/issues/668)) - Fixed potential UB in `RawTableInner::fallible_with_capacity` ([#​692](https://redirect.github.com/rust-lang/hashbrown/issues/692)) - Fixed incorrect length if a hasher panics during rehash ([#​710](https://redirect.github.com/rust-lang/hashbrown/issues/710))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 13 ++++++++++++- Cargo.toml | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c4d18c4ebd..14fc72b077e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4418,6 +4418,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "heapless" version = "0.8.0" @@ -11084,7 +11095,7 @@ name = "vortex-utils" version = "0.1.0" dependencies = [ "dashmap", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "parking_lot", "vortex-error", ] diff --git a/Cargo.toml b/Cargo.toml index 5edb453219e..63d6795506c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ get_dir = "0.5.0" glob = "0.3.2" goldenfile = "1" half = { version = "2.7.1", features = ["std", "num-traits"] } -hashbrown = "0.16.1" +hashbrown = "0.17.0" humansize = "2.1.3" indicatif = "0.18.0" insta = "1.43" From 5d1035573aa062b628189739c3db1056678803f5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 04:03:23 +0000 Subject: [PATCH 130/250] Update Rust crate tokio to v1.52.1 (#7556) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [tokio](https://tokio.rs) ([source](https://redirect.github.com/tokio-rs/tokio)) | workspace.dependencies | minor | `1.50.0` → `1.52.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
tokio-rs/tokio (tokio) ### [`v1.52.1`](https://redirect.github.com/tokio-rs/tokio/releases/tag/tokio-1.52.1): Tokio v1.52.1 [Compare Source](https://redirect.github.com/tokio-rs/tokio/compare/tokio-1.52.0...tokio-1.52.1) ### 1.52.1 (April 16th, 2026) #### Fixed - runtime: revert [#​7757] to fix [a regression][#​8056] that causes `spawn_blocking` to hang ([#​8057]) [#​7757]: https://redirect.github.com/tokio-rs/tokio/pull/7757 [#​8056]: https://redirect.github.com/tokio-rs/tokio/pull/8056 [#​8057]: https://redirect.github.com/tokio-rs/tokio/pull/8057 ### [`v1.52.0`](https://redirect.github.com/tokio-rs/tokio/releases/tag/tokio-1.52.0): Tokio v1.52.0 [Compare Source](https://redirect.github.com/tokio-rs/tokio/compare/tokio-1.51.1...tokio-1.52.0) ### 1.52.0 (April 14th, 2026) #### Added - io: `AioSource::register_borrowed` for I/O safety support ([#​7992]) - net: add `try_io` function to `unix::pipe` sender and receiver types ([#​8030]) #### Added (unstable) - runtime: `Builder::enable_eager_driver_handoff` setting enable eager hand off of the I/O and time drivers before polling tasks ([#​8010]) - taskdump: add `trace_with()` for customized task dumps ([#​8025]) - taskdump: allow `impl FnMut()` in `trace_with` instead of just `fn()` ([#​8040]) - fs: support `io_uring` in `AsyncRead` for `File` ([#​7907]) #### Changed - runtime: improve `spawn_blocking` scalability with sharded queue ([#​7757]) - runtime: use `compare_exchange_weak()` in worker queue ([#​8028]) #### Fixed - runtime: overflow second half of tasks when local queue is filled instead of first half ([#​8029]) #### Documented - docs: fix typo in `oneshot::Sender::send` docs ([#​8026]) - docs: hide #\[tokio::main] attribute in the docs of `sync::watch` ([#​8035]) - net: add docs on `ConnectionRefused` errors with UDP sockets ([#​7870]) [#​7757]: https://redirect.github.com/tokio-rs/tokio/pull/7757 [#​7870]: https://redirect.github.com/tokio-rs/tokio/pull/7870 [#​7907]: https://redirect.github.com/tokio-rs/tokio/pull/7907 [#​7992]: https://redirect.github.com/tokio-rs/tokio/pull/7992 [#​8010]: https://redirect.github.com/tokio-rs/tokio/pull/8010 [#​8025]: https://redirect.github.com/tokio-rs/tokio/pull/8025 [#​8026]: https://redirect.github.com/tokio-rs/tokio/pull/8026 [#​8028]: https://redirect.github.com/tokio-rs/tokio/pull/8028 [#​8029]: https://redirect.github.com/tokio-rs/tokio/pull/8029 [#​8030]: https://redirect.github.com/tokio-rs/tokio/pull/8030 [#​8035]: https://redirect.github.com/tokio-rs/tokio/pull/8035 [#​8040]: https://redirect.github.com/tokio-rs/tokio/pull/8040 ### [`v1.51.1`](https://redirect.github.com/tokio-rs/tokio/releases/tag/tokio-1.51.1): Tokio v1.51.1 [Compare Source](https://redirect.github.com/tokio-rs/tokio/compare/tokio-1.51.0...tokio-1.51.1) ##### 1.51.1 (April 8th, 2026) ##### Fixed - sync: fix semaphore reopens after forget ([#​8021]) - net: surface errors from `SO_ERROR` on `recv` for UDP sockets on Linux ([#​8001]) ##### Fixed (unstable) - metrics: fix `worker_local_schedule_count` test ([#​8008]) - rt: do not leak fd when cancelling io\_uring open operation ([#​7983]) [#​7983]: https://redirect.github.com/tokio-rs/tokio/pull/7983 [#​8001]: https://redirect.github.com/tokio-rs/tokio/pull/8001 [#​8008]: https://redirect.github.com/tokio-rs/tokio/pull/8008 [#​8021]: https://redirect.github.com/tokio-rs/tokio/pull/8021 ### [`v1.51.0`](https://redirect.github.com/tokio-rs/tokio/releases/tag/tokio-1.51.0): Tokio v1.51.0 [Compare Source](https://redirect.github.com/tokio-rs/tokio/compare/tokio-1.50.0...tokio-1.51.0) ##### 1.51.0 (April 3rd, 2026) ##### Added - net: implement `get_peer_cred` on Hurd ([#​7989]) - runtime: add `tokio::runtime::worker_index()` ([#​7921]) - runtime: add runtime name ([#​7924]) - runtime: stabilize `LocalRuntime` ([#​7557]) - wasm: add wasm32-wasip2 networking support ([#​7933]) ##### Changed - runtime: steal tasks from the LIFO slot ([#​7431]) ##### Fixed - docs: do not show "Available on non-loom only." doc label ([#​7977]) - macros: improve overall macro hygiene ([#​7997]) - sync: fix `notify_waiters` priority in `Notify` ([#​7996]) - sync: fix panic in `Chan::recv_many` when called with non-empty vector on closed channel ([#​7991]) [#​7431]: https://redirect.github.com/tokio-rs/tokio/pull/7431 [#​7557]: https://redirect.github.com/tokio-rs/tokio/pull/7557 [#​7921]: https://redirect.github.com/tokio-rs/tokio/pull/7921 [#​7924]: https://redirect.github.com/tokio-rs/tokio/pull/7924 [#​7933]: https://redirect.github.com/tokio-rs/tokio/pull/7933 [#​7977]: https://redirect.github.com/tokio-rs/tokio/pull/7977 [#​7989]: https://redirect.github.com/tokio-rs/tokio/pull/7989 [#​7991]: https://redirect.github.com/tokio-rs/tokio/pull/7991 [#​7996]: https://redirect.github.com/tokio-rs/tokio/pull/7996 [#​7997]: https://redirect.github.com/tokio-rs/tokio/pull/7997
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14fc72b077e..8298783e045 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3534,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4909,7 +4909,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4990,7 +4990,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6086,9 +6086,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -6316,7 +6316,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8183,7 +8183,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8241,7 +8241,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9320,7 +9320,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9339,7 +9339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9554,9 +9554,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", @@ -9571,9 +9571,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -11361,7 +11361,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] From 0e8924993fbc575ac570316c6ed2fb08cf9aa23f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 04:13:12 +0000 Subject: [PATCH 131/250] Update Rust crate zip to v8.5.1 (#7558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [zip](https://redirect.github.com/zip-rs/zip2) | workspace.dependencies | minor | `8.3.1` → `8.5.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
zip-rs/zip2 (zip) ### [`v8.5.1`](https://redirect.github.com/zip-rs/zip2/blob/HEAD/CHANGELOG.md#851---2026-04-06) [Compare Source](https://redirect.github.com/zip-rs/zip2/compare/v8.5.0...v8.5.1) ##### 🚜 Refactor - change magic finder to stack buffer ([#​763](https://redirect.github.com/zip-rs/zip2/pull/763)) - simplify extra field parsing ([#​764](https://redirect.github.com/zip-rs/zip2/pull/764)) ### [`v8.5.0`](https://redirect.github.com/zip-rs/zip2/blob/HEAD/CHANGELOG.md#850---2026-04-01) [Compare Source](https://redirect.github.com/zip-rs/zip2/compare/v8.4.0...v8.5.0) ##### 🐛 Bug Fixes - remove `zip64 comment` and add `zip64 extensible data sector` ([#​747](https://redirect.github.com/zip-rs/zip2/pull/747)) ##### 🚜 Refactor - remove useless magic in struct ([#​730](https://redirect.github.com/zip-rs/zip2/pull/730)) - change extra\_field from Arc\> to Arc<\[u8]> ([#​741](https://redirect.github.com/zip-rs/zip2/pull/741)) ##### ⚙️ Miscellaneous Tasks - cleanup README ([#​758](https://redirect.github.com/zip-rs/zip2/pull/758)) ### [`v8.4.0`](https://redirect.github.com/zip-rs/zip2/blob/HEAD/CHANGELOG.md#840---2026-03-23) [Compare Source](https://redirect.github.com/zip-rs/zip2/compare/v8.3.1...v8.4.0) ##### 🚀 Features - add a check for building benches ([#​748](https://redirect.github.com/zip-rs/zip2/pull/748)) ##### 🚜 Refactor - split part of `read.rs` for code readability ([#​744](https://redirect.github.com/zip-rs/zip2/pull/744)) - remove unused allow ([#​745](https://redirect.github.com/zip-rs/zip2/pull/745)) ##### ⚡ Performance - skip BufReader for Stored files in make\_reader ([#​739](https://redirect.github.com/zip-rs/zip2/pull/739)) ##### ⚙️ Miscellaneous Tasks - move pull request template to correct folder ([#​749](https://redirect.github.com/zip-rs/zip2/pull/749))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8298783e045..7b6afce8874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11954,9 +11954,9 @@ dependencies = [ [[package]] name = "zip" -version = "8.3.1" +version = "8.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c546feb4481b0fbafb4ef0d79b6204fc41c6f9884b1b73b1d73f82442fc0845" +checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59" dependencies = [ "aes", "bzip2", From 8dde6d188012cc9da8b5ea45e2d72244fe55d225 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:27:57 +0100 Subject: [PATCH 132/250] Update codecov/codecov-action action to v6 (#7561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [codecov/codecov-action](https://redirect.github.com/codecov/codecov-action) | action | major | `v5` → `v6` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
codecov/codecov-action (codecov/codecov-action) ### [`v6`](https://redirect.github.com/codecov/codecov-action/compare/v5...v6) [Compare Source](https://redirect.github.com/codecov/codecov-action/compare/v5...v6)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/rust-instrumented.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index 20bd674093a..9af1471f44f 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -63,7 +63,7 @@ jobs: --ignore benchmarks/* --ignore 'vortex-test/*' \ -o ${{ env.GRCOV_OUTPUT_FILE }} - name: Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 with: name: run-${{ matrix.suite }} files: ${{ env.GRCOV_OUTPUT_FILE }} From dcd7097aba6f65800e2f1860cb4fbe8ccb993c70 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:28:25 +0100 Subject: [PATCH 133/250] Update arrow to v19 (major) (#7559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [org.apache.arrow:arrow-memory-netty](https://arrow.apache.org/) ([source](https://redirect.github.com/apache/arrow-java)) | `18.3.0` → `19.0.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/org.apache.arrow:arrow-memory-netty/19.0.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/org.apache.arrow:arrow-memory-netty/18.3.0/19.0.0?slim=true) | | [org.apache.arrow:arrow-memory-core](https://arrow.apache.org/) ([source](https://redirect.github.com/apache/arrow-java)) | `18.3.0` → `19.0.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/org.apache.arrow:arrow-memory-core/19.0.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/org.apache.arrow:arrow-memory-core/18.3.0/19.0.0?slim=true) | | [org.apache.arrow:arrow-c-data](https://arrow.apache.org/) ([source](https://redirect.github.com/apache/arrow-java)) | `18.3.0` → `19.0.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/org.apache.arrow:arrow-c-data/19.0.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/org.apache.arrow:arrow-c-data/18.3.0/19.0.0?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
apache/arrow-java (org.apache.arrow:arrow-memory-netty) ### [`v19.0.0`](https://redirect.github.com/apache/arrow-java/releases/tag/v19.0.0): Apache Arrow Java 19.0.0 ##### What's Changed ##### Breaking Changes - [GH-774](https://redirect.github.com/apache/arrow-java/issues/774): Consoliate `BitVectorHelper.getValidityBufferSize` and `BaseValueVector.getValidityBufferSizeFromCount` by [@​rtadepalli](https://redirect.github.com/rtadepalli) in [#​775](https://redirect.github.com/apache/arrow-java/pull/775) - [GH-586](https://redirect.github.com/apache/arrow-java/issues/586): Override fixedSizeBinary method for UnionMapWriter by [@​axreldable](https://redirect.github.com/axreldable) in [#​885](https://redirect.github.com/apache/arrow-java/pull/885) - [GH-891](https://redirect.github.com/apache/arrow-java/issues/891): Add ExtensionTypeWriterFactory to TransferPair by [@​jhrotko](https://redirect.github.com/jhrotko) in [#​892](https://redirect.github.com/apache/arrow-java/pull/892) - [GH-948](https://redirect.github.com/apache/arrow-java/issues/948): Use buffer indexing for UUID vector by [@​jhrotko](https://redirect.github.com/jhrotko) in [#​949](https://redirect.github.com/apache/arrow-java/pull/949) - [GH-139](https://redirect.github.com/apache/arrow-java/issues/139): \[Flight] Stop return null from MetadataAdapter.getAll(String) and getAllByte(String) by [@​axreldable](https://redirect.github.com/axreldable) in [#​1016](https://redirect.github.com/apache/arrow-java/pull/1016) ##### New Features and Enhancements - [GH-52](https://redirect.github.com/apache/arrow-java/issues/52): Make RangeEqualsVisitor of RunEndEncodedVector more efficient by [@​ViggoC](https://redirect.github.com/ViggoC) in [#​761](https://redirect.github.com/apache/arrow-java/pull/761) - [GH-765](https://redirect.github.com/apache/arrow-java/issues/765): Do not close/free imported BaseStruct objects by [@​pepijnve](https://redirect.github.com/pepijnve) in [#​766](https://redirect.github.com/apache/arrow-java/pull/766) - [GH-79](https://redirect.github.com/apache/arrow-java/issues/79): Move `splitAndTransferValidityBuffer` to `BaseValueVector` by [@​rtadepalli](https://redirect.github.com/rtadepalli) in [#​777](https://redirect.github.com/apache/arrow-java/pull/777) - [GH-731](https://redirect.github.com/apache/arrow-java/issues/731): Avro adapter, output dictionary-encoded fields as enums by [@​martin-traverse](https://redirect.github.com/martin-traverse) in [#​779](https://redirect.github.com/apache/arrow-java/pull/779) - [GH-725](https://redirect.github.com/apache/arrow-java/issues/725): Added ExtensionReader by [@​xxlaykxx](https://redirect.github.com/xxlaykxx) in [#​726](https://redirect.github.com/apache/arrow-java/pull/726) - [GH-882](https://redirect.github.com/apache/arrow-java/issues/882): Add support for loading native library from a user specified location by [@​pepijnve](https://redirect.github.com/pepijnve) in [#​883](https://redirect.github.com/apache/arrow-java/pull/883) - [GH-109](https://redirect.github.com/apache/arrow-java/issues/109): Implement Vector Validators for StringView by [@​ViggoC](https://redirect.github.com/ViggoC) in [#​886](https://redirect.github.com/apache/arrow-java/pull/886) - [GH-900](https://redirect.github.com/apache/arrow-java/issues/900): Fix gandiva groupId in arrow-bom by [@​XN137](https://redirect.github.com/XN137) in [#​901](https://redirect.github.com/apache/arrow-java/pull/901) - [GH-762](https://redirect.github.com/apache/arrow-java/issues/762): Implement VectorAppender for RunEndEncodedVector by [@​ViggoC](https://redirect.github.com/ViggoC) in [#​884](https://redirect.github.com/apache/arrow-java/pull/884) - [GH-825](https://redirect.github.com/apache/arrow-java/issues/825): Add UUID canonical extension type by [@​jhrotko](https://redirect.github.com/jhrotko) in [#​903](https://redirect.github.com/apache/arrow-java/pull/903) - [GH-110](https://redirect.github.com/apache/arrow-java/issues/110): Flight SQL JDBC related StringView components implementation by [@​ViggoC](https://redirect.github.com/ViggoC) in [#​905](https://redirect.github.com/apache/arrow-java/pull/905) - [GH-863](https://redirect.github.com/apache/arrow-java/issues/863): \[JDBC] Suppress benign exceptions from gRPC layer on ArrowFlightSqlClientHandler#close by [@​ennuite](https://redirect.github.com/ennuite) in [#​910](https://redirect.github.com/apache/arrow-java/pull/910) - [GH-929](https://redirect.github.com/apache/arrow-java/issues/929): Add UUID support in JDBC driver by [@​xborder](https://redirect.github.com/xborder) in [#​930](https://redirect.github.com/apache/arrow-java/pull/930) - [GH-952](https://redirect.github.com/apache/arrow-java/issues/952): Add OAuth support by [@​xborder](https://redirect.github.com/xborder) in [#​953](https://redirect.github.com/apache/arrow-java/pull/953) - [GH-946](https://redirect.github.com/apache/arrow-java/issues/946): Add Variant extension type support by [@​tmater](https://redirect.github.com/tmater) in [#​947](https://redirect.github.com/apache/arrow-java/pull/947) - [GH-130](https://redirect.github.com/apache/arrow-java/issues/130): Fix AutoCloseables to work with [@​Nullable](https://redirect.github.com/Nullable) structures by [@​axreldable](https://redirect.github.com/axreldable) in [#​1017](https://redirect.github.com/apache/arrow-java/pull/1017) - [GH-1038](https://redirect.github.com/apache/arrow-java/issues/1038): Trim object memory for ArrowBuf by [@​lriggs](https://redirect.github.com/lriggs) in [#​1044](https://redirect.github.com/apache/arrow-java/pull/1044) - [GH-1061](https://redirect.github.com/apache/arrow-java/issues/1061): Add codegen classifier jar for arrow-vector. by [@​lriggs](https://redirect.github.com/lriggs) in [#​1062](https://redirect.github.com/apache/arrow-java/pull/1062) - [GH-301](https://redirect.github.com/apache/arrow-java/issues/301): \[Vector] Allow adding a vector at the end of VectorSchemaRoot by [@​axreldable](https://redirect.github.com/axreldable) in [#​1013](https://redirect.github.com/apache/arrow-java/pull/1013) - [GH-552](https://redirect.github.com/apache/arrow-java/issues/552): \[Vector] Add absent methods to the UnionFixedSizeListWriter by [@​axreldable](https://redirect.github.com/axreldable) in [#​1052](https://redirect.github.com/apache/arrow-java/pull/1052) ##### Bug Fixes - MINOR: add missing SOURCE\_DIR in dev/release/release.sh by [@​wgtmac](https://redirect.github.com/wgtmac) in [#​755](https://redirect.github.com/apache/arrow-java/pull/755) - MINOR: Empty stream double check by [@​adampolomski](https://redirect.github.com/adampolomski) in [#​742](https://redirect.github.com/apache/arrow-java/pull/742) - [GH-759](https://redirect.github.com/apache/arrow-java/issues/759): Get length of byte\[] in TryCopyLastError by [@​hnwyllmm](https://redirect.github.com/hnwyllmm) in [#​760](https://redirect.github.com/apache/arrow-java/pull/760) - [GH-899](https://redirect.github.com/apache/arrow-java/issues/899): \[Dataset] Initialize compute module by [@​lidavidm](https://redirect.github.com/lidavidm) in [#​893](https://redirect.github.com/apache/arrow-java/pull/893) - [GH-399](https://redirect.github.com/apache/arrow-java/issues/399): Check for null writers in DenseUnionWriter#setPosition by [@​Kaustav-Sarkar](https://redirect.github.com/Kaustav-Sarkar) in [#​938](https://redirect.github.com/apache/arrow-java/pull/938) - [GH-942](https://redirect.github.com/apache/arrow-java/issues/942): Fix JDBC Connection.setCatalog() by [@​eickler](https://redirect.github.com/eickler) in [#​943](https://redirect.github.com/apache/arrow-java/pull/943) - [GH-951](https://redirect.github.com/apache/arrow-java/issues/951): Fix CI completely, especially JNI on Windows 2022 and MacOS platforms by [@​jbonofre](https://redirect.github.com/jbonofre) in [#​925](https://redirect.github.com/apache/arrow-java/pull/925) - MINOR: Add private constructor to UuidType singleton by [@​tmater](https://redirect.github.com/tmater) in [#​945](https://redirect.github.com/apache/arrow-java/pull/945) - [GH-964](https://redirect.github.com/apache/arrow-java/issues/964): Fix IndexOutOfBoundsException in Array.getResultSet() for JDBC clients by [@​xborder](https://redirect.github.com/xborder) in [#​965](https://redirect.github.com/apache/arrow-java/pull/965) - [GH-932](https://redirect.github.com/apache/arrow-java/issues/932): \[JDBC] Fix memory leak on Connection#close due to unclosed Statement(s) by [@​ennuite](https://redirect.github.com/ennuite) in [#​933](https://redirect.github.com/apache/arrow-java/pull/933) - [GH-125](https://redirect.github.com/apache/arrow-java/issues/125): Allow null timestamp holder sans timezone by [@​Kaustav-Sarkar](https://redirect.github.com/Kaustav-Sarkar) in [#​941](https://redirect.github.com/apache/arrow-java/pull/941) - [GH-343](https://redirect.github.com/apache/arrow-java/issues/343): Fix ListVector offset buffer not properly serialized for nested empty arrays by [@​Yicong-Huang](https://redirect.github.com/Yicong-Huang) in [#​967](https://redirect.github.com/apache/arrow-java/pull/967) - [GH-990](https://redirect.github.com/apache/arrow-java/issues/990): \[JDBC] Fix memory leak on Connection#close due to unclosed ResultSet(s) by [@​ennuite](https://redirect.github.com/ennuite) in [#​991](https://redirect.github.com/apache/arrow-java/pull/991) - [GH-993](https://redirect.github.com/apache/arrow-java/issues/993): Fix missing pipe in milestone assignment script by [@​tmater](https://redirect.github.com/tmater) in [#​992](https://redirect.github.com/apache/arrow-java/pull/992) - [GH-470](https://redirect.github.com/apache/arrow-java/issues/470): \[Vector] Fix ListViewVector.getElementEndIndex(index) method by [@​axreldable](https://redirect.github.com/axreldable) in [#​1019](https://redirect.github.com/apache/arrow-java/pull/1019) - [GH-1007](https://redirect.github.com/apache/arrow-java/issues/1007): fix: does not break class loading if direct buffer allocator is not available by [@​torito](https://redirect.github.com/torito) in [#​1008](https://redirect.github.com/apache/arrow-java/pull/1008) - [GH-994](https://redirect.github.com/apache/arrow-java/issues/994): Fix DatabaseMetaData NPEs when SqlInfo is unavailable by [@​ennuite](https://redirect.github.com/ennuite) in [#​995](https://redirect.github.com/apache/arrow-java/pull/995) - [GH-1004](https://redirect.github.com/apache/arrow-java/issues/1004): \[JDBC] Fix NPE in ArrowFlightJdbcDriver#connect​(final String url, final Properties info) by [@​ennuite](https://redirect.github.com/ennuite) in [#​1005](https://redirect.github.com/apache/arrow-java/pull/1005) - [GH-343](https://redirect.github.com/apache/arrow-java/issues/343): Fix BaseVariableWidthVector and BaseLargeVariableWidthVector offset buffer serialization by [@​Yicong-Huang](https://redirect.github.com/Yicong-Huang) in [#​989](https://redirect.github.com/apache/arrow-java/pull/989) ##### Other Changes - MINOR: Bump checker.framework.version from 3.49.2 to 3.49.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​752](https://redirect.github.com/apache/arrow-java/pull/752) - MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.54.1 to 2.56.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​750](https://redirect.github.com/apache/arrow-java/pull/750) - MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.1.8 to 3.2.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​747](https://redirect.github.com/apache/arrow-java/pull/747) - MINOR: Bump version to 19.0.0-SNAPSHOT by [@​wgtmac](https://redirect.github.com/wgtmac) in [#​754](https://redirect.github.com/apache/arrow-java/pull/754) - [GH-768](https://redirect.github.com/apache/arrow-java/issues/768): Use apache/arrow-js for JS in integration test by [@​kou](https://redirect.github.com/kou) in [#​769](https://redirect.github.com/apache/arrow-java/pull/769) - [GH-770](https://redirect.github.com/apache/arrow-java/issues/770): Ensure updating Homebrew Python on macos-13 by [@​kou](https://redirect.github.com/kou) in [#​771](https://redirect.github.com/apache/arrow-java/pull/771) - [GH-70](https://redirect.github.com/apache/arrow-java/issues/70): Move from `hamcrest` to `assertj` in `flight-sql` by [@​rtadepalli](https://redirect.github.com/rtadepalli) in [#​772](https://redirect.github.com/apache/arrow-java/pull/772) - MINOR: Add missing permission to milestone assignment bot by [@​lidavidm](https://redirect.github.com/lidavidm) in [#​673](https://redirect.github.com/apache/arrow-java/pull/673) - MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.1 to 2.0.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​785](https://redirect.github.com/apache/arrow-java/pull/785) - MINOR: Bump io.grpc:grpc-bom from 1.71.0 to 1.73.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​781](https://redirect.github.com/apache/arrow-java/pull/781) - MINOR: Fix format by [@​lidavidm](https://redirect.github.com/lidavidm) in [#​809](https://redirect.github.com/apache/arrow-java/pull/809) - [GH-804](https://redirect.github.com/apache/arrow-java/issues/804): Prepend JDBC FlightSQL version to user agent by [@​xborder](https://redirect.github.com/xborder) in [#​806](https://redirect.github.com/apache/arrow-java/pull/806) - MINOR: \[CI] Bump actions/cache from 4.2.3 to 4.2.4 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​813](https://redirect.github.com/apache/arrow-java/pull/813) - MINOR: \[CI] Bump docker/login-action from 3.4.0 to 3.5.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​814](https://redirect.github.com/apache/arrow-java/pull/814) - MINOR: \[CI] Bump actions/download-artifact from 4.3.0 to 5.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​815](https://redirect.github.com/apache/arrow-java/pull/815) - [GH-816](https://redirect.github.com/apache/arrow-java/issues/816): Presize JsonStringArrayList vector results by [@​schlosna](https://redirect.github.com/schlosna) in [#​817](https://redirect.github.com/apache/arrow-java/pull/817) - [GH-797](https://redirect.github.com/apache/arrow-java/issues/797): \[JDBC] Fix PreparedStatement#execute for DML/DDL by [@​ennuite](https://redirect.github.com/ennuite) in [#​811](https://redirect.github.com/apache/arrow-java/pull/811) - [GH-841](https://redirect.github.com/apache/arrow-java/issues/841): Use apache/arrow-dotnet for integration test by [@​kou](https://redirect.github.com/kou) in [#​842](https://redirect.github.com/apache/arrow-java/pull/842) - MINOR: \[CI] Bump actions/setup-java from 4.6.0 to 5.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​847](https://redirect.github.com/apache/arrow-java/pull/847) - MINOR: \[CI] Bump actions/setup-python from 5 to 6 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​843](https://redirect.github.com/apache/arrow-java/pull/843) - MINOR: \[CI] Bump actions/github-script from 7.0.1 to 8.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​844](https://redirect.github.com/apache/arrow-java/pull/844) - [GH-858](https://redirect.github.com/apache/arrow-java/issues/858): Fix error handling in CompositeJdbcConsumer by [@​aiguofer](https://redirect.github.com/aiguofer) in [#​857](https://redirect.github.com/apache/arrow-java/pull/857) - [GH-859](https://redirect.github.com/apache/arrow-java/issues/859): Fix ARROW\_STRUCT\_CONFLICT\_POLICY env var by [@​wForget](https://redirect.github.com/wForget) in [#​860](https://redirect.github.com/apache/arrow-java/pull/860) - [GH-836](https://redirect.github.com/apache/arrow-java/issues/836): Added support of ExtensionType for ComplexCopier by [@​xxlaykxx](https://redirect.github.com/xxlaykxx) in [#​837](https://redirect.github.com/apache/arrow-java/pull/837) - [GH-848](https://redirect.github.com/apache/arrow-java/issues/848): TypedValue should be treated as Nullable in bind function in AvaticaParameterBinder by [@​XenoAmess](https://redirect.github.com/XenoAmess) in [#​849](https://redirect.github.com/apache/arrow-java/pull/849) - fix: issue with class names in arrow-c jni calls by [@​milenkovicm](https://redirect.github.com/milenkovicm) in [#​867](https://redirect.github.com/apache/arrow-java/pull/867) - [GH-839](https://redirect.github.com/apache/arrow-java/issues/839): Fix support for ResultSet.getObject for TIMESTAMP\_WITH\_TIMEZONE by [@​aiguofer](https://redirect.github.com/aiguofer) in [#​840](https://redirect.github.com/apache/arrow-java/pull/840) - [GH-880](https://redirect.github.com/apache/arrow-java/issues/880): \[CI] Fix syntax error in `dev_pr.yml` by [@​kou](https://redirect.github.com/kou) in [#​881](https://redirect.github.com/apache/arrow-java/pull/881) - [GH-592](https://redirect.github.com/apache/arrow-java/issues/592): \[Release] Use relative path in .sha\* by [@​kou](https://redirect.github.com/kou) in [#​879](https://redirect.github.com/apache/arrow-java/pull/879) - MINOR: \[CI] Bump docker/login-action from 3.5.0 to 3.6.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​870](https://redirect.github.com/apache/arrow-java/pull/870) - [GH-898](https://redirect.github.com/apache/arrow-java/issues/898): Upgrade to Apache POM 35 and identify fixes needed to have CI happy by [@​jbonofre](https://redirect.github.com/jbonofre) in [#​865](https://redirect.github.com/apache/arrow-java/pull/865) - MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.1.127.Final by [@​SirOibaf](https://redirect.github.com/SirOibaf) in [#​855](https://redirect.github.com/apache/arrow-java/pull/855) - MINOR: Bump logback.version from 1.5.18 to 1.5.20 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​897](https://redirect.github.com/apache/arrow-java/pull/897) - MINOR: Bump com.github.luben:zstd-jni from 1.5.7-2 to 1.5.7-6 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​896](https://redirect.github.com/apache/arrow-java/pull/896) - MINOR: \[CI] Bump actions/download-artifact from 5.0.0 to 6.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​895](https://redirect.github.com/apache/arrow-java/pull/895) - MINOR: Bump commons-codec:commons-codec from 1.18.0 to 1.19.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​871](https://redirect.github.com/apache/arrow-java/pull/871) - MINOR: \[CI] Bump actions/checkout from 4 to 5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​820](https://redirect.github.com/apache/arrow-java/pull/820) - MINOR: \[CI] Bump actions/upload-artifact from 4.6.2 to 5.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​894](https://redirect.github.com/apache/arrow-java/pull/894) - MINOR: Bump checker.framework.version from 3.49.3 to 3.49.5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​800](https://redirect.github.com/apache/arrow-java/pull/800) - MINOR: Bump error\_prone\_core.version from 2.37.0 to 2.42.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​749](https://redirect.github.com/apache/arrow-java/pull/749) - MINOR: Bump org.apache.orc:orc-core from 2.1.1 to 2.2.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​874](https://redirect.github.com/apache/arrow-java/pull/874) - MINOR: Bump com.google.protobuf:protobuf-bom from 4.30.2 to 4.33.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​888](https://redirect.github.com/apache/arrow-java/pull/888) - MINOR: \[CI] Bump actions/checkout from 5 to 6 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​911](https://redirect.github.com/apache/arrow-java/pull/911) - MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.18.0 to 2.20.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​912](https://redirect.github.com/apache/arrow-java/pull/912) - MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.80 to 1.82 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​919](https://redirect.github.com/apache/arrow-java/pull/919) - MINOR: Bump org.apache.drill.tools:drill-fmpp-maven-plugin from 1.21.2 to 1.22.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​918](https://redirect.github.com/apache/arrow-java/pull/918) - MINOR: Bump dep.hadoop.version from 3.4.1 to 3.4.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​915](https://redirect.github.com/apache/arrow-java/pull/915) - MINOR: \[CI] Bump actions/upload-artifact from 5.0.0 to 6.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​934](https://redirect.github.com/apache/arrow-java/pull/934) - MINOR: \[CI] Bump actions/download-artifact from 6.0.0 to 7.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​935](https://redirect.github.com/apache/arrow-java/pull/935) - MINOR: \[CI] Bump actions/cache from 4 to 5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​936](https://redirect.github.com/apache/arrow-java/pull/936) - MINOR: Update macos amd64 runner by [@​lidavidm](https://redirect.github.com/lidavidm) in [#​940](https://redirect.github.com/apache/arrow-java/pull/940) - MINOR: Bump checker.framework.version from 3.52.0 to 3.52.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​927](https://redirect.github.com/apache/arrow-java/pull/927) - MINOR: Bump org.jacoco:jacoco-maven-plugin from 0.8.13 to 0.8.14 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​924](https://redirect.github.com/apache/arrow-java/pull/924) - MINOR: Bump org.immutables:value from 2.10.1 to 2.11.7 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​922](https://redirect.github.com/apache/arrow-java/pull/922) - MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.56.0 to 2.63.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​920](https://redirect.github.com/apache/arrow-java/pull/920) - MINOR: Bump io.netty:netty-bom from 4.1.119.Final to 4.2.7.Final by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​887](https://redirect.github.com/apache/arrow-java/pull/887) - MINOR: Bump parquet.version from 1.15.2 to 1.16.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​913](https://redirect.github.com/apache/arrow-java/pull/913) - MINOR: Bump org.immutables:value-annotations from 2.10.1 to 2.11.7 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​917](https://redirect.github.com/apache/arrow-java/pull/917) - MINOR: Bump logback.version from 1.5.21 to 1.5.24 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​962](https://redirect.github.com/apache/arrow-java/pull/962) - MINOR: Bump org.codehaus.mojo:exec-maven-plugin from 3.5.0 to 3.6.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​959](https://redirect.github.com/apache/arrow-java/pull/959) - MINOR: Bump org.apache.commons:commons-text from 1.13.1 to 1.15.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​956](https://redirect.github.com/apache/arrow-java/pull/956) - MINOR: Bump io.grpc:grpc-bom from 1.73.0 to 1.78.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​958](https://redirect.github.com/apache/arrow-java/pull/958) - MINOR: Bump com.github.ben-manes.caffeine:caffeine from 3.2.0 to 3.2.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​960](https://redirect.github.com/apache/arrow-java/pull/960) - MINOR: Bump org.apache.avro:avro from 1.12.0 to 1.12.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​955](https://redirect.github.com/apache/arrow-java/pull/955) - MINOR: Bump org.bouncycastle:bcpkix-jdk18on from 1.82 to 1.83 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​969](https://redirect.github.com/apache/arrow-java/pull/969) - MINOR: Bump logback.version from 1.5.24 to 1.5.25 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​975](https://redirect.github.com/apache/arrow-java/pull/975) - MINOR: Bump com.fasterxml.jackson:jackson-bom from 2.18.3 to 2.21.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​973](https://redirect.github.com/apache/arrow-java/pull/973) - MINOR: Bump parquet.version from 1.16.0 to 1.17.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​968](https://redirect.github.com/apache/arrow-java/pull/968) - MINOR: Bump commons-io:commons-io from 2.19.0 to 2.21.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​974](https://redirect.github.com/apache/arrow-java/pull/974) - MINOR: Bump com.gradle:develocity-maven-extension from 2.0 to 2.3.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​976](https://redirect.github.com/apache/arrow-java/pull/976) - MINOR: Bump org.apache.orc:orc-core from 2.2.1 to 2.2.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​971](https://redirect.github.com/apache/arrow-java/pull/971) - MINOR: Bump org.apache.commons:commons-dbcp2 from 2.13.0 to 2.14.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​983](https://redirect.github.com/apache/arrow-java/pull/983) - MINOR: Bump org.apache.commons:commons-compress from 1.27.1 to 1.28.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​985](https://redirect.github.com/apache/arrow-java/pull/985) - MINOR: Bump org.assertj:assertj-core from 3.27.3 to 3.27.7 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​988](https://redirect.github.com/apache/arrow-java/pull/988) - MINOR: Bump org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​987](https://redirect.github.com/apache/arrow-java/pull/987) - MINOR: Bump logback.version from 1.5.25 to 1.5.26 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​981](https://redirect.github.com/apache/arrow-java/pull/981) - MINOR: Bump com.google.protobuf:protobuf-bom from 4.33.1 to 4.33.4 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​984](https://redirect.github.com/apache/arrow-java/pull/984) - [GH-1011](https://redirect.github.com/apache/arrow-java/issues/1011): \[Docs] Fix broken Java API reference links in documentation by [@​axreldable](https://redirect.github.com/axreldable) in [#​1012](https://redirect.github.com/apache/arrow-java/pull/1012) - [GH-141](https://redirect.github.com/apache/arrow-java/issues/141): Correct capacity behavior in BufferAllocator.buffer docstrings by [@​axreldable](https://redirect.github.com/axreldable) in [#​1010](https://redirect.github.com/apache/arrow-java/pull/1010) - [GH-1014](https://redirect.github.com/apache/arrow-java/issues/1014): \[Docs] Fix broken and obsolete links in the README.md by [@​axreldable](https://redirect.github.com/axreldable) in [#​1015](https://redirect.github.com/apache/arrow-java/pull/1015) - MINOR: \[Docs] Remove extra line in README.md (fix pre-commit) by [@​axreldable](https://redirect.github.com/axreldable) in [#​1018](https://redirect.github.com/apache/arrow-java/pull/1018) - [GH-1021](https://redirect.github.com/apache/arrow-java/issues/1021): Use released apache/arrow instead of main by [@​kou](https://redirect.github.com/kou) in [#​1022](https://redirect.github.com/apache/arrow-java/pull/1022) - MINOR: Bump org.mockito:mockito-bom from 5.17.0 to 5.21.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1000](https://redirect.github.com/apache/arrow-java/pull/1000) - MINOR: Bump com.gradle:develocity-maven-extension from 2.3.1 to 2.3.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1001](https://redirect.github.com/apache/arrow-java/pull/1001) - MINOR: Bump logback.version from 1.5.26 to 1.5.27 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​999](https://redirect.github.com/apache/arrow-java/pull/999) - MINOR: \[CI] Bump docker/login-action from 3.6.0 to 3.7.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​996](https://redirect.github.com/apache/arrow-java/pull/996) - MINOR: Bump commons-codec:commons-codec from 1.20.0 to 1.21.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​997](https://redirect.github.com/apache/arrow-java/pull/997) - MINOR: Fix minor issue with README by [@​paliwalashish](https://redirect.github.com/paliwalashish) in [#​1026](https://redirect.github.com/apache/arrow-java/pull/1026) - MINOR: Bump commons-cli:commons-cli from 1.9.0 to 1.11.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1028](https://redirect.github.com/apache/arrow-java/pull/1028) - MINOR: Bump org.codehaus.mojo:versions-maven-plugin from 2.20.0 to 2.21.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1029](https://redirect.github.com/apache/arrow-java/pull/1029) - MINOR: Bump com.google.api.grpc:proto-google-common-protos from 2.63.2 to 2.66.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1034](https://redirect.github.com/apache/arrow-java/pull/1034) - MINOR: \[CI] Bump actions/upload-artifact from 6.0.0 to 7.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1045](https://redirect.github.com/apache/arrow-java/pull/1045) - MINOR: Bump checker.framework.version from 3.53.0 to 3.53.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1046](https://redirect.github.com/apache/arrow-java/pull/1046) - MINOR: \[CI] Bump actions/download-artifact from 7.0.0 to 8.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1047](https://redirect.github.com/apache/arrow-java/pull/1047) - MINOR: Bump org.codehaus.mojo:build-helper-maven-plugin from 3.6.0 to 3.6.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1049](https://redirect.github.com/apache/arrow-java/pull/1049) - MINOR: Fix flaky TestBasicAuth memory leak by waiting for async buffer release by [@​jbonofre](https://redirect.github.com/jbonofre) in [#​1058](https://redirect.github.com/apache/arrow-java/pull/1058) - MINOR: Bump org.apache.orc:orc-core from 2.2.2 to 2.3.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1056](https://redirect.github.com/apache/arrow-java/pull/1056) - MINOR: \[CI] Increase JNI macOS job timeout from 45 to 60 minutes by [@​jbonofre](https://redirect.github.com/jbonofre) in [#​1060](https://redirect.github.com/apache/arrow-java/pull/1060) - MINOR: \[CI] Bump docker/login-action from 3.7.0 to 4.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1053](https://redirect.github.com/apache/arrow-java/pull/1053) - MINOR: Bump dep.hadoop.version from 3.4.2 to 3.4.3 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1055](https://redirect.github.com/apache/arrow-java/pull/1055) - MINOR: Bump io.grpc:grpc-bom from 1.78.0 to 1.79.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​1048](https://redirect.github.com/apache/arrow-java/pull/1048) - MINOR: Bump com.gradle:common-custom-user-data-maven-extension from 2.0.3 to 2.1.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​998](https://redirect.github.com/apache/arrow-java/pull/998) - MINOR: Bump version to 19.0.0 by [@​jbonofre](https://redirect.github.com/jbonofre) in [#​1066](https://redirect.github.com/apache/arrow-java/pull/1066) ##### New Contributors - [@​adampolomski](https://redirect.github.com/adampolomski) made their first contribution in [#​742](https://redirect.github.com/apache/arrow-java/pull/742) - [@​rtadepalli](https://redirect.github.com/rtadepalli) made their first contribution in [#​772](https://redirect.github.com/apache/arrow-java/pull/772) - [@​xborder](https://redirect.github.com/xborder) made their first contribution in [#​806](https://redirect.github.com/apache/arrow-java/pull/806) - [@​schlosna](https://redirect.github.com/schlosna) made their first contribution in [#​817](https://redirect.github.com/apache/arrow-java/pull/817) - [@​ennuite](https://redirect.github.com/ennuite) made their first contribution in [#​811](https://redirect.github.com/apache/arrow-java/pull/811) - [@​wForget](https://redirect.github.com/wForget) made their first contribution in [#​860](https://redirect.github.com/apache/arrow-java/pull/860) - [@​XenoAmess](https://redirect.github.com/XenoAmess) made their first contribution in [#​849](https://redirect.github.com/apache/arrow-java/pull/849) - [@​milenkovicm](https://redirect.github.com/milenkovicm) made their first contribution in [#​867](https://redirect.github.com/apache/arrow-java/pull/867) - [@​XN137](https://redirect.github.com/XN137) made their first contribution in [#​901](https://redirect.github.com/apache/arrow-java/pull/901) - [@​SirOibaf](https://redirect.github.com/SirOibaf) made their first contribution in [#​855](https://redirect.github.com/apache/arrow-java/pull/855) - [@​axreldable](https://redirect.github.com/axreldable) made their first contribution in [#​885](https://redirect.github.com/apache/arrow-java/pull/885) - [@​jhrotko](https://redirect.github.com/jhrotko) made their first contribution in [#​903](https://redirect.github.com/apache/arrow-java/pull/903) - [@​Kaustav-Sarkar](https://redirect.github.com/Kaustav-Sarkar) made their first contribution in [#​938](https://redirect.github.com/apache/arrow-java/pull/938) - [@​eickler](https://redirect.github.com/eickler) made their first contribution in [#​943](https://redirect.github.com/apache/arrow-java/pull/943) - [@​tmater](https://redirect.github.com/tmater) made their first contribution in [#​945](https://redirect.github.com/apache/arrow-java/pull/945) - [@​Yicong-Huang](https://redirect.github.com/Yicong-Huang) made their first contribution in [#​967](https://redirect.github.com/apache/arrow-java/pull/967) - [@​paliwalashish](https://redirect.github.com/paliwalashish) made their first contribution in [#​1026](https://redirect.github.com/apache/arrow-java/pull/1026) - [@​torito](https://redirect.github.com/torito) made their first contribution in [#​1008](https://redirect.github.com/apache/arrow-java/pull/1008) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index 3762b901617..6854f2d5403 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors [versions] -arrow = "18.3.0" +arrow = "19.0.0" errorprone = "2.36.0" guava = "33.6.0-jre" immutables = "2.12.1" From 6995b9983fd4b59bfaab17fb2cc80ee3e44cbe32 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:32:15 +0000 Subject: [PATCH 134/250] Update Rust crate uuid to v1.23.1 (#7557) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [uuid](https://redirect.github.com/uuid-rs/uuid) | workspace.dependencies | minor | `1.22.0` → `1.23.1` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
uuid-rs/uuid (uuid) ### [`v1.23.1`](https://redirect.github.com/uuid-rs/uuid/releases/tag/v1.23.1) [Compare Source](https://redirect.github.com/uuid-rs/uuid/compare/v1.23.0...v1.23.1) #### What's Changed - Remove deprecated `msrv` feature from wasm-bindgen dependency by [@​guybedford](https://redirect.github.com/guybedford) in [#​877](https://redirect.github.com/uuid-rs/uuid/pull/877) - fix: Timestamp::from\_gregorian deprecation note by [@​aznashwan](https://redirect.github.com/aznashwan) in [#​878](https://redirect.github.com/uuid-rs/uuid/pull/878) - Prepare for 1.23.1 release by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​879](https://redirect.github.com/uuid-rs/uuid/pull/879) #### New Contributors - [@​guybedford](https://redirect.github.com/guybedford) made their first contribution in [#​877](https://redirect.github.com/uuid-rs/uuid/pull/877) - [@​aznashwan](https://redirect.github.com/aznashwan) made their first contribution in [#​878](https://redirect.github.com/uuid-rs/uuid/pull/878) **Full Changelog**: ### [`v1.23.0`](https://redirect.github.com/uuid-rs/uuid/releases/tag/v1.23.0) [Compare Source](https://redirect.github.com/uuid-rs/uuid/compare/v1.22.0...v1.23.0) #### What's Changed - feat: add support for 'hyphenated' format in the serde module by [@​FrenchDilettante](https://redirect.github.com/FrenchDilettante) in [#​865](https://redirect.github.com/uuid-rs/uuid/pull/865) - Fix a number of bugs in time-related code by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​872](https://redirect.github.com/uuid-rs/uuid/pull/872) - Reword invalid char error message by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​873](https://redirect.github.com/uuid-rs/uuid/pull/873) - Impl cleanups by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​874](https://redirect.github.com/uuid-rs/uuid/pull/874) - Use LazyLock to synchronize v1/v6 context initialization by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​875](https://redirect.github.com/uuid-rs/uuid/pull/875) - Prepare for 1.23.0 release by [@​KodrAus](https://redirect.github.com/KodrAus) in [#​876](https://redirect.github.com/uuid-rs/uuid/pull/876) #### New Contributors - [@​FrenchDilettante](https://redirect.github.com/FrenchDilettante) made their first contribution in [#​865](https://redirect.github.com/uuid-rs/uuid/pull/865) #### Special thanks [@​meng-xu-cs](https://redirect.github.com/meng-xu-cs) raised a series of bugs against the timestamp logic in `uuid` using automated tooling. The issues themselves were reasonably and responsibly presented and the end result is a better `uuid` library for everyone. Thanks! ### Deprecations This release includes the following deprecations: - `Context`: Renamed to `ContextV1` - `Timestamp::from_gregorian`: Renamed to `Timestamp::from_gregorian_time` ### Change to `Version::Max` `Version::Max`'s `u8` representation has changed from `0xff` to `0x0f` to match the value returned by `Uuid::get_version_num`. ### Change to `Uuid::get_version` for the max UUID `Uuid::get_version` will only return `Some(Version::Max)` if the UUID is actually the max UUID (all bytes are `0xff`). Previously it would return `Some` if only the version field was `0x0f`. This change matches the behaviour of the nil UUID, which only returns `Some(Version::Nil)` if the UUID is the nil UUID (all bytes are `0x00`). **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- Cargo.lock | 6 +++--- vortex-array/src/extension/uuid/metadata.rs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b6afce8874..4ed61984eb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9317,7 +9317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", "windows-sys 0.52.0", @@ -10052,9 +10052,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", diff --git a/vortex-array/src/extension/uuid/metadata.rs b/vortex-array/src/extension/uuid/metadata.rs index 7e7dd8b16d8..a9f595bac70 100644 --- a/vortex-array/src/extension/uuid/metadata.rs +++ b/vortex-array/src/extension/uuid/metadata.rs @@ -21,6 +21,8 @@ pub(crate) fn u8_to_version(b: u8) -> VortexResult { 6 => Ok(Version::SortMac), 7 => Ok(Version::SortRand), 8 => Ok(Version::Custom), + // UUID crate changed from 0xff to 0x0f for maximum uuid version in 1.23.0 + 0x0f => Ok(Version::Max), 0xff => Ok(Version::Max), _ => vortex_bail!("unknown UUID version discriminant: {b}"), } From d0a6dba42c6699dabffc6f0b0f5f5c39de961d3c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:37:52 +0000 Subject: [PATCH 135/250] Update Rust crate cudarc to 0.19.0 (#7545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [cudarc](https://redirect.github.com/chelsea0x3b/cudarc) | workspace.dependencies | minor | `0.18.2` → `0.19.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
chelsea0x3b/cudarc (cudarc) ### [`v0.19.4`](https://redirect.github.com/chelsea0x3b/cudarc/releases/tag/v0.19.4): - cuda 13.2 [Compare Source](https://redirect.github.com/chelsea0x3b/cudarc/compare/v0.19.3...v0.19.4) #### What's Changed - Add total\_mem() and mem\_get\_info() safe methods to CudaContext by [@​OneThing98](https://redirect.github.com/OneThing98) in [#​534](https://redirect.github.com/chelsea0x3b/cudarc/pull/534) - driver: add upload() and raw accessors to CudaGraph by [@​OneThing98](https://redirect.github.com/OneThing98) in [#​535](https://redirect.github.com/chelsea0x3b/cudarc/pull/535) - docs: add cufft by [@​mayocream](https://redirect.github.com/mayocream) in [#​538](https://redirect.github.com/chelsea0x3b/cudarc/pull/538) - feat: wire FP4 packed types from float4 0.2.0 by [@​jorgeantonio21](https://redirect.github.com/jorgeantonio21) in [#​540](https://redirect.github.com/chelsea0x3b/cudarc/pull/540) - Add cuCtxCreate\_v4 bindings, CudaContext::new\_non\_primary() and new\_cig() by [@​w4nderlust](https://redirect.github.com/w4nderlust) in [#​539](https://redirect.github.com/chelsea0x3b/cudarc/pull/539) - driver: add result-level memory pool wrappers by [@​OneThing98](https://redirect.github.com/OneThing98) in [#​544](https://redirect.github.com/chelsea0x3b/cudarc/pull/544) - Remove cuCtxCreate\_v4 from blocklist by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​550](https://redirect.github.com/chelsea0x3b/cudarc/pull/550) - Add CUDA 13.2 Support by [@​TannerRogalsky](https://redirect.github.com/TannerRogalsky) in [#​546](https://redirect.github.com/chelsea0x3b/cudarc/pull/546) - Exposing `has_async_alloc` field in `CudaContext` by [@​LateinCecer](https://redirect.github.com/LateinCecer) in [#​553](https://redirect.github.com/chelsea0x3b/cudarc/pull/553) - Adds cuda 13.2 support by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​551](https://redirect.github.com/chelsea0x3b/cudarc/pull/551) - Parallelize bindings-generator by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​552](https://redirect.github.com/chelsea0x3b/cudarc/pull/552) #### New Contributors - [@​OneThing98](https://redirect.github.com/OneThing98) made their first contribution in [#​534](https://redirect.github.com/chelsea0x3b/cudarc/pull/534) - [@​jorgeantonio21](https://redirect.github.com/jorgeantonio21) made their first contribution in [#​540](https://redirect.github.com/chelsea0x3b/cudarc/pull/540) - [@​w4nderlust](https://redirect.github.com/w4nderlust) made their first contribution in [#​539](https://redirect.github.com/chelsea0x3b/cudarc/pull/539) - [@​TannerRogalsky](https://redirect.github.com/TannerRogalsky) made their first contribution in [#​546](https://redirect.github.com/chelsea0x3b/cudarc/pull/546) - [@​LateinCecer](https://redirect.github.com/LateinCecer) made their first contribution in [#​553](https://redirect.github.com/chelsea0x3b/cudarc/pull/553) **Full Changelog**: ### [`v0.19.3`](https://redirect.github.com/chelsea0x3b/cudarc/releases/tag/v0.19.3): - safe cufft [Compare Source](https://redirect.github.com/chelsea0x3b/cudarc/compare/v0.19.2...v0.19.3) #### What's Changed - Implement `DeviceRepr` for arrays by [@​kaathewisegit](https://redirect.github.com/kaathewisegit) in [#​523](https://redirect.github.com/chelsea0x3b/cudarc/pull/523) - feat: cufft safe API by [@​mayocream](https://redirect.github.com/mayocream) in [#​532](https://redirect.github.com/chelsea0x3b/cudarc/pull/532) **Full Changelog**: ### [`v0.19.2`](https://redirect.github.com/chelsea0x3b/cudarc/releases/tag/v0.19.2): - fixes for dynamic loading with cufft & cudnn 9 [Compare Source](https://redirect.github.com/chelsea0x3b/cudarc/compare/v0.19.1...v0.19.2) #### What's Changed - fix: add support for cufft 12.x by [@​mayocream](https://redirect.github.com/mayocream) in [#​530](https://redirect.github.com/chelsea0x3b/cudarc/pull/530) - Add lib{name}.so.9 by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​531](https://redirect.github.com/chelsea0x3b/cudarc/pull/531) **Full Changelog**: ### [`v0.19.1`](https://redirect.github.com/chelsea0x3b/cudarc/releases/tag/v0.19.1): - bump float8 & libloading versions [Compare Source](https://redirect.github.com/chelsea0x3b/cudarc/compare/v0.19.0...v0.19.1) #### What's Changed - Bump float8 to 0.7.0 by [@​EricLBuehler](https://redirect.github.com/EricLBuehler) in [#​527](https://redirect.github.com/chelsea0x3b/cudarc/pull/527) - Bump libloading 0.9.0 by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​528](https://redirect.github.com/chelsea0x3b/cudarc/pull/528) **Full Changelog**: ### [`v0.19.0`](https://redirect.github.com/chelsea0x3b/cudarc/releases/tag/v0.19.0): - small updates [Compare Source](https://redirect.github.com/chelsea0x3b/cudarc/compare/v0.18.2...v0.19.0) #### What's Changed - Fix memory safety issue in CudaSlice::leak and optimize Drop by [@​wizenink](https://redirect.github.com/wizenink) in [#​516](https://redirect.github.com/chelsea0x3b/cudarc/pull/516) - \[Breaking] get\_global returns CudaViewMut by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​517](https://redirect.github.com/chelsea0x3b/cudarc/pull/517) - Add fallback for loading like there is for version by [@​wingertge](https://redirect.github.com/wingertge) in [#​518](https://redirect.github.com/chelsea0x3b/cudarc/pull/518) - fixes a few issues with multi gpu usage in both candle and mistralrs by [@​krampenschiesser](https://redirect.github.com/krampenschiesser) in [#​520](https://redirect.github.com/chelsea0x3b/cudarc/pull/520) - unify memcpy peer & memcpy dtod by [@​chelsea0x3b](https://redirect.github.com/chelsea0x3b) in [#​522](https://redirect.github.com/chelsea0x3b/cudarc/pull/522) #### New Contributors - [@​wingertge](https://redirect.github.com/wingertge) made their first contribution in [#​518](https://redirect.github.com/chelsea0x3b/cudarc/pull/518) - [@​krampenschiesser](https://redirect.github.com/krampenschiesser) made their first contribution in [#​520](https://redirect.github.com/chelsea0x3b/cudarc/pull/520) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 16 +++++++++++++--- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ed61984eb2..cb4668115b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1932,12 +1932,12 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.18.2" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa12038120eb13347a6ae2ffab1d34efe78150125108627fd85044dd4d6ff1e" +checksum = "f071cd6a7b5d51607df76aa2d426aaabc7a74bc6bdb885b8afa63a880572ad9b" dependencies = [ "half", - "libloading 0.8.9", + "libloading 0.9.0", ] [[package]] @@ -5766,6 +5766,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "liblzma" version = "0.4.6" diff --git a/Cargo.toml b/Cargo.toml index 63d6795506c..89d3e73c0d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ chrono = "0.4.42" clap = "4.5" criterion = "0.8" crossterm = "0.29" -cudarc = { version = "0.18.2", features = [ +cudarc = { version = "0.19.0", features = [ # The NSight Compute version available on lambda.ai hosts does not inject a symbol for # several functions that cudarc expects to load when this is set to "cuda-12080". We # don't use any CUDA 12.8 specific features currently so this is fine. From 12457aaf264dbf20b4291ee5af4774b80923eb15 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:42:05 -0400 Subject: [PATCH 136/250] Clean up `vortex-tensor` (#7525) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Given our fast velocity on this crate, quite a few things slipped through the cracks. This change cleans up the `vortex-tensor` crate by clearly defining the abstraction points, fixing a few bugs (the only real bug was a Tensor `Display` bug), cleaning up some TODOs, and generally raising the quality. ## API Changes The only relevant change is new helper functions ## Testing Just fixed up existing tests. --------- Signed-off-by: Connor Tsui --- vortex-array/public-api.lock | 2 + vortex-array/src/arrays/extension/array.rs | 15 + vortex-array/src/dtype/extension/vtable.rs | 20 +- vortex-tensor/public-api.lock | 14 +- vortex-tensor/src/encodings/l2_denorm.rs | 3 +- .../src/encodings/turboquant/centroids.rs | 10 +- .../src/encodings/turboquant/compress.rs | 249 ++++++++-------- vortex-tensor/src/encodings/turboquant/mod.rs | 25 +- .../src/encodings/turboquant/scheme.rs | 40 +-- .../src/encodings/turboquant/tests/compute.rs | 10 +- .../src/encodings/turboquant/tests/mod.rs | 72 +---- .../encodings/turboquant/tests/nullable.rs | 8 +- .../encodings/turboquant/tests/roundtrip.rs | 68 ++--- .../encodings/turboquant/tests/structural.rs | 12 +- vortex-tensor/src/fixed_shape/matcher.rs | 8 +- vortex-tensor/src/fixed_shape/metadata.rs | 39 +++ vortex-tensor/src/matcher.rs | 6 +- .../src/scalar_fns/cosine_similarity.rs | 99 +++---- vortex-tensor/src/scalar_fns/inner_product.rs | 278 +++++------------- vortex-tensor/src/scalar_fns/l2_denorm.rs | 143 ++++----- vortex-tensor/src/scalar_fns/l2_norm.rs | 30 +- .../src/scalar_fns/sorf_transform/vtable.rs | 68 +++-- vortex-tensor/src/utils.rs | 227 +++++++++----- vortex-tensor/src/vector/matcher.rs | 2 +- vortex-tensor/src/vector/mod.rs | 44 +++ vortex-tensor/src/vector_search.rs | 174 ++--------- vortex/benches/single_encoding_throughput.rs | 2 +- 27 files changed, 724 insertions(+), 944 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 8ae349c3da5..859ee6829e2 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -21546,6 +21546,8 @@ pub fn vortex_array::Array::new(ext_dtype: vort pub fn vortex_array::Array::try_new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_array: vortex_array::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_new_from_vtable(vtable: V, metadata: ::Metadata, storage_array: vortex_array::ArrayRef) -> vortex_error::VortexResult + impl vortex_array::Array pub fn vortex_array::Array::new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> Self diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 4ff2ad1bcfb..0a79774a2b5 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -13,7 +13,9 @@ use crate::array::ArrayParts; use crate::array::TypedArrayRef; use crate::arrays::Extension; use crate::dtype::DType; +use crate::dtype::extension::ExtDType; use crate::dtype::extension::ExtDTypeRef; +use crate::dtype::extension::ExtVTable; /// The backing storage array for this extension array. pub(super) const STORAGE_SLOT: usize = 0; @@ -163,4 +165,17 @@ impl Array { ) }) } + + /// Creates a new [`ExtensionArray`](crate::arrays::ExtensionArray) from a vtable, metadata, and + /// a storage array. + pub fn try_new_from_vtable( + vtable: V, + metadata: V::Metadata, + storage_array: ArrayRef, + ) -> VortexResult { + let ext_dtype = + ExtDType::::try_with_vtable(vtable, metadata, storage_array.dtype().clone())? + .erased(); + Self::try_new(ext_dtype, storage_array) + } } diff --git a/vortex-array/src/dtype/extension/vtable.rs b/vortex-array/src/dtype/extension/vtable.rs index 1b763f53635..d4a00fbdec4 100644 --- a/vortex-array/src/dtype/extension/vtable.rs +++ b/vortex-array/src/dtype/extension/vtable.rs @@ -14,8 +14,8 @@ use crate::scalar::ScalarValue; /// The public API for defining new extension types. /// -/// This is the non-object-safe trait that plugin authors implement to define a new extension -/// type. It specifies the type's identity, metadata, serialization, and validation. +/// This is the non-object-safe trait that plugin authors implement to define a new extension type. +/// It specifies the type's identity, metadata, serialization, and validation. pub trait ExtVTable: 'static + Sized + Send + Sync + Clone + Debug + Eq + Hash { /// Associated type containing the deserialized metadata for this extension type. type Metadata: 'static + Send + Sync + Clone + Debug + Display + Eq + Hash; @@ -39,11 +39,11 @@ pub trait ExtVTable: 'static + Sized + Send + Sync + Clone + Debug + Eq + Hash { /// Validate that the given storage type is compatible with this extension type. fn validate_dtype(ext_dtype: &ExtDType) -> VortexResult<()>; - /// Can a value of `other` be implicitly widened into this type? - /// e.g. GeographyType might accept Point, LineString, etc. + /// Can a value of `other` be implicitly widened into this type? (e.g. GeographyType might + /// accept Point, LineString, etc.) /// - /// Implementors only need to override one of `can_coerce_from` or `can_coerce_to` — both - /// exist so that either side of the coercion can provide the logic. + /// Implementors only need to override one of `can_coerce_from` or `can_coerce_to`. We have both + /// so that either side of the coercion can provide the logic. fn can_coerce_from(ext_dtype: &ExtDType, other: &DType) -> bool { let _ = (ext_dtype, other); false @@ -51,14 +51,15 @@ pub trait ExtVTable: 'static + Sized + Send + Sync + Clone + Debug + Eq + Hash { /// Can this type be implicitly widened into `other`? /// - /// Implementors only need to override one of `can_coerce_from` or `can_coerce_to` — both - /// exist so that either side of the coercion can provide the logic. + /// Implementors only need to override one of `can_coerce_from` or `can_coerce_to`. We have both + /// so that either side of the coercion can provide the logic. fn can_coerce_to(ext_dtype: &ExtDType, other: &DType) -> bool { let _ = (ext_dtype, other); false } /// Given two types in a Uniform context, what is their least supertype? + /// /// Return None if no supertype exists. fn least_supertype(ext_dtype: &ExtDType, other: &DType) -> Option { let _ = (ext_dtype, other); @@ -69,7 +70,8 @@ pub trait ExtVTable: 'static + Sized + Send + Sync + Clone + Debug + Eq + Hash { /// Validate the given storage value is compatible with the extension type. /// - /// By default, this calls [`unpack_native()`](ExtVTable::unpack_native) and discards the result. + /// By default, this calls [`unpack_native()`](ExtVTable::unpack_native) and discards the + /// result. /// /// # Errors /// diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 96b95e1e91e..d233322a3bf 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -80,7 +80,7 @@ pub const vortex_tensor::encodings::turboquant::MIN_DIMENSION: u32 pub fn vortex_tensor::encodings::turboquant::tq_validate_vector_dtype(dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult -pub fn vortex_tensor::encodings::turboquant::turboquant_encode(ext: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::extension::vtable::Extension>, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::turboquant::turboquant_encode(input: vortex_array::array::erased::ArrayRef, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub unsafe fn vortex_tensor::encodings::turboquant::turboquant_encode_unchecked(ext: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::extension::vtable::Extension>, config: &vortex_tensor::encodings::turboquant::TurboQuantConfig, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -142,7 +142,7 @@ impl vortex_tensor::fixed_shape::FixedShapeTensorMatcherMetadata<'_> pub fn vortex_tensor::fixed_shape::FixedShapeTensorMatcherMetadata<'_>::element_ptype(&self) -> vortex_array::dtype::ptype::PType -pub fn vortex_tensor::fixed_shape::FixedShapeTensorMatcherMetadata<'_>::list_size(&self) -> usize +pub fn vortex_tensor::fixed_shape::FixedShapeTensorMatcherMetadata<'_>::flat_list_size(&self) -> u32 pub fn vortex_tensor::fixed_shape::FixedShapeTensorMatcherMetadata<'_>::metadata(&self) -> &vortex_tensor::fixed_shape::FixedShapeTensorMetadata @@ -222,7 +222,7 @@ impl vortex_tensor::matcher::TensorMatch<'_> pub fn vortex_tensor::matcher::TensorMatch<'_>::element_ptype(self) -> vortex_array::dtype::ptype::PType -pub fn vortex_tensor::matcher::TensorMatch<'_>::list_size(self) -> usize +pub fn vortex_tensor::matcher::TensorMatch<'_>::list_size(self) -> u32 impl<'a> core::clone::Clone for vortex_tensor::matcher::TensorMatch<'a> @@ -382,7 +382,7 @@ pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::validity(&self, _options: pub fn vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm(input: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::l2_denorm::validate_l2_normalized_rows(input: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_tensor::scalar_fns::l2_denorm::validate_l2_normalized_rows_against_norms(normalized: &vortex_array::array::erased::ArrayRef, norms: core::option::Option<&vortex_array::array::erased::ArrayRef>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> pub mod vortex_tensor::scalar_fns::l2_norm @@ -502,7 +502,7 @@ pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::child_name(&sel pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::vtable::ExecutionArgs, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::expression::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::fmt_sql(&self, options: &Self::Options, expr: &vortex_array::expr::expression::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::id(&self) -> vortex_array::scalar_fn::ScalarFnId @@ -600,12 +600,8 @@ impl core::marker::StructuralPartialEq for vortex_tensor::vector::VectorMatcherM pub mod vortex_tensor::vector_search -pub fn vortex_tensor::vector_search::build_constant_query_vector>(query: &[T], num_rows: usize) -> vortex_error::VortexResult - pub fn vortex_tensor::vector_search::build_similarity_search_tree>(data: vortex_array::array::erased::ArrayRef, query: &[T], threshold: T) -> vortex_error::VortexResult -pub fn vortex_tensor::vector_search::compress_turboquant(data: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - pub const vortex_tensor::SCALAR_FN_ARRAY_TENSOR_PLUGIN_ENV: &str pub fn vortex_tensor::initialize(session: &vortex_session::VortexSession) diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index 172191abf6e..d29b2e94daf 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -19,9 +19,8 @@ use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; pub struct L2DenormScheme; impl Scheme for L2DenormScheme { - // TODO(connor): FIX THIS!!! fn scheme_name(&self) -> &'static str { - "vortex.tensor.UNSTABLE.l2_denorm" + "vortex.tensor.l2_denorm" } fn matches(&self, canonical: &Canonical) -> bool { diff --git a/vortex-tensor/src/encodings/turboquant/centroids.rs b/vortex-tensor/src/encodings/turboquant/centroids.rs index cd7b8b889ce..3111034e68f 100644 --- a/vortex-tensor/src/encodings/turboquant/centroids.rs +++ b/vortex-tensor/src/encodings/turboquant/centroids.rs @@ -11,6 +11,7 @@ use std::sync::LazyLock; +use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_utils::aliases::dash_map::DashMap; @@ -27,16 +28,15 @@ const CONVERGENCE_EPSILON: f64 = 1e-12; /// Number of numerical integration points for computing conditional expectations. const INTEGRATION_POINTS: usize = 1000; -// TODO(connor): Maybe we should just store an `ArrayRef` here? /// Global centroid cache keyed by (dimension, bit_width). -static CENTROID_CACHE: LazyLock>> = LazyLock::new(DashMap::default); +static CENTROID_CACHE: LazyLock>> = LazyLock::new(DashMap::default); /// Get or compute cached centroids for the given dimension and bit width. /// /// Returns `2^bit_width` centroids sorted in ascending order, representing optimal scalar /// quantization levels for the coordinate distribution after random rotation in /// `dimension`-dimensional space. -pub fn get_centroids(dimension: u32, bit_width: u8) -> VortexResult> { +pub fn get_centroids(dimension: u32, bit_width: u8) -> VortexResult> { vortex_ensure!( (1..=MAX_BIT_WIDTH).contains(&bit_width), "TurboQuant bit_width must be 1-{}, got {bit_width}", @@ -92,7 +92,7 @@ impl HalfIntExponent { /// The probability distribution function is: /// `f(x) = C_d * (1 - x^2)^((d-3)/2)` on `[-1, 1]` /// where `C_d` is the normalizing constant. -fn max_lloyd_centroids(dimension: u32, bit_width: u8) -> Vec { +fn max_lloyd_centroids(dimension: u32, bit_width: u8) -> Buffer { debug_assert!((1..=MAX_BIT_WIDTH).contains(&bit_width)); let num_centroids = 1usize << bit_width; @@ -288,7 +288,7 @@ mod tests { #[case(128, 4)] fn centroids_within_bounds(#[case] dim: u32, #[case] bits: u8) -> VortexResult<()> { let centroids = get_centroids(dim, bits)?; - for &val in ¢roids { + for &val in centroids.iter() { assert!( (-1.0..=1.0).contains(&val), "centroid out of [-1, 1]: {val}", diff --git a/vortex-tensor/src/encodings/turboquant/compress.rs b/vortex-tensor/src/encodings/turboquant/compress.rs index b0173bbe36c..40e24807091 100644 --- a/vortex-tensor/src/encodings/turboquant/compress.rs +++ b/vortex-tensor/src/encodings/turboquant/compress.rs @@ -15,16 +15,15 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Extension; -use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArray; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::dtype::Nullability; -use vortex_array::dtype::extension::ExtDType; -use vortex_array::extension::EmptyMetadata; use vortex_array::validity::Validity; +use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -35,7 +34,8 @@ use crate::encodings::turboquant::MIN_DIMENSION; use crate::encodings::turboquant::centroids::compute_centroid_boundaries; use crate::encodings::turboquant::centroids::find_nearest_centroid; use crate::encodings::turboquant::centroids::get_centroids; -use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows; +use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; @@ -64,129 +64,45 @@ impl Default for TurboQuantConfig { } } -/// Shared intermediate results from the quantization loop. -struct QuantizationResult { - centroids: Vec, - all_indices: BufferMut, - padded_dim: usize, -} - -/// Core quantization: rotate and quantize already-normalized rows. +/// Apply the full TurboQuant compression pipeline to a [`Vector`](crate::vector::Vector) +/// extension array: normalize the rows via [`normalize_as_l2_denorm`], quantize the normalized +/// child via [`turboquant_encode_unchecked`], and reattach the stored norms as the outer +/// [`L2Denorm`] wrapper. /// -/// The input `fsl` must contain non-nullable, unit-norm vectors (already L2-normalized). Null -/// vectors are not supported and must be zeroed out before reaching this function. The rotation -/// and centroid lookup happen in f32. -fn turboquant_quantize_core( - fsl: &FixedSizeListArray, - seed: u64, - bit_width: u8, - num_rounds: u8, - ctx: &mut ExecutionCtx, -) -> VortexResult { - let dimension = - usize::try_from(fsl.list_size()).vortex_expect("u32 FixedSizeList dimension fits in usize"); - let num_rows = fsl.len(); - - let rotation = SorfMatrix::try_new(seed, dimension, num_rounds as usize)?; - let padded_dim = rotation.padded_dim(); - let padded_dim_u32 = - u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); - - let elements_prim: PrimitiveArray = fsl.elements().clone().execute(ctx)?; - let f32_elements = cast_to_f32(elements_prim)?; - - let centroids = get_centroids(padded_dim_u32, bit_width)?; - let boundaries = compute_centroid_boundaries(¢roids); - - let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); - let mut padded = vec![0.0f32; padded_dim]; - let mut rotated = vec![0.0f32; padded_dim]; - - let f32_slice = f32_elements.as_slice(); - for row in 0..num_rows { - let x = &f32_slice[row * dimension..(row + 1) * dimension]; - - // Zero-pad to the next power of 2. - padded[..dimension].copy_from_slice(x); - padded[dimension..].fill(0.0); - - rotation.rotate(&padded, &mut rotated); - - for j in 0..padded_dim { - all_indices.push(find_nearest_centroid(rotated[j], &boundaries)); - } - } - - Ok(QuantizationResult { - centroids, - all_indices, - padded_dim, - }) -} - -/// Build a quantized representation: `FSL(DictArray(codes, centroids), padded_dim)`. +/// The returned array has the canonical TurboQuant shape: /// -/// This is a Dict-encoded FixedSizeList where each row of `padded_dim` u8 codes -/// indexes into the centroid codebook. The Dict can be independently sliced, taken, -/// or executed (dequantized) without knowledge of the rotation. -fn build_quantized_fsl( - num_rows: usize, - all_indices: BufferMut, - centroids: &[f32], - padded_dim: usize, -) -> VortexResult { - let codes = PrimitiveArray::new::(all_indices.freeze(), Validity::NonNullable); - - let mut centroids_buf = BufferMut::::with_capacity(centroids.len()); - centroids_buf.extend_from_slice(centroids); - let centroids_array = PrimitiveArray::new::(centroids_buf.freeze(), Validity::NonNullable); - - let dict = DictArray::try_new(codes.into_array(), centroids_array.into_array())?; - - let padded_dim_u32 = - u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); - Ok(FixedSizeListArray::try_new( - dict.into_array(), - padded_dim_u32, - Validity::NonNullable, - num_rows, - )? - .into_array()) -} - -/// Encode a non-nullable, L2-normalized [`Vector`](crate::vector::Vector) extension array into a -/// `ScalarFnArray(SorfTransform, [FSL(Dict(codes, centroids))])`. +/// ```text +/// ScalarFnArray(L2Denorm, [ +/// ScalarFnArray(SorfTransform, [FSL(Dict(codes, centroids))]), +/// norms, +/// ]) +/// ``` /// -/// The input must be a non-nullable Vector extension array whose rows are already unit-norm. -/// **Null vectors are not supported.** The caller must normalize and strip nullability before -/// calling this function, for example via [`normalize_as_l2_denorm`]. +/// # Errors /// -/// This function validates that every row is L2-normalized (or is exactly 0.0). Use -/// [`turboquant_encode_unchecked`] to skip this check when the caller has just performed -/// normalization. -/// -/// The returned array is a `SorfTransform` ScalarFnArray wrapping `FSL(Dict)` that decompresses -/// to unit-norm vectors. The caller is responsible for wrapping it in an [`L2Denorm`] ScalarFnArray -/// if the original magnitudes need to be restored. -/// -/// [`normalize_as_l2_denorm`]: crate::scalar_fns::l2_denorm::normalize_as_l2_denorm -/// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm +/// Returns an error if `input` is not a tensor-like extension array, if normalization fails, or +/// if [`turboquant_encode_unchecked`] rejects the input shape. pub fn turboquant_encode( - ext: ArrayView, + input: ArrayRef, config: &TurboQuantConfig, ctx: &mut ExecutionCtx, ) -> VortexResult { - let ext_dtype = ext.dtype().clone(); - - vortex_ensure!( - !ext_dtype.is_nullable(), - "TurboQuant input must be non-nullable (normalize first via L2Denorm), got {ext_dtype}", - ); - - validate_l2_normalized_rows(ext.as_ref(), ctx)?; - - // SAFETY: We just validated that the input is non-nullable and all rows are unit-norm. - unsafe { turboquant_encode_unchecked(ext, config, ctx) } + // We must normalize the array before we can encode it with TurboQuant. + let l2_denorm = normalize_as_l2_denorm(input, ctx)?; + let normalized = l2_denorm.child_at(0).clone(); + let norms = l2_denorm.child_at(1).clone(); + let num_rows = l2_denorm.len(); + + let normalized_ext = normalized + .as_opt::() + .vortex_expect("normalize_as_l2_denorm always produces an Extension array child"); + + // SAFETY: `normalize_as_l2_denorm` guarantees every row is unit-norm (or zero for null rows). + let tq = unsafe { turboquant_encode_unchecked(normalized_ext, config, ctx) }?; + + // SAFETY: TurboQuant is a lossy approximation of the normalized child, so we intentionally + // bypass the strict normalized-row validation when reattaching the stored norms. + Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) } /// Encode a non-nullable, L2-normalized [`Vector`](crate::vector::Vector) extension array into a @@ -240,7 +156,7 @@ pub unsafe fn turboquant_encode_unchecked( Validity::NonNullable, 0, )?; - let empty_padded_vector = wrap_padded_as_vector(empty_fsl.into_array())?; + let empty_padded_vector = Vector::try_new_vector_array(empty_fsl.into_array())?; let sorf_options = SorfOptions { seed, @@ -255,8 +171,8 @@ pub unsafe fn turboquant_encode_unchecked( let core = turboquant_quantize_core(&fsl, seed, config.bit_width, config.num_rounds, ctx)?; let quantized_fsl = - build_quantized_fsl(num_rows, core.all_indices, &core.centroids, core.padded_dim)?; - let padded_vector = wrap_padded_as_vector(quantized_fsl)?; + build_quantized_fsl(num_rows, core.all_indices, core.centroids, core.padded_dim)?; + let padded_vector = Vector::try_new_vector_array(quantized_fsl)?; let sorf_options = SorfOptions { seed, @@ -267,9 +183,88 @@ pub unsafe fn turboquant_encode_unchecked( Ok(SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array()) } -/// Wrap an `FSL` in a [`Vector`](crate::vector::Vector) extension so it can be -/// passed as the child of [`SorfTransform`], which expects a `Vector` input. -fn wrap_padded_as_vector(fsl: ArrayRef) -> VortexResult { - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl).into_array()) +/// Shared intermediate results from the quantization loop. +struct QuantizationResult { + centroids: Buffer, + all_indices: Buffer, + padded_dim: usize, +} + +/// Core quantization: rotate and quantize already-normalized rows. +/// +/// The input `fsl` must contain non-nullable, unit-norm vectors (already L2-normalized). Null +/// vectors are not supported and must be zeroed out before reaching this function. The rotation +/// and centroid lookup happen in f32. +fn turboquant_quantize_core( + fsl: &FixedSizeListArray, + seed: u64, + bit_width: u8, + num_rounds: u8, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let dimension = fsl.list_size() as usize; + let num_rows = fsl.len(); + + let rotation = SorfMatrix::try_new(seed, dimension, num_rounds as usize)?; + let padded_dim = rotation.padded_dim(); + let padded_dim_u32 = + u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); + + let elements_prim: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + let f32_elements = cast_to_f32(elements_prim)?; + + let centroids = get_centroids(padded_dim_u32, bit_width)?; + let boundaries = compute_centroid_boundaries(¢roids); + + let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); + let mut padded = vec![0.0f32; padded_dim]; + let mut rotated = vec![0.0f32; padded_dim]; + + let f32_slice = f32_elements.as_slice(); + for row in 0..num_rows { + let x = &f32_slice[row * dimension..(row + 1) * dimension]; + + // Zero-pad to the next power of 2. + padded[..dimension].copy_from_slice(x); + padded[dimension..].fill(0.0); + + rotation.rotate(&padded, &mut rotated); + + for j in 0..padded_dim { + all_indices.push(find_nearest_centroid(rotated[j], &boundaries)); + } + } + + Ok(QuantizationResult { + centroids, + all_indices: all_indices.freeze(), + padded_dim, + }) +} + +/// Build a quantized representation: `FSL(DictArray(codes, centroids), padded_dim)`. +/// +/// This is a Dict-encoded FixedSizeList where each row of `padded_dim` u8 codes indexes into the +/// centroid codebook. The Dict can be independently sliced, taken, or executed (dequantized) +/// without knowledge of the rotation. +fn build_quantized_fsl( + num_rows: usize, + all_indices: Buffer, + centroids: Buffer, + padded_dim: usize, +) -> VortexResult { + let codes = PrimitiveArray::new::(all_indices, Validity::NonNullable); + let centroids_array = PrimitiveArray::new::(centroids, Validity::NonNullable); + + let dict = DictArray::try_new(codes.into_array(), centroids_array.into_array())?; + + let padded_dim_u32 = + u32::try_from(padded_dim).vortex_expect("padded_dim stays representable as u32"); + Ok(FixedSizeListArray::try_new( + dict.into_array(), + padded_dim_u32, + Validity::NonNullable, + num_rows, + )? + .into_array()) } diff --git a/vortex-tensor/src/encodings/turboquant/mod.rs b/vortex-tensor/src/encodings/turboquant/mod.rs index 49e53effd0d..51480955594 100644 --- a/vortex-tensor/src/encodings/turboquant/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/mod.rs @@ -95,16 +95,12 @@ //! use vortex_array::arrays::ExtensionArray; //! use vortex_array::arrays::FixedSizeListArray; //! use vortex_array::arrays::PrimitiveArray; -//! use vortex_array::arrays::Extension; -//! use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -//! use vortex_array::dtype::extension::ExtDType; //! use vortex_array::extension::EmptyMetadata; +//! use vortex_array::session::ArraySession; //! use vortex_array::validity::Validity; //! use vortex_buffer::BufferMut; -//! use vortex_array::session::ArraySession; //! use vortex_session::VortexSession; -//! use vortex_tensor::encodings::turboquant::{TurboQuantConfig, turboquant_encode_unchecked}; -//! use vortex_tensor::scalar_fns::l2_denorm::normalize_as_l2_denorm; +//! use vortex_tensor::encodings::turboquant::{TurboQuantConfig, turboquant_encode}; //! use vortex_tensor::vector::Vector; //! //! // Create a Vector extension array of 100 random 128-d vectors. @@ -118,22 +114,15 @@ //! let fsl = FixedSizeListArray::try_new( //! elements.into_array(), dim, Validity::NonNullable, num_rows, //! ).unwrap(); -//! let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) -//! .unwrap().erased(); -//! let ext = ExtensionArray::new(ext_dtype, fsl.into_array()); +//! let vector = ExtensionArray::try_new_from_vtable(Vector, EmptyMetadata, fsl.into_array()) +//! .map(|ext| ext.into_array()) +//! .unwrap(); //! -//! // Normalize, then quantize the normalized child at 2 bits per coordinate. +//! // Normalize and quantize at 2 bits per coordinate in one pass. //! let session = VortexSession::empty().with::(); //! let mut ctx = session.create_execution_ctx(); -//! let l2_denorm = normalize_as_l2_denorm(ext.into_array(), &mut ctx).unwrap(); -//! let normalized = l2_denorm.child_at(0).clone(); -//! -//! let normalized_ext = normalized.as_opt::().unwrap(); //! let config = TurboQuantConfig { bit_width: 2, seed: Some(42), num_rounds: 3 }; -//! // SAFETY: We just normalized the input. -//! let tq = unsafe { -//! turboquant_encode_unchecked(normalized_ext, &config, &mut ctx).unwrap() -//! }; +//! let tq = turboquant_encode(vector, &config, &mut ctx).unwrap(); //! //! // Verify compression: 100 vectors x 128 dims x 4 bytes = 51200 bytes input. //! assert!(tq.nbytes() < 51200); diff --git a/vortex-tensor/src/encodings/turboquant/scheme.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs index b603f12e16e..81fca209a8d 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -3,8 +3,7 @@ //! TurboQuant compression scheme. //! -//! The scheme first normalizes the input via [`normalize_as_l2_denorm`], then encodes the -//! normalized child via [`turboquant_encode_unchecked`]. The result is: +//! The scheme is a thin [`Scheme`] adapter over [`turboquant_encode`], which produces: //! //! ```text //! ScalarFnArray(L2Denorm, [ @@ -18,14 +17,10 @@ //! //! Decompression is automatic: executing the outer array walks the ScalarFn tree. //! -//! [`normalize_as_l2_denorm`]: crate::scalar_fns::l2_denorm::normalize_as_l2_denorm -//! [`turboquant_encode_unchecked`]: crate::encodings::turboquant::turboquant_encode_unchecked +//! [`turboquant_encode`]: crate::encodings::turboquant::turboquant_encode use vortex_array::ArrayRef; use vortex_array::Canonical; -use vortex_array::IntoArray; -use vortex_array::arrays::Extension; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; use vortex_compressor::estimate::CompressionEstimate; @@ -38,9 +33,7 @@ use vortex_error::VortexResult; use crate::encodings::turboquant::MAX_CENTROIDS; use crate::encodings::turboquant::TurboQuantConfig; use crate::encodings::turboquant::tq_validate_vector_dtype; -use crate::encodings::turboquant::turboquant_encode_unchecked; -use crate::scalar_fns::l2_denorm::L2Denorm; -use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use crate::encodings::turboquant::turboquant_encode; /// TurboQuant compression scheme for [`Vector`] extension types. /// @@ -105,33 +98,8 @@ impl Scheme for TurboQuantScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { - let ext_array = data - .array() - .as_opt::() - .vortex_expect("expected an extension array"); - let mut ctx = compressor.execution_ctx(); - - // 1. Normalize: produces L2Denorm(normalized_vectors, norms). - let l2_denorm = normalize_as_l2_denorm(ext_array.as_ref().clone(), &mut ctx)?; - let normalized = l2_denorm.child_at(0).clone(); - let norms = l2_denorm.child_at(1).clone(); - let num_rows = l2_denorm.len(); - - // 2. Quantize the normalized child: SorfTransform(FSL(Dict)). - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - - let config = TurboQuantConfig::default(); - // SAFETY: We just normalized the input via `normalize_as_l2_denorm`, so all rows are - // guaranteed to be unit-norm (or zero for originally-null rows). - let sorf_dict = unsafe { turboquant_encode_unchecked(normalized_ext, &config, &mut ctx)? }; - - // 3. Wrap back in L2Denorm: the SorfTransform is the "normalized" child. - // SAFETY: TurboQuant is a lossy approximation of the normalized child, so we intentionally - // bypass the strict normalized-row validation when reattaching the stored norms. - Ok(unsafe { L2Denorm::new_array_unchecked(sorf_dict, norms, num_rows) }?.into_array()) + turboquant_encode(data.array().clone(), &TurboQuantConfig::default(), &mut ctx) } } diff --git a/vortex-tensor/src/encodings/turboquant/tests/compute.rs b/vortex-tensor/src/encodings/turboquant/tests/compute.rs index 0a9e0ab7a18..a26f0c54cfc 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/compute.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/compute.rs @@ -44,7 +44,7 @@ fn slice_preserves_data() -> VortexResult<()> { num_rounds: 4, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; // Full decompress then slice. let mut ctx = SESSION.create_execution_ctx(); @@ -89,7 +89,7 @@ fn scalar_at_matches_decompress() -> VortexResult<()> { num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let full_decoded = encoded.clone().execute::(&mut ctx)?; @@ -112,7 +112,7 @@ fn l2_norm_readthrough() -> VortexResult<()> { num_rounds: 5, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); // Stored norms should match the actual L2 norms of the input. @@ -150,7 +150,7 @@ fn l2_norm_readthrough_is_authoritative_for_lossy_storage() -> VortexResult<()> num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); let stored_norms: PrimitiveArray = norms_child.execute(&mut ctx)?; @@ -187,7 +187,7 @@ fn cosine_similarity_readthrough_is_authoritative_for_lossy_storage() -> VortexR num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let encoded_cos = execute_cosine_similarity(encoded.clone(), encoded.clone(), num_rows, &mut ctx)?; diff --git a/vortex-tensor/src/encodings/turboquant/tests/mod.rs b/vortex-tensor/src/encodings/turboquant/tests/mod.rs index b111c6e28ba..5b7e325b9b7 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/mod.rs @@ -16,7 +16,6 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Dict; -use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; @@ -25,17 +24,13 @@ use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -use vortex_array::dtype::extension::ExtDType; -use vortex_array::extension::EmptyMetadata; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::encodings::turboquant::TurboQuantConfig; -use crate::encodings::turboquant::turboquant_encode_unchecked; -use crate::scalar_fns::l2_denorm::L2Denorm; -use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; +use crate::encodings::turboquant::turboquant_encode; use crate::tests::SESSION; use crate::vector::Vector; @@ -71,31 +66,9 @@ fn make_fsl(num_rows: usize, dim: usize, seed: u64) -> FixedSizeListArray { } /// Wrap a `FixedSizeListArray` in a `Vector` extension array. -fn make_vector_ext(fsl: &FixedSizeListArray) -> ExtensionArray { - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone()) - .unwrap() - .erased(); - ExtensionArray::new(ext_dtype, fsl.clone().into_array()) -} - -/// Full encode pipeline: normalize → TQ-encode → wrap in L2Denorm. -fn normalize_and_encode( - ext: &ExtensionArray, - config: &TurboQuantConfig, - ctx: &mut vortex_array::ExecutionCtx, -) -> VortexResult { - let l2_denorm = normalize_as_l2_denorm(ext.as_ref().clone(), ctx)?; - let normalized = l2_denorm.child_at(0).clone(); - let norms = l2_denorm.child_at(1).clone(); - let num_rows = l2_denorm.len(); - - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - // SAFETY: We just normalized the input via `normalize_as_l2_denorm`. - let tq = unsafe { turboquant_encode_unchecked(normalized_ext, config, ctx)? }; - - Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) +fn make_vector_ext(fsl: &FixedSizeListArray) -> ArrayRef { + Vector::try_new_vector_array(fsl.clone().into_array()) + .vortex_expect("test FSL satisfies Vector storage constraints") } /// Unwrap an L2Denorm ScalarFnArray into (sorf_child, norms_child). @@ -103,17 +76,7 @@ fn unwrap_l2denorm(encoded: &ArrayRef) -> (ArrayRef, ArrayRef) { let sfn = encoded .as_opt::() .expect("expected ScalarFnArray (L2Denorm)"); - let sorf_child = sfn.child_at(0).clone(); - let norms_child = sfn.child_at(1).clone(); - (sorf_child, norms_child) -} - -/// Unwrap a SorfTransform ScalarFnArray to get the FSL(Dict) child. -fn unwrap_sorf(sorf: &ArrayRef) -> ArrayRef { - let sfn = sorf - .as_opt::() - .expect("expected ScalarFnArray (SorfTransform)"); - sfn.child_at(0).clone() + (sfn.child_at(0).clone(), sfn.child_at(1).clone()) } /// Navigate the full tree to get (codes, centroids, norms) as flat arrays. @@ -122,7 +85,11 @@ fn unwrap_codes_centroids_norms( ctx: &mut vortex_array::ExecutionCtx, ) -> VortexResult<(PrimitiveArray, PrimitiveArray, PrimitiveArray)> { let (sorf_child, norms_child) = unwrap_l2denorm(encoded); - let padded_vector_child = unwrap_sorf(&sorf_child); + let padded_vector_child = sorf_child + .as_opt::() + .expect("expected SorfTransform ScalarFnArray") + .child_at(0) + .clone(); // Vector wrapping FSL(Dict(codes, centroids)) let padded_vector: ExtensionArray = padded_vector_child.execute(ctx)?; @@ -177,8 +144,7 @@ fn encode_decode( let prim = fsl.elements().clone().execute::(&mut ctx)?; prim.as_slice::().to_vec() }; - let ext = make_vector_ext(fsl); - let encoded = normalize_and_encode(&ext, config, &mut ctx)?; + let encoded = turboquant_encode(make_vector_ext(fsl), config, &mut ctx)?; let decoded_ext = encoded.execute::(&mut ctx)?; let decoded_fsl = decoded_ext .storage_array() @@ -193,19 +159,3 @@ fn encode_decode( }; Ok((original, decoded_elements)) } - -fn make_fsl_small(dim: usize) -> FixedSizeListArray { - let mut buf = BufferMut::::with_capacity(dim); - for i in 0..dim { - buf.push(i as f32 + 1.0); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), - Validity::NonNullable, - 1, - ) - .unwrap() -} diff --git a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs index 6fc19bb93ec..41124c27c80 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs @@ -27,7 +27,7 @@ fn nullable_vectors_roundtrip() -> VortexResult<()> { num_rounds: 4, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; assert_eq!(encoded.len(), 10); assert!(encoded.dtype().is_nullable()); @@ -88,7 +88,7 @@ fn nullable_norms_match_validity() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); let norms_validity = norms_child.validity()?; @@ -118,7 +118,7 @@ fn nullable_l2_norm_readthrough() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let norm_sfn = L2Norm::try_new_array(encoded, 5)?; let norms: PrimitiveArray = norm_sfn.into_array().execute(&mut ctx)?; @@ -160,7 +160,7 @@ fn nullable_slice_preserves_validity() -> VortexResult<()> { num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let sliced = encoded.slice(1..6)?; assert_eq!(sliced.len(), 5); diff --git a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs index cd61d9193da..dbe04f2e606 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs @@ -4,6 +4,7 @@ use rstest::rstest; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; @@ -12,6 +13,8 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use super::*; +use crate::encodings::turboquant::turboquant_encode_unchecked; +use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; #[rstest] #[case(128, 1)] @@ -130,7 +133,7 @@ fn roundtrip_edge_cases(#[case] num_rows: usize) -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let decoded = encoded.execute::(&mut ctx)?; assert_eq!(decoded.len(), num_rows); Ok(()) @@ -141,7 +144,17 @@ fn roundtrip_edge_cases(#[case] num_rows: usize) -> VortexResult<()> { #[case(64)] #[case(127)] fn rejects_dimension_below_128(#[case] dim: usize) { - let fsl = make_fsl_small(dim); + let elements = PrimitiveArray::new::( + BufferMut::from_iter((0..dim).map(|i| i as f32 + 1.0)).freeze(), + Validity::NonNullable, + ); + let fsl = FixedSizeListArray::try_new( + elements.into_array(), + dim.try_into().expect("dim fits u32"), + Validity::NonNullable, + 1, + ) + .unwrap(); let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 2, @@ -149,9 +162,7 @@ fn rejects_dimension_below_128(#[case] dim: usize) { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - assert!( - crate::encodings::turboquant::turboquant_encode(ext.as_view(), &config, &mut ctx).is_err() - ); + assert!(turboquant_encode(ext, &config, &mut ctx).is_err()); } #[rstest] @@ -166,7 +177,7 @@ fn rejects_invalid_bit_width(#[case] bit_width: u8) { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let normalized = normalize_as_l2_denorm(ext.as_ref().clone(), &mut ctx) + let normalized = normalize_as_l2_denorm(ext, &mut ctx) .unwrap() .child_at(0) .clone(); @@ -255,7 +266,7 @@ fn f64_input_encodes_successfully() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); assert_eq!(norms_child.len(), num_rows); Ok(()) @@ -288,7 +299,7 @@ fn f16_input_encodes_successfully() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_sorf_child, norms_child) = unwrap_l2denorm(&encoded); assert_eq!(norms_child.len(), num_rows); @@ -300,44 +311,3 @@ fn f16_input_encodes_successfully() -> VortexResult<()> { assert_eq!(decoded_fsl.len(), num_rows); Ok(()) } - -/// Verify that the checked encode accepts normalized f16 input. -#[test] -fn checked_encode_accepts_normalized_f16_input() -> VortexResult<()> { - let num_rows = 10; - let dim = 128; - let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f32, 1.0).unwrap(); - - let mut buf = BufferMut::::with_capacity(num_rows * dim); - for _ in 0..(num_rows * dim) { - buf.push(half::f16::from_f32(normal.sample(&mut rng))); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim.try_into().unwrap(), - Validity::NonNullable, - num_rows, - )?; - - let ext = make_vector_ext(&fsl); - let config = TurboQuantConfig { - bit_width: 3, - seed: Some(42), - num_rounds: 3, - }; - - let mut ctx = SESSION.create_execution_ctx(); - let normalized = normalize_as_l2_denorm(ext.as_ref().clone(), &mut ctx)? - .child_at(0) - .clone(); - let normalized_ext = normalized - .as_opt::() - .vortex_expect("normalized child should be an Extension array"); - - let encoded = - crate::encodings::turboquant::turboquant_encode(normalized_ext, &config, &mut ctx)?; - assert_eq!(encoded.len(), num_rows); - Ok(()) -} diff --git a/vortex-tensor/src/encodings/turboquant/tests/structural.rs b/vortex-tensor/src/encodings/turboquant/tests/structural.rs index 87b59836b38..7cd2cdfcf66 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/structural.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/structural.rs @@ -24,7 +24,7 @@ fn stored_centroids_match_computed() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let (_codes, centroids, _norms) = unwrap_codes_centroids_norms(&encoded, &mut ctx)?; let stored = centroids.as_slice::(); @@ -52,7 +52,7 @@ fn seed_deterministic_rotation_produces_correct_decode() -> VortexResult<()> { // Encode twice with the same seed → should produce identical results. let mut ctx = SESSION.create_execution_ctx(); - let encoded1 = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded1 = turboquant_encode(ext.clone(), &config, &mut ctx)?; let decoded1 = encoded1.execute::(&mut ctx)?; let fsl1 = decoded1 .storage_array() @@ -64,7 +64,7 @@ fn seed_deterministic_rotation_produces_correct_decode() -> VortexResult<()> { .execute::(&mut ctx)?; let mut ctx = SESSION.create_execution_ctx(); - let encoded2 = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded2 = turboquant_encode(ext, &config, &mut ctx)?; let decoded2 = encoded2.execute::(&mut ctx)?; let fsl2 = decoded2 .storage_array() @@ -94,7 +94,7 @@ fn encoded_dtype_is_vector_extension() -> VortexResult<()> { num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; assert!( encoded.dtype().is_extension(), @@ -119,7 +119,7 @@ fn cosine_similarity_quantized_accuracy() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let input_prim = fsl.elements().clone().execute::(&mut ctx)?; let input_f32 = input_prim.as_slice::(); @@ -176,7 +176,7 @@ fn dot_product_quantized_accuracy() -> VortexResult<()> { num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); - let encoded = normalize_and_encode(&ext, &config, &mut ctx)?; + let encoded = turboquant_encode(ext, &config, &mut ctx)?; let input_prim = fsl.elements().clone().execute::(&mut ctx)?; let input_f32 = input_prim.as_slice::(); diff --git a/vortex-tensor/src/fixed_shape/matcher.rs b/vortex-tensor/src/fixed_shape/matcher.rs index 5248ca6514b..f703ccba978 100644 --- a/vortex-tensor/src/fixed_shape/matcher.rs +++ b/vortex-tensor/src/fixed_shape/matcher.rs @@ -31,7 +31,7 @@ pub struct FixedShapeTensorMatcherMetadata<'a> { /// /// This matches the `FixedSizeList` list size in the storage dtype, which is the product of /// the logical shape dimensions. - flat_list_size: usize, + flat_list_size: u32, } impl Matcher for AnyFixedShapeTensor { @@ -64,7 +64,7 @@ impl Matcher for AnyFixedShapeTensor { Some(FixedShapeTensorMatcherMetadata { metadata, element_ptype: element_dtype.as_ptype(), - flat_list_size: *list_size as usize, + flat_list_size: *list_size, }) } } @@ -81,7 +81,7 @@ impl FixedShapeTensorMatcherMetadata<'_> { } /// Returns the flattened element count for each tensor row. - pub fn list_size(&self) -> usize { + pub fn flat_list_size(&self) -> u32 { self.flat_list_size } } @@ -118,7 +118,7 @@ mod tests { let metadata = ext_dtype.metadata::(); assert_eq!(metadata.element_ptype(), PType::F32); - assert_eq!(metadata.list_size(), 24); + assert_eq!(metadata.flat_list_size(), 24); assert_eq!(metadata.metadata().logical_shape(), &[2, 3, 4]); Ok(()) } diff --git a/vortex-tensor/src/fixed_shape/metadata.rs b/vortex-tensor/src/fixed_shape/metadata.rs index 264d18453c4..757138d3e50 100644 --- a/vortex-tensor/src/fixed_shape/metadata.rs +++ b/vortex-tensor/src/fixed_shape/metadata.rs @@ -215,6 +215,7 @@ impl fmt::Display for FixedShapeTensorMetadata { } if let Some(perm) = &self.permutation { + write!(f, ", [")?; for (i, p) in perm.iter().enumerate() { if i > 0 { write!(f, ", ")?; @@ -353,6 +354,44 @@ mod tests { Ok(()) } + // -- Display -- + + #[test] + fn display_shape_only() { + let m = FixedShapeTensorMetadata::new(vec![2, 3, 4]); + assert_eq!(m.to_string(), "Tensor(2, 3, 4)"); + } + + #[test] + fn display_scalar_0d() { + let m = FixedShapeTensorMetadata::new(vec![]); + assert_eq!(m.to_string(), "Tensor()"); + } + + #[test] + fn display_with_dim_names() -> VortexResult<()> { + let m = FixedShapeTensorMetadata::new(vec![3, 4]) + .with_dim_names(vec!["rows".into(), "cols".into()])?; + assert_eq!(m.to_string(), "Tensor(rows: 3, cols: 4)"); + Ok(()) + } + + #[test] + fn display_with_permutation() -> VortexResult<()> { + let m = FixedShapeTensorMetadata::new(vec![2, 3, 4]).with_permutation(vec![1, 0, 2])?; + assert_eq!(m.to_string(), "Tensor(2, 3, 4, [1, 0, 2])"); + Ok(()) + } + + #[test] + fn display_with_dim_names_and_permutation() -> VortexResult<()> { + let m = FixedShapeTensorMetadata::new(vec![2, 3, 4]) + .with_dim_names(vec!["x".into(), "y".into(), "z".into()])? + .with_permutation(vec![1, 2, 0])?; + assert_eq!(m.to_string(), "Tensor(x: 2, y: 3, z: 4, [1, 2, 0])"); + Ok(()) + } + #[test] fn dim_names_wrong_length() { let result = FixedShapeTensorMetadata::new(vec![2, 3]).with_dim_names(vec!["x".into()]); diff --git a/vortex-tensor/src/matcher.rs b/vortex-tensor/src/matcher.rs index 973562be3da..9ff58d17c4f 100644 --- a/vortex-tensor/src/matcher.rs +++ b/vortex-tensor/src/matcher.rs @@ -42,10 +42,10 @@ impl TensorMatch<'_> { } /// Returns the flattened element count for each logical tensor row. - pub fn list_size(self) -> usize { + pub fn list_size(self) -> u32 { match self { - Self::FixedShapeTensor(metadata) => metadata.list_size(), - Self::Vector(metadata) => metadata.dimensions() as usize, + Self::FixedShapeTensor(metadata) => metadata.flat_list_size(), + Self::Vector(metadata) => metadata.dimensions(), } } } diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 9f8eff0b361..85d16236c8c 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -11,7 +11,6 @@ use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; -use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; @@ -32,16 +31,15 @@ use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; -use vortex_error::vortex_ensure; use vortex_session::VortexSession; use crate::scalar_fns::inner_product::BinaryTensorOpMetadata; use crate::scalar_fns::inner_product::InnerProduct; -use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::DenormOrientation; use crate::scalar_fns::l2_denorm::try_build_constant_l2_denorm; use crate::scalar_fns::l2_norm::L2Norm; use crate::utils::extract_l2_denorm_children; -use crate::utils::validate_tensor_float_input; +use crate::utils::validate_binary_tensor_float_inputs; /// Cosine similarity between two columns. /// @@ -59,6 +57,7 @@ use crate::utils::validate_tensor_float_input; /// /// [`FixedShapeTensor`]: crate::fixed_shape::FixedShapeTensor /// [`Vector`]: crate::vector::Vector +/// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm #[derive(Clone)] pub struct CosineSimilarity; @@ -84,7 +83,7 @@ impl ScalarFnVTable for CosineSimilarity { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.tensor.cosine_similarity") + ScalarFnId::new("vortex.tensor.cosine_similarity") } fn arity(&self, _options: &Self::Options) -> Arity { @@ -116,16 +115,8 @@ impl ScalarFnVTable for CosineSimilarity { let lhs = &arg_dtypes[0]; let rhs = &arg_dtypes[1]; - // Both must have the same dtype (ignoring top-level nullability). - vortex_ensure!( - lhs.eq_ignore_nullability(rhs), - "CosineSimilarity requires both inputs to have the same dtype, got {lhs} and {rhs}" - ); - - // We don't need to look at rhs anymore since we know lhs and rhs are equal. - let tensor_match = validate_tensor_float_input(lhs)?; + let tensor_match = validate_binary_tensor_float_inputs("CosineSimilarity", lhs, rhs)?; let ptype = tensor_match.element_ptype(); - let nullability = Nullability::from(lhs.is_nullable() || rhs.is_nullable()); Ok(DType::Primitive(ptype, nullability)) } @@ -141,32 +132,24 @@ impl ScalarFnVTable for CosineSimilarity { let len = args.row_count(); // If either side is a constant tensor-like extension array, eagerly normalize the single - // stored row and re-wrap it as an `L2Denorm` whose children are both [`ConstantArray`]s. + // stored row and re-wrap it as an `L2Denorm` whose children are both `ConstantArray`s. // The L2Denorm fast path below then picks it up. - if let Some(lhs_constant) = - try_build_constant_l2_denorm(&lhs_ref, len, ctx)?.map(|sfn| sfn.into_array()) - { - lhs_ref = lhs_constant; + if let Some(sfn) = try_build_constant_l2_denorm(&lhs_ref, len, ctx)? { + lhs_ref = sfn.into_array(); } - if let Some(rhs_constant) = - try_build_constant_l2_denorm(&rhs_ref, len, ctx)?.map(|sfn| sfn.into_array()) - { - rhs_ref = rhs_constant; + if let Some(sfn) = try_build_constant_l2_denorm(&rhs_ref, len, ctx)? { + rhs_ref = sfn.into_array(); } - // Check if any of our children have be already normalized. - { - let lhs_is_denorm = lhs_ref.is::>(); - let rhs_is_denorm = rhs_ref.is::>(); - - if lhs_is_denorm && rhs_is_denorm { - return self.execute_both_denorm(&lhs_ref, &rhs_ref, len, ctx); - } else if lhs_is_denorm || rhs_is_denorm { - if rhs_is_denorm { - (lhs_ref, rhs_ref) = (rhs_ref, lhs_ref); - } - return self.execute_one_denorm(&lhs_ref, &rhs_ref, len, ctx); + // Take any L2Denorm-wrapped fast path that applies. + match DenormOrientation::classify(&lhs_ref, &rhs_ref) { + DenormOrientation::Both { lhs, rhs } => { + return self.execute_both_denorm(lhs, rhs, len); + } + DenormOrientation::One { denorm, plain } => { + return self.execute_one_denorm(denorm, plain, len, ctx); } + DenormOrientation::Neither => {} } // Compute combined validity. @@ -266,7 +249,6 @@ impl CosineSimilarity { lhs_ref: &ArrayRef, rhs_ref: &ArrayRef, len: usize, - _ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = lhs_ref.validity()?.and(rhs_ref.validity()?)?; @@ -347,9 +329,10 @@ mod tests { use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; - use crate::utils::test_helpers::constant_vector_array; + use crate::utils::test_helpers::l2_denorm_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; + use crate::vector::Vector; /// Evaluates cosine similarity between two tensor arrays and returns the result as `Vec`. fn eval_cosine_similarity(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { @@ -508,7 +491,7 @@ mod tests { 1.0, 0.0, 0.0, // vector 3 ], )?; - let query = constant_vector_array(&[1.0, 0.0, 0.0], 4)?; + let query = Vector::constant_array(&[1.0, 0.0, 0.0], 4)?; assert_close( &eval_cosine_similarity(data, query, 4)?, @@ -536,25 +519,13 @@ mod tests { Ok(()) } - /// Creates an `L2Denorm` scalar function array from pre-normalized elements and norms. - fn l2_denorm_array( - shape: &[usize], - normalized_elements: &[f64], - norms: &[f64], - ) -> VortexResult { - let len = norms.len(); - let normalized = tensor_array(shape, normalized_elements)?; - let norms = PrimitiveArray::from_iter(norms.iter().copied()).into_array(); - let mut ctx = SESSION.create_execution_ctx(); - Ok(L2Denorm::try_new_array(normalized, norms, len, &mut ctx)?.into_array()) - } - #[test] fn both_denorm_self_similarity() -> VortexResult<()> { // [3.0, 4.0] has norm 5.0, normalized [0.6, 0.8]. // [1.0, 0.0] has norm 1.0, normalized [1.0, 0.0]. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; - let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; // Self-similarity should always be 1.0. assert_close(&eval_cosine_similarity(lhs, rhs, 2)?, &[1.0, 1.0]); @@ -565,8 +536,9 @@ mod tests { fn both_denorm_orthogonal() -> VortexResult<()> { // [3.0, 0.0] normalized [1.0, 0.0], norm 3.0. // [0.0, 4.0] normalized [0.0, 1.0], norm 4.0. - let lhs = l2_denorm_array(&[2], &[1.0, 0.0], &[3.0])?; - let rhs = l2_denorm_array(&[2], &[0.0, 1.0], &[4.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[1.0, 0.0], &[3.0], &mut ctx)?; + let rhs = l2_denorm_array(&[2], &[0.0, 1.0], &[4.0], &mut ctx)?; assert_close(&eval_cosine_similarity(lhs, rhs, 1)?, &[0.0]); Ok(()) @@ -575,8 +547,9 @@ mod tests { #[test] fn both_denorm_zero_norm() -> VortexResult<()> { // Zero-norm row: normalized is [0.0, 0.0], norm is 0.0. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 0.0, 0.0], &[5.0, 0.0])?; - let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 0.0, 0.0], &[5.0, 0.0], &mut ctx)?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; // Row 0: dot([0.6, 0.8], [0.6, 0.8]) = 1.0, row 1: dot([0,0], [1,0]) = 0.0. assert_close(&eval_cosine_similarity(lhs, rhs, 2)?, &[1.0, 0.0]); @@ -588,7 +561,8 @@ mod tests { // LHS is L2Denorm([0.6, 0.8], 5.0) representing [3.0, 4.0]. // RHS is plain [3.0, 4.0]. // cosine_similarity([3.0, 4.0], [3.0, 4.0]) = 1.0. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0], &mut ctx)?; let rhs = tensor_array(&[2], &[3.0, 4.0])?; assert_close(&eval_cosine_similarity(lhs, rhs, 1)?, &[1.0]); @@ -599,8 +573,9 @@ mod tests { fn one_side_denorm_rhs() -> VortexResult<()> { // LHS is plain [1.0, 0.0], RHS is L2Denorm([0.6, 0.8], 5.0) representing [3.0, 4.0]. // cosine_similarity([1.0, 0.0], [3.0, 4.0]) = 3.0 / (1.0 * 5.0) = 0.6. + let mut ctx = SESSION.create_execution_ctx(); let lhs = tensor_array(&[2], &[1.0, 0.0])?; - let rhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0])?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0], &mut ctx)?; assert_close(&eval_cosine_similarity(lhs, rhs, 1)?, &[0.6]); Ok(()) @@ -609,11 +584,11 @@ mod tests { #[test] fn both_denorm_null_norms() -> VortexResult<()> { // Row 0: valid, row 1: null (via nullable norms on rhs). - let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; let normalized_r = tensor_array(&[2], &[0.6, 0.8, 1.0, 0.0])?; let norms_r = PrimitiveArray::from_option_iter([Some(5.0f64), None]).into_array(); - let mut ctx = SESSION.create_execution_ctx(); let rhs = L2Denorm::try_new_array(normalized_r, norms_r, 2, &mut ctx)?.into_array(); let scalar_fn = CosineSimilarity::new().erased(); @@ -711,7 +686,7 @@ mod tests { #[test] fn vector_constant_matches_plain() -> VortexResult<()> { // Exercise the `Vector` extension variant through the new pre-pass. - let lhs = constant_vector_array(&[1.0, 2.0, 2.0], 4)?; + let lhs = Vector::constant_array(&[1.0, 2.0, 2.0], 4)?; let rhs = vector_array( 3, &[ diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index dd9c2a7381f..22b28f9e4d5 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -4,7 +4,6 @@ //! Inner product expression for tensor-like types. use std::fmt::Formatter; -use std::sync::Arc; use num_traits::Float; use prost::Message; @@ -32,13 +31,10 @@ use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::expr::and; -use vortex_array::extension::EmptyMetadata; use vortex_array::match_each_float_ptype; -use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; @@ -56,11 +52,13 @@ use vortex_error::vortex_err; use vortex_session::VortexSession; use crate::matcher::AnyTensor; -use crate::scalar_fns::l2_denorm::L2Denorm; +use crate::scalar_fns::l2_denorm::DenormOrientation; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfTransform; +use crate::utils::extract_constant_flat_row; use crate::utils::extract_flat_elements; use crate::utils::extract_l2_denorm_children; +use crate::utils::validate_binary_tensor_float_inputs; use crate::vector::Vector; /// Inner product (dot product) between two columns. @@ -99,7 +97,7 @@ impl ScalarFnVTable for InnerProduct { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.tensor.inner_product") + ScalarFnId::new("vortex.tensor.inner_product") } fn arity(&self, _options: &Self::Options) -> Arity { @@ -131,32 +129,9 @@ impl ScalarFnVTable for InnerProduct { let lhs = &arg_dtypes[0]; let rhs = &arg_dtypes[1]; - // Both must have the same dtype (ignoring top-level nullability). - vortex_ensure!( - lhs.eq_ignore_nullability(rhs), - "InnerProduct requires both inputs to have the same dtype, got {lhs} and {rhs}" - ); - - // Both inputs must be tensor-like extension types. - let lhs_ext = lhs - .as_extension_opt() - .ok_or_else(|| vortex_err!("InnerProduct lhs must be an extension type, got {lhs}"))?; - - vortex_ensure!( - lhs_ext.is::(), - "InnerProduct inputs must be an `AnyTensor`, got {lhs}" - ); - - let tensor_match = lhs_ext - .metadata_opt::() - .ok_or_else(|| vortex_err!("InnerProduct inputs must be an `AnyTensor`, got {lhs}"))?; + // TODO(connor): relax the float-only gate once integer tensors are supported. + let tensor_match = validate_binary_tensor_float_inputs("InnerProduct", lhs, rhs)?; let ptype = tensor_match.element_ptype(); - // TODO(connor): This should support integer tensors! - vortex_ensure!( - ptype.is_float(), - "InnerProduct element dtype must be a float primitive, got {ptype}" - ); - let nullability = Nullability::from(lhs.is_nullable() || rhs.is_nullable()); Ok(DType::Primitive(ptype, nullability)) } @@ -167,23 +142,19 @@ impl ScalarFnVTable for InnerProduct { args: &dyn ExecutionArgs, ctx: &mut ExecutionCtx, ) -> VortexResult { - let mut lhs_ref = args.get(0)?; - let mut rhs_ref = args.get(1)?; + let lhs_ref = args.get(0)?; + let rhs_ref = args.get(1)?; let len = args.row_count(); - // Check if any of our children have be already normalized. - { - let lhs_is_denorm = lhs_ref.is::>(); - let rhs_is_denorm = rhs_ref.is::>(); - - if lhs_is_denorm && rhs_is_denorm { - return self.execute_both_denorm(&lhs_ref, &rhs_ref, len, ctx); - } else if lhs_is_denorm || rhs_is_denorm { - if rhs_is_denorm { - (lhs_ref, rhs_ref) = (rhs_ref, lhs_ref); - } - return self.execute_one_denorm(&lhs_ref, &rhs_ref, len, ctx); + // Take any L2Denorm-wrapped fast path that applies. + match DenormOrientation::classify(&lhs_ref, &rhs_ref) { + DenormOrientation::Both { lhs, rhs } => { + return self.execute_both_denorm(lhs, rhs, len, ctx); } + DenormOrientation::One { denorm, plain } => { + return self.execute_one_denorm(denorm, plain, len, ctx); + } + DenormOrientation::Neither => {} } // Reduction case 1: `InnerProduct(SorfTransform(x), const)` rewrites to @@ -212,7 +183,7 @@ impl ScalarFnVTable for InnerProduct { let tensor_match = ext .metadata_opt::() .vortex_expect("we already validated this in `return_dtype`"); - let dimensions = tensor_match.list_size(); + let dimensions = tensor_match.list_size() as usize; // Extract the storage array from each extension input. We pass the storage (FSL) rather // than the extension array to avoid canonicalizing the extension wrapper. @@ -409,20 +380,21 @@ impl InnerProduct { /// Fast path when one side is `ExactScalarFn` and the other side is a /// constant-backed tensor-like extension. Rewrites to /// `InnerProduct(sorf_child, forward_rotate(zero_pad(const_query)))` because SORF is - /// orthogonal, so ` = ` where `T` is the truncation - /// from `padded_dim` to `dim` applied by `SorfTransform` and `R` is the SORF forward - /// matrix. See the proof in the crate-level docs and in the plan file. + /// orthogonal, so ` = ` where `T` is the truncation from + /// `padded_dim` to `dim` applied by `SorfTransform` and `R` is the SORF forward matrix. See the + /// proof in the crate-level docs and in the plan file. /// - /// Returns `Ok(None)` if neither side matches or when `element_ptype` is not `F32`. The - /// caller is expected to fall through to the standard path in that case. + /// Returns `Ok(None)` if neither side matches, when the operand element type is not `F32`, or + /// when the constant side is not a constant-backed tensor extension. The caller is expected to + /// fall through to the standard path in that case. /// - /// # TODO(connor): + /// # F32-only /// - /// This rewrite is only sound for `PType::F32` because `SorfTransform` applies an - /// `f32 -> element_ptype` cast at the end of its execute (see `sorf_transform/vtable.rs` - /// line ~218). For F16/F64 the cast changes the inner product's rounding and would - /// change the semantics of the rewrite. Until we push the cast through `InnerProduct`, - /// this path only fires for F32. + /// TODO(connor): this rewrite is only sound for `PType::F32` because `SorfTransform` applies an + /// `f32 -> element_ptype` cast at the end of its `execute`. For `F16`/`F64` the cast changes + /// the inner product's rounding and the rewrite would not be semantically equivalent. Until we + /// push the cast through `InnerProduct`, both the SorfTransform output ptype and the + /// constant-side element ptype must be `F32` here. fn try_execute_sorf_constant( &self, lhs_ref: &ArrayRef, @@ -440,10 +412,6 @@ impl InnerProduct { return Ok(None); }; - // TODO(connor): pull-through is only sound for F32 because SorfTransform applies an - // `f32 -> element_ptype` cast at the end of its execute. For F16/F64 the rewrite - // would change the inner product's rounding semantics. Fall through so the standard - // path (which does the cast before inner product) handles it. if sorf_view.options.element_ptype != PType::F32 { return Ok(None); } @@ -458,45 +426,24 @@ impl InnerProduct { let seed = sorf_view.options.seed; let padded_dim = dim.next_power_of_two(); - // Extract the single stored row of the constant via the stride-0 short-circuit. - let flat = extract_flat_elements(&const_storage, dim, ctx)?; + // Extract the single stored row of the constant. + let flat = extract_constant_flat_row(&const_storage, ctx)?; if flat.ptype() != PType::F32 { - // TODO(connor): as above, f16/f64 are not supported by this rewrite yet. The - // standard path handles them correctly. return Ok(None); } // Zero-pad the query from `dim` to `padded_dim` and forward-rotate. let mut padded_query = vec![0.0f32; padded_dim]; - padded_query[..dim].copy_from_slice(flat.row::(0)); + padded_query[..dim].copy_from_slice(flat.as_slice::()); let rotation = SorfMatrix::try_new(seed, dim, num_rounds)?; let mut rotated_query = vec![0.0f32; padded_dim]; rotation.rotate(&padded_query, &mut rotated_query); - // Build the rewritten constant as a `Vector` extension scalar. We reuse - // the original storage FSL nullability so the new extension dtype stays consistent with - // whatever the original tree expected. - let storage_fsl_nullability = const_storage.dtype().nullability(); - let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - let children: Vec = rotated_query - .into_iter() - .map(|v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let fsl_scalar = - Scalar::fixed_size_list(element_dtype.clone(), children, storage_fsl_nullability); - - // Build a fresh `Vector` extension dtype. We cannot reuse the - // original extension dtype because that one has `dim`, not `padded_dim`. - let padded_dim_u32 = u32::try_from(padded_dim).vortex_expect("padded_dim fits u32"); - let new_fsl_dtype = DType::FixedSizeList( - Arc::new(element_dtype), - padded_dim_u32, - storage_fsl_nullability, - ); - let new_ext_dtype = ExtDType::::try_new(EmptyMetadata, new_fsl_dtype)?.erased(); - let new_constant = - ConstantArray::new(Scalar::extension_ref(new_ext_dtype, fsl_scalar), len).into_array(); + // Wrap the rotated query as a `Vector` constant broadcast to `len` + // rows. The new extension dtype has `padded_dim` instead of `dim`, matching the + // SorfTransform child we are about to dot it with. + let new_constant = Vector::constant_array(&rotated_query, len)?; // Extract the SorfTransform child (the already-padded Vector). let sorf_child = sorf_view @@ -575,8 +522,7 @@ impl InnerProduct { // Gate: u8 codes and f32 centroids. if codes_prim.ptype() != PType::U8 { - // TODO(connor): support wider code widths (u16, u32). TurboQuant only emits u8 - // codes today, so this is the only path we need for now. + // TODO(connor): Should we support wider codes? return Ok(None); } if values_prim.ptype() != PType::F32 { @@ -587,7 +533,7 @@ impl InnerProduct { let padded_dim = usize::try_from(fsl.list_size()).vortex_expect("fsl list_size fits usize"); - let flat = extract_flat_elements(&const_storage, padded_dim, ctx)?; + let flat = extract_constant_flat_row(&const_storage, ctx)?; if flat.ptype() != PType::F32 { // TODO(connor): case 2 is f32-only. For f16/f64 we fall through to the standard // path, which computes the inner product with the correct element type. @@ -606,14 +552,14 @@ impl InnerProduct { return Ok(Some(empty.into_array())); } - let q: &[f32] = flat.row::(0); + let q: &[f32] = flat.as_slice::(); debug_assert_eq!(q.len(), padded_dim); let codes: &[u8] = codes_prim.as_slice::(); let values: &[f32] = values_prim.as_slice::(); debug_assert_eq!(codes.len(), len * padded_dim); - // The hot loop is extracted into [`execute_dict_constant_inner_product`] with - // unchecked indexing so the compiler can vectorize the inner gather-accumulate. + // The hot loop is extracted into [`execute_dict_constant_inner_product`] so the compiler + // can prove the chunked indices stay in bounds and vectorize the inner gather-accumulate. let out = execute_dict_constant_inner_product(q, values, codes, len, padded_dim); // SAFETY: the buffer length equals `len`, which matches the validity length. @@ -644,10 +590,10 @@ fn inner_product_row(a: &[T], b: &[T]) -> T { /// Compute inner products between a constant query vector and dictionary-encoded rows. /// -/// For each row, computes `sum(q[j] * values[codes[row * dim + j]])` using the codebook -/// `values` directly instead of decoding the dictionary into dense vectors. +/// For each row, computes `sum(q[j] * values[codes[row * dim + j]])` using the codebook `values` +/// directly instead of decoding the dictionary into dense vectors. /// -/// The inner loop uses four independent accumulators so the CPU can pipeline FP additions +/// The inner loop uses `PARTIAL_SUMS` independent accumulators so the CPU can pipeline FP additions /// instead of waiting for each `fadd` to retire before starting the next. fn execute_dict_constant_inner_product( q: &[f32], @@ -704,6 +650,7 @@ mod tests { use crate::scalar_fns::l2_denorm::L2Denorm; use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; + use crate::utils::test_helpers::l2_denorm_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; @@ -818,28 +765,14 @@ mod tests { Ok(()) } - /// Creates an `L2Denorm` scalar function array from pre-normalized elements and norms. - fn l2_denorm_array( - shape: &[usize], - normalized_elements: &[f64], - norms: &[f64], - ) -> VortexResult { - use vortex_array::IntoArray; - - let len = norms.len(); - let normalized = tensor_array(shape, normalized_elements)?; - let norms = PrimitiveArray::from_iter(norms.iter().copied()).into_array(); - let mut ctx = SESSION.create_execution_ctx(); - Ok(L2Denorm::try_new_array(normalized, norms, len, &mut ctx)?.into_array()) - } - #[test] fn both_denorm() -> VortexResult<()> { // LHS: [3.0, 4.0] = L2Denorm([0.6, 0.8], 5.0). // RHS: [1.0, 0.0] = L2Denorm([1.0, 0.0], 1.0). // dot([3.0, 4.0], [1.0, 0.0]) = 3.0. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0])?; - let rhs = l2_denorm_array(&[2], &[1.0, 0.0], &[1.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0], &mut ctx)?; + let rhs = l2_denorm_array(&[2], &[1.0, 0.0], &[1.0], &mut ctx)?; // Expected: 5.0 * 1.0 * dot([0.6, 0.8], [1.0, 0.0]) = 5.0 * 0.6 = 3.0. assert_close(&eval_inner_product(lhs, rhs, 1)?, &[3.0]); @@ -850,8 +783,9 @@ mod tests { fn both_denorm_multiple_rows() -> VortexResult<()> { // Row 0: [3.0, 4.0] dot [3.0, 4.0] = 25.0. // Row 1: [1.0, 0.0] dot [0.0, 1.0] = 0.0. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; - let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 0.0, 1.0], &[5.0, 1.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 0.0, 1.0], &[5.0, 1.0], &mut ctx)?; assert_close(&eval_inner_product(lhs, rhs, 2)?, &[25.0, 0.0]); Ok(()) @@ -862,7 +796,8 @@ mod tests { // LHS: L2Denorm([0.6, 0.8], 5.0) representing [3.0, 4.0]. // RHS: plain [1.0, 2.0]. // dot([3.0, 4.0], [1.0, 2.0]) = 3.0 + 8.0 = 11.0. - let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0])?; + let mut ctx = SESSION.create_execution_ctx(); + let lhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0], &mut ctx)?; let rhs = tensor_array(&[2], &[1.0, 2.0])?; assert_close(&eval_inner_product(lhs, rhs, 1)?, &[11.0]); @@ -874,8 +809,9 @@ mod tests { // LHS: plain [1.0, 2.0]. // RHS: L2Denorm([0.6, 0.8], 5.0) representing [3.0, 4.0]. // dot([1.0, 2.0], [3.0, 4.0]) = 3.0 + 8.0 = 11.0. + let mut ctx = SESSION.create_execution_ctx(); let lhs = tensor_array(&[2], &[1.0, 2.0])?; - let rhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0])?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8], &[5.0], &mut ctx)?; assert_close(&eval_inner_product(lhs, rhs, 1)?, &[11.0]); Ok(()) @@ -889,7 +825,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); let lhs = L2Denorm::try_new_array(normalized_l, norms_l, 2, &mut ctx)?.into_array(); - let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0])?; + let rhs = l2_denorm_array(&[2], &[0.6, 0.8, 1.0, 0.0], &[5.0, 1.0], &mut ctx)?; let scalar_fn = InnerProduct::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 2)?; @@ -948,15 +884,11 @@ mod tests { reason = "tests build small fixtures with deterministic in-range indices" )] mod constant_query_optimizations { - use std::sync::LazyLock; - use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; - use vortex_array::arrays::ConstantArray; - use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::ScalarFnArray; @@ -964,67 +896,23 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::dtype::extension::ExtDType; - use vortex_array::extension::EmptyMetadata; - use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::inner_product::constant_tensor_storage; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; + use crate::tests::SESSION; use crate::utils::extract_flat_elements; + use crate::utils::test_helpers::literal_vector_array; + use crate::utils::test_helpers::vector_array; use crate::vector::Vector; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); - - /// Compact f32 Vector extension over a column-major `elements` slice. - fn vector_f32(dim: u32, elements: &[f32]) -> VortexResult { - let row_count = elements.len() / dim as usize; - let elems: ArrayRef = Buffer::copy_from(elements).into_array(); - let fsl = FixedSizeListArray::new(elems, dim, Validity::NonNullable, row_count); - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) - } - - /// Compact constant-backed f32 Vector extension with a single stored row. - fn constant_vector_f32(elements: &[f32], len: usize) -> VortexResult { - let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - let children: Vec = elements - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = - Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - let storage = ConstantArray::new(storage_scalar, len).into_array(); - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, storage).into_array()) - } - - /// Expression-literal shape: a ConstantArray whose scalar itself is a Vector extension. - fn literal_vector_f32(elements: &[f32], len: usize) -> ArrayRef { - let element_dtype = DType::Primitive(PType::F32, Nullability::NonNullable); - let children: Vec = elements - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = - Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - let vector_scalar = Scalar::extension::(EmptyMetadata, storage_scalar); - ConstantArray::new(vector_scalar, len).into_array() - } - - /// Build an `ExtensionArray>` whose storage is - /// `FSL(DictArray(codes: u8, values: f32))`. This mirrors the shape that - /// TurboQuant produces as the SorfTransform child. + /// Build a `Vector` whose storage is `FSL(DictArray(codes: u8, values: + /// f32))`. This mirrors the shape that TurboQuant produces as the SorfTransform child. fn dict_vector_f32(list_size: u32, codes: &[u8], values: &[f32]) -> VortexResult { let num_rows = codes.len() / list_size as usize; let codes_arr = @@ -1040,9 +928,7 @@ mod tests { Validity::NonNullable, num_rows, )?; - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + Vector::try_new_vector_array(fsl.into_array()) } /// Execute an inner product and return the flat `f32` results. @@ -1128,7 +1014,7 @@ mod tests { #[test] fn constant_tensor_storage_accepts_extension_scalar_literal() -> VortexResult<()> { - let literal = literal_vector_f32(&[1.0, 2.0, 3.0], 5); + let literal = literal_vector_array(&[1.0f32, 2.0, 3.0], 5); let storage = constant_tensor_storage(&literal).expect("literal vector should be recognized"); @@ -1164,7 +1050,7 @@ mod tests { // Query has `dim` elements. let query_elems: Vec = (0..dim).map(|i| (i as f32 * 0.1).sin()).collect(); - let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + let const_rhs = Vector::constant_array(&query_elems, num_rows)?; // Ground truth: decode LHS to plain f32 vectors, dot each with the query. let decoded = decode_sorf_dict( @@ -1202,7 +1088,7 @@ mod tests { build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; let query_elems: Vec = (0..dim).map(|i| (i as f32 * 0.2).cos()).collect(); - let const_lhs = constant_vector_f32(&query_elems, num_rows)?; + let const_lhs = Vector::constant_array(&query_elems, num_rows)?; let decoded = decode_sorf_dict( &codes, @@ -1240,7 +1126,7 @@ mod tests { assert_eq!(padded_dim, dim as usize); let query_elems: Vec = (0..dim).map(|i| i as f32 * 0.01 - 0.5).collect(); - let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + let const_rhs = Vector::constant_array(&query_elems, num_rows)?; let decoded = decode_sorf_dict( &codes, @@ -1277,7 +1163,7 @@ mod tests { build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; let query_elems: Vec = vec![0.0; dim as usize]; - let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + let const_rhs = Vector::constant_array(&query_elems, num_rows)?; let actual = eval_ip_f32(sorf, const_rhs, num_rows)?; assert_eq!(actual.len(), 0); @@ -1300,7 +1186,7 @@ mod tests { let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; let query: Vec = (0..list_size).map(|i| (i as f32 + 1.0) * 0.3).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let expected: Vec = (0..num_rows) .map(|row| { @@ -1330,7 +1216,7 @@ mod tests { let dict_rhs = dict_vector_f32(list_size, &codes, &values)?; let query: Vec = vec![0.5, -1.0, 2.5, -0.25]; - let const_lhs = constant_vector_f32(&query, num_rows)?; + let const_lhs = Vector::constant_array(&query, num_rows)?; let expected: Vec = (0..num_rows) .map(|row| { @@ -1375,12 +1261,10 @@ mod tests { Validity::NonNullable, num_rows, )?; - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - let dict_lhs = ExtensionArray::new(ext_dtype, fsl.into_array()).into_array(); + let dict_lhs = Vector::try_new_vector_array(fsl.into_array())?; let query: Vec = vec![1.0, 2.0, 3.0, 4.0]; - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; // Build expected by decoding by hand. let expected: Vec = (0..num_rows) @@ -1408,10 +1292,10 @@ mod tests { let lhs_elems: Vec = (0..num_rows * dim as usize) .map(|i| i as f32 * 0.25) .collect(); - let plain_lhs = vector_f32(dim, &lhs_elems)?; + let plain_lhs = vector_array(dim, &lhs_elems)?; let query: Vec = vec![1.0, 2.0, 3.0, 4.0]; - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let expected: Vec = (0..num_rows) .map(|row| { @@ -1438,7 +1322,7 @@ mod tests { let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; let query: Vec = vec![0.0; 4]; - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let actual = eval_ip_f32(dict_lhs, const_rhs, num_rows)?; assert_eq!(actual.len(), 0); @@ -1458,7 +1342,7 @@ mod tests { build_sorf_with_dict_child(dim, num_rows, seed, num_rounds)?; let query_elems: Vec = (0..dim).map(|i| ((i as f32) * 0.15).sin() * 0.4).collect(); - let const_rhs = constant_vector_f32(&query_elems, num_rows)?; + let const_rhs = Vector::constant_array(&query_elems, num_rows)?; // Ground truth via full decode + naive dot. let decoded = decode_sorf_dict( @@ -1531,7 +1415,7 @@ mod tests { let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; let query: Vec = (0..list_size).map(|_| rng.next_f32()).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let expected: Vec = (0..num_rows) .map(|row| { @@ -1575,7 +1459,7 @@ mod tests { // has cancellation. let mut rng = XorShift64::new(seed ^ 0xABCD_1234); let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let decoded = decode_sorf_dict( &codes, @@ -1621,7 +1505,7 @@ mod tests { let dict_lhs = dict_vector_f32(list_size, &codes, &values)?; let query: Vec = (0..list_size).map(|_| rng.next_f32()).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let expected: Vec = (0..num_rows) .map(|row| { @@ -1671,7 +1555,7 @@ mod tests { SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array(); let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let decoded = decode_sorf_dict( &codes, @@ -1721,7 +1605,7 @@ mod tests { let mut rng = XorShift64::new(seed ^ (num_rounds as u64)); let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); - let const_rhs = constant_vector_f32(&query, num_rows)?; + let const_rhs = Vector::constant_array(&query, num_rows)?; let decoded = decode_sorf_dict( &codes, @@ -1755,7 +1639,7 @@ mod tests { let mut rng = XorShift64::new(seed); let query: Vec = (0..dim).map(|_| rng.next_f32()).collect(); - let const_lhs = constant_vector_f32(&query, num_rows)?; + let const_lhs = Vector::constant_array(&query, num_rows)?; let decoded = decode_sorf_dict( &codes, diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 673c454b56b..875d7f57e2f 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -22,6 +22,7 @@ use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; @@ -59,6 +60,7 @@ use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_norm::L2Norm; +use crate::utils::extract_constant_flat_row; use crate::utils::extract_flat_elements; use crate::utils::validate_tensor_float_input; @@ -113,7 +115,7 @@ impl L2Denorm { len: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - validate_l2_denorm_children(&normalized, &norms, ctx)?; + validate_l2_normalized_rows_against_norms(&normalized, Some(&norms), ctx)?; // SAFETY: We just validated that it is normalized. unsafe { Self::new_array_unchecked(normalized, norms, len) } @@ -212,7 +214,11 @@ impl ScalarFnVTable for L2Denorm { if let Some(const_norms) = norms_ref.as_opt::() { let norm_scalar = const_norms.scalar(); - vortex_ensure!(norm_scalar.dtype().is_float()); + vortex_ensure!( + norm_scalar.dtype().is_float(), + "L2Denorm constant norms must be a float scalar, got {}", + norm_scalar.dtype(), + ); if let Some(norm_value) = norm_scalar.value() { return execute_l2_denorm_constant_norms( @@ -235,13 +241,11 @@ impl ScalarFnVTable for L2Denorm { .as_extension() .metadata_opt::() .vortex_expect("we already validated this in `return_dtype`"); - let tensor_flat_size = tensor_match.list_size(); + let tensor_flat_size = tensor_match.list_size() as usize; let flat = extract_flat_elements(normalized.storage_array(), tensor_flat_size, ctx)?; - // TODO(connor): Theoretically we could model this as a multiplication between the - // normalized array and a `RunEnd(Sequence(0, dimensions), norms)`. But since we have - // already canonicalized the array, it is probably not faster to do that. + // TODO(connor): Do we want a "broadcast" expression for the List types, or is this fine? match_each_float_ptype!(flat.ptype(), |T| { let norms = norms.as_slice::(); @@ -418,7 +422,7 @@ pub fn normalize_as_l2_denorm( ) -> VortexResult { let row_count = input.len(); let tensor_match = validate_tensor_float_input(input.dtype())?; - let tensor_flat_size = tensor_match.list_size(); + let tensor_flat_size = tensor_match.list_size() as usize; // Constant fast path: if the input is a constant-backed extension, normalize the single // stored row once and return an `L2Denorm` whose children are both `ConstantArray`s. @@ -515,17 +519,17 @@ pub(crate) fn try_build_constant_l2_denorm( .as_extension() .metadata_opt::() .vortex_expect("caller validated input has AnyTensor metadata"); - let list_size = tensor_match.list_size(); + let list_size = tensor_match.list_size() as usize; let original_nullability = input.dtype().nullability(); let ext_dtype = input.dtype().as_extension().clone(); let storage_fsl_nullability = storage.dtype().nullability(); - // `extract_flat_elements` takes the stride-0 single-row path for `Constant` storage, so - // this is cheap and does not expand the constant to the full column length. - let flat = extract_flat_elements(storage, list_size, ctx)?; + // Materialize just the single stored row; this does not expand the constant to the full + // column length. + let flat = extract_constant_flat_row(storage, ctx)?; let (normalized_fsl_scalar, norms_scalar) = match_each_float_ptype!(flat.ptype(), |T| { - let row = flat.row::(0); + let row = flat.as_slice::(); let mut sum_sq = T::zero(); for &x in row { @@ -600,25 +604,14 @@ fn unit_norm_tolerance(element_ptype: PType) -> f64 { } } -/// Validates that every valid row of `input` is already L2-normalized (either length 1 or 0). -pub fn validate_l2_normalized_rows(input: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { - validate_l2_normalized_rows_impl(input, None, ctx) -} - -/// Validates that the `normalized` and `norms` children jointly satisfy the [`L2Denorm`] -/// invariants, which are: +/// Validates that `normalized` and (when supplied) the matching `norms` jointly satisfy the +/// [`L2Denorm`] invariants: /// -/// - All vectors in the normalized array have length 1 or 0. -/// - If the vector has a norm of 0, then the vector must be all 0s. -fn validate_l2_denorm_children( - normalized: &ArrayRef, - norms: &ArrayRef, - ctx: &mut ExecutionCtx, -) -> VortexResult<()> { - validate_l2_normalized_rows_impl(normalized, Some(norms), ctx) -} - -fn validate_l2_normalized_rows_impl( +/// - Every valid row of `normalized` has L2 norm `1.0` or `0.0` (within element-precision +/// tolerance). +/// - When `norms` is supplied, every stored norm is non-negative and any row whose stored norm is +/// `0.0` is exactly the zero vector in `normalized`. +pub fn validate_l2_normalized_rows_against_norms( normalized: &ArrayRef, norms: Option<&ArrayRef>, ctx: &mut ExecutionCtx, @@ -631,7 +624,7 @@ fn validate_l2_normalized_rows_impl( let tensor_match = validate_tensor_float_input(normalized.dtype())?; let element_ptype = tensor_match.element_ptype(); let tolerance = unit_norm_tolerance(element_ptype); - let tensor_flat_size = tensor_match.list_size(); + let tensor_flat_size = tensor_match.list_size() as usize; if let Some(norms) = norms { vortex_ensure_eq!( @@ -697,6 +690,51 @@ fn validate_l2_normalized_rows_impl( Ok(()) } +/// Classification of a binary operand pair by which side (if any) is wrapped in [`L2Denorm`]. +/// +/// Symmetric binary tensor operators (e.g. [`CosineSimilarity`], [`InnerProduct`]) have identical +/// fast paths for "only the lhs is denormalized" and "only the rhs is denormalized", and a separate +/// fast path for "both are denormalized". Rather than hand-rolling the commutative swap at every +/// call site, callers classify their operands with [`Self::classify`] and pattern-match on the +/// returned variant. +/// +/// [`CosineSimilarity`]: crate::scalar_fns::cosine_similarity::CosineSimilarity +/// [`InnerProduct`]: crate::scalar_fns::inner_product::InnerProduct +pub(crate) enum DenormOrientation<'a> { + /// Both operands are [`ExactScalarFn`] arrays. + Both { + lhs: &'a ArrayRef, + rhs: &'a ArrayRef, + }, + /// Exactly one operand is an [`ExactScalarFn`]; the other is plain. + One { + denorm: &'a ArrayRef, + plain: &'a ArrayRef, + }, + /// Neither operand is an [`ExactScalarFn`]. + Neither, +} + +impl<'a> DenormOrientation<'a> { + /// Classify `(lhs, rhs)` by which side (if any) is wrapped in [`L2Denorm`]. + pub(crate) fn classify(lhs: &'a ArrayRef, rhs: &'a ArrayRef) -> Self { + let lhs_denorm = lhs.is::>(); + let rhs_denorm = rhs.is::>(); + match (lhs_denorm, rhs_denorm) { + (true, true) => Self::Both { lhs, rhs }, + (true, false) => Self::One { + denorm: lhs, + plain: rhs, + }, + (false, true) => Self::One { + denorm: rhs, + plain: lhs, + }, + (false, false) => Self::Neither, + } + } +} + #[cfg(test)] mod tests { @@ -719,23 +757,18 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; - use vortex_array::extension::EmptyMetadata; use vortex_array::extension::datetime::Date; use vortex_array::extension::datetime::TimeUnit; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; - use vortex_buffer::Buffer; use vortex_error::VortexResult; - use crate::fixed_shape::FixedShapeTensor; - use crate::fixed_shape::FixedShapeTensorMetadata; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; - use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows; + use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows_against_norms; use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; - use crate::utils::test_helpers::constant_vector_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; use crate::vector::Vector; @@ -747,20 +780,6 @@ mod tests { result.into_array().execute(&mut ctx) } - fn integer_tensor_array(shape: &[usize], elements: &[i32]) -> VortexResult { - let list_size: u32 = shape.iter().product::().max(1).try_into().unwrap(); - let row_count = elements.len() / list_size as usize; - - let elems: ArrayRef = Buffer::copy_from(elements).into_array(); - let fsl = FixedSizeListArray::new(elems, list_size, Validity::NonNullable, row_count); - - let metadata = FixedShapeTensorMetadata::new(shape.to_vec()); - let ext_dtype = - ExtDType::::try_new(metadata, fsl.dtype().clone())?.erased(); - - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) - } - fn non_tensor_extension_array() -> VortexResult { let storage = PrimitiveArray::from_iter([1i32, 2]).into_array(); let ext_dtype = @@ -768,16 +787,6 @@ mod tests { Ok(ExtensionArray::new(ext_dtype, storage).into_array()) } - fn f16_vector_array(dim: u32, elements: &[f32]) -> VortexResult { - let row_count = elements.len() / dim as usize; - let values: Vec<_> = elements.iter().copied().map(half::f16::from_f32).collect(); - let elems: ArrayRef = Buffer::copy_from(values.as_slice()).into_array(); - let fsl = FixedSizeListArray::new(elems, dim, Validity::NonNullable, row_count); - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) - } - fn tensor_snapshot(array: ArrayRef) -> VortexResult<(DType, Vec, Vec)> { let mut ctx = SESSION.create_execution_ctx(); let ext: ExtensionArray = array.execute(&mut ctx)?; @@ -866,7 +875,7 @@ mod tests { #[test] fn l2_denorm_rejects_integer_tensor_lhs() -> VortexResult<()> { - let lhs = integer_tensor_array(&[2], &[1, 2, 3, 4])?; + let lhs = tensor_array(&[2], &[1i32, 2, 3, 4])?; let rhs = PrimitiveArray::from_iter([1.0f64, 1.0]).into_array(); let mut ctx = SESSION.create_execution_ctx(); @@ -888,10 +897,10 @@ mod tests { #[test] fn validate_l2_normalized_rows_accepts_normalized_f16_input() -> VortexResult<()> { - let input = f16_vector_array(2, &[3.0, 4.0, 0.0, 0.0])?; + let input = vector_array(2, &[3.0f32, 4.0, 0.0, 0.0].map(half::f16::from_f32))?; let mut ctx = SESSION.create_execution_ctx(); let roundtrip = normalize_as_l2_denorm(input, &mut ctx)?; - validate_l2_normalized_rows(&roundtrip.child_at(0).clone(), &mut ctx)?; + validate_l2_normalized_rows_against_norms(&roundtrip.child_at(0).clone(), None, &mut ctx)?; Ok(()) } @@ -899,7 +908,7 @@ mod tests { fn validate_l2_normalized_rows_rejects_unnormalized_input() -> VortexResult<()> { let input = vector_array(2, &[3.0, 4.0, 1.0, 0.0])?; let mut ctx = SESSION.create_execution_ctx(); - let result = validate_l2_normalized_rows(&input, &mut ctx); + let result = validate_l2_normalized_rows_against_norms(&input, None, &mut ctx); assert!(result.is_err()); Ok(()) } @@ -982,7 +991,7 @@ mod tests { #[test] fn normalize_as_l2_denorm_supports_constant_vectors() -> VortexResult<()> { - let input = constant_vector_array(&[3.0, 4.0], 2)?; + let input = Vector::constant_array(&[3.0, 4.0], 2)?; let mut ctx = SESSION.create_execution_ctx(); let roundtrip = normalize_as_l2_denorm(input.clone(), &mut ctx)?; let actual = roundtrip.into_array().execute(&mut ctx)?; @@ -996,7 +1005,7 @@ mod tests { // The constant fast path in `normalize_as_l2_denorm` must produce an `L2Denorm` whose // normalized storage and norms child are both still `ConstantArray`s. This is what // allows downstream ops (cosine similarity, inner product) to short-circuit. - let input = constant_vector_array(&[3.0, 4.0], 16)?; + let input = Vector::constant_array(&[3.0, 4.0], 16)?; let mut ctx = SESSION.create_execution_ctx(); let roundtrip = normalize_as_l2_denorm(input, &mut ctx)?; diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 59a2e5a45b1..47ba68a35c4 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -47,6 +47,7 @@ use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::utils::extract_flat_elements; +use crate::utils::extract_l2_denorm_children; use crate::utils::validate_tensor_float_input; /// L2 norm (Euclidean norm) of a tensor or vector column. @@ -84,7 +85,7 @@ impl ScalarFnVTable for L2Norm { type Options = EmptyOptions; fn id(&self) -> ScalarFnId { - ScalarFnId::from("vortex.tensor.l2_norm") + ScalarFnId::new("vortex.tensor.l2_norm") } fn arity(&self, _options: &Self::Options) -> Arity { @@ -131,7 +132,7 @@ impl ScalarFnVTable for L2Norm { let tensor_match = ext .metadata_opt::() .vortex_expect("we already validated this in `return_dtype`"); - let tensor_flat_size = tensor_match.list_size(); + let tensor_flat_size = tensor_match.list_size() as usize; let element_ptype = tensor_match.element_ptype(); let norm_dtype = DType::Primitive(element_ptype, ext.nullability()); @@ -139,13 +140,9 @@ impl ScalarFnVTable for L2Norm { // L2Norm(L2Denorm(normalized, norms)) is defined to read back the authoritative stored // norms. Exact callers of lossy encodings like TurboQuant opt into that storage semantics // instead of forcing a decode-and-recompute path here. - if let Some(sfn) = input_ref.as_opt::>() { - let norms = sfn - .nth_child(1) - .vortex_expect("L2Denom must have at 2 children"); - + if input_ref.is::>() { + let (_, norms) = extract_l2_denorm_children(&input_ref); vortex_ensure_eq!(norms.dtype(), &norm_dtype); - return Ok(norms); } @@ -290,6 +287,7 @@ mod tests { use crate::scalar_fns::l2_norm::L2Norm; use crate::tests::SESSION; use crate::utils::test_helpers::assert_close; + use crate::utils::test_helpers::literal_vector_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; use crate::vector::Vector; @@ -363,27 +361,13 @@ mod tests { Ok(()) } - /// Builds a [`ConstantArray`] whose scalar is a [`Vector`] extension scalar wrapping a - /// fixed-size list of `elements`, broadcast to `len` rows. - fn constant_vector_extension_array(elements: &[f64], len: usize) -> ArrayRef { - let element_dtype = DType::Primitive(PType::F64, Nullability::NonNullable); - let children: Vec = elements - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = - Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - let ext_scalar = Scalar::extension::(EmptyMetadata, storage_scalar); - ConstantArray::new(ext_scalar, len).into_array() - } - /// A constant input whose scalar is a non-null tensor should short-circuit to a /// [`ConstantArray`] output whose scalar is the precomputed norm. Uses [`execute_until`] so /// execution stops at the [`Constant`] encoding instead of canonicalizing into a /// [`PrimitiveArray`]. #[test] fn constant_non_null_input_yields_constant_output() -> VortexResult<()> { - let input = constant_vector_extension_array(&[3.0, 4.0], 4); + let input = literal_vector_array(&[3.0f64, 4.0], 4); let scalar_fn = L2Norm::new().erased(); let result = ScalarFnArray::try_new(scalar_fn, vec![input], 4)?.into_array(); diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs index 64f92da384f..b54158a753f 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -70,13 +70,13 @@ impl ScalarFnVTable for SorfTransform { fn fmt_sql( &self, - _options: &Self::Options, + options: &Self::Options, expr: &Expression, f: &mut Formatter<'_>, ) -> fmt::Result { write!(f, "sorf_transform(")?; expr.child(0).fmt_sql(f)?; - write!(f, ")") + write!(f, ", {options})") } fn return_dtype(&self, options: &Self::Options, arg_dtypes: &[DType]) -> VortexResult { @@ -143,9 +143,7 @@ impl ScalarFnVTable for SorfTransform { validity, 0, )?; - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + Vector::try_new_vector_array(fsl.into_array()) }); } @@ -220,15 +218,8 @@ impl ScalarFnArrayVTable for SorfTransform { view: &ScalarFnArrayView, _session: &VortexSession, ) -> VortexResult>> { - let options = view.options; Ok(Some( - SorfTransformMetadata { - seed: options.seed, - num_rounds: u32::from(options.num_rounds), - dimension: options.dimension, - element_ptype: options.element_ptype as i32, - } - .encode_to_vec(), + SorfTransformMetadata::from(view.options).encode_to_vec(), )) } @@ -240,20 +231,9 @@ impl ScalarFnArrayVTable for SorfTransform { children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult> { - let metadata = SorfTransformMetadata::decode(metadata) - .map_err(|e| vortex_err!("Failed to decode SorfTransformMetadata: {e}"))?; - let options = SorfOptions { - seed: metadata.seed, - num_rounds: u8::try_from(metadata.num_rounds).map_err(|_| { - vortex_err!( - "SorfTransform num_rounds {} does not fit in u8", - metadata.num_rounds - ) - })?, - dimension: metadata.dimension, - element_ptype: metadata.element_ptype(), - }; - validate_sorf_options(&options)?; + let options = SorfTransformMetadata::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode SorfTransformMetadata: {e}"))? + .to_options()?; // `return_dtype` sets the output FSL's nullability to the child's nullability (see // `return_dtype` above), so we read the child nullability back from the parent dtype. @@ -316,7 +296,37 @@ fn inverse_rotate_typed( let elements = PrimitiveArray::new::(output.freeze(), Validity::NonNullable); let fsl = FixedSizeListArray::try_new(elements.into_array(), dim_u32, validity, num_rows)?; + Vector::try_new_vector_array(fsl.into_array()) +} - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) +impl From<&SorfOptions> for SorfTransformMetadata { + fn from(options: &SorfOptions) -> Self { + Self { + seed: options.seed, + num_rounds: u32::from(options.num_rounds), + dimension: options.dimension, + element_ptype: options.element_ptype as i32, + } + } +} + +impl SorfTransformMetadata { + /// Rebuild the [`SorfOptions`] this metadata was serialized from, validating that the wire + /// values are in range. + fn to_options(&self) -> VortexResult { + let num_rounds = u8::try_from(self.num_rounds).map_err(|_| { + vortex_err!( + "SorfTransform num_rounds {} does not fit in u8", + self.num_rounds + ) + })?; + let options = SorfOptions { + seed: self.seed, + num_rounds, + dimension: self.dimension, + element_ptype: self.element_ptype(), + }; + validate_sorf_options(&options)?; + Ok(options) + } } diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 76d65ce9eef..3434058f082 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -9,6 +9,7 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -43,6 +44,23 @@ pub fn validate_tensor_float_input(input_dtype: &DType) -> VortexResult( + op_name: &str, + lhs: &'a DType, + rhs: &DType, +) -> VortexResult> { + vortex_ensure!( + lhs.eq_ignore_nullability(rhs), + "{op_name} requires both inputs to have the same dtype, got {lhs} and {rhs}" + ); + validate_tensor_float_input(lhs) +} + /// Cast a float [`PrimitiveArray`] to a `Buffer`. /// /// Several operations in this crate (SORF transform, TurboQuant quantization) work exclusively @@ -80,12 +98,12 @@ pub fn cast_to_f32(prim: PrimitiveArray) -> VortexResult> { /// The flat primitive elements of a tensor storage array, with typed row access. /// /// This struct hides the stride detail that arises from the [`ConstantArray`] optimization: a -/// constant input materializes only a single row (stride=0), while a full array uses -/// stride=list_size. +/// constant-backed input materializes only a single row that every index reads (`is_constant = +/// true`), while a full array stores one row per index. pub struct FlatElements { elems: PrimitiveArray, - stride: usize, list_size: usize, + is_constant: bool, } impl FlatElements { @@ -96,47 +114,101 @@ impl FlatElements { } /// Returns the `i`-th row as a typed slice of length `list_size`. + /// + /// When the source was a constant-backed storage, all indices resolve to the single stored + /// row. #[must_use] pub fn row(&self, i: usize) -> &[T] { + let row_idx = if self.is_constant { 0 } else { i }; let slice = self.elems.as_slice::(); - &slice[i * self.stride..][..self.list_size] + &slice[row_idx * self.list_size..][..self.list_size] } } -// TODO(connor): Usage of this function is sometimes incorrect / not performant. -// Make sure to fix them. /// Extracts the flat primitive elements from a tensor storage array (FixedSizeList). /// /// When the input is a [`ConstantArray`] (e.g., a literal query vector), only a single row is -/// materialized to avoid expanding it to the full column length. +/// materialized to avoid expanding it to the full column length. Callers that have already +/// confirmed the storage is constant-backed should prefer [`extract_constant_flat_row`]. pub fn extract_flat_elements( storage: &ArrayRef, list_size: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(constant) = storage.as_opt::() { - // Rewrite the array as a length 1 array so when we canonicalize, we do not duplicate a huge - // amount of data. + // Constant-backed storage: materialize just the single stored row so canonicalization does + // not expand the array to the full column length. + let (source, is_constant) = if let Some(constant) = storage.as_opt::() { let single = ConstantArray::new(constant.scalar().clone(), 1).into_array(); - let fsl: FixedSizeListArray = single.execute(ctx)?; - let elems: PrimitiveArray = fsl.elements().clone().execute(ctx)?; - return Ok(FlatElements { - elems, - stride: 0, - list_size, - }); - } + (single, true) + } else { + (storage.clone(), false) + }; - // Otherwise we have to fully expand all of the data. - let fsl: FixedSizeListArray = storage.clone().execute(ctx)?; + let fsl: FixedSizeListArray = source.execute(ctx)?; let elems: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + vortex_ensure!( + !elems.nullability().is_nullable(), + "tensor storage elements must be non-nullable, got {}", + elems.dtype(), + ); Ok(FlatElements { elems, - stride: list_size, list_size, + is_constant, }) } +/// The single stored row of a constant-backed tensor storage array. +/// +/// Contrast with [`FlatElements`], which exposes arbitrary row indices: a `FlatRow` statically +/// encodes "there is exactly one row available," so call sites that have gated on a constant input +/// read the row via [`Self::as_slice`] instead of `row(0)`. +pub struct FlatRow { + elems: PrimitiveArray, +} + +impl FlatRow { + /// Returns the [`PType`] of the underlying elements. + #[must_use] + pub fn ptype(&self) -> PType { + self.elems.ptype() + } + + /// Returns the stored row as a typed slice. Its length equals the storage scalar's + /// fixed-size-list size. + #[must_use] + pub fn as_slice(&self) -> &[T] { + self.elems.as_slice::() + } +} + +/// Extracts the single stored row from a [`Constant`]-backed tensor storage array. +/// +/// The caller must have confirmed that `storage` is a [`Constant`] encoding whose scalar is a +/// non-null fixed-size list. This is the fast path for constant query vectors: exactly one row is +/// materialized regardless of the column length. +/// +/// # Panics +/// +/// Panics if `storage` is not a [`Constant`] encoding. +pub fn extract_constant_flat_row( + storage: &ArrayRef, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let constant = storage + .as_opt::() + .vortex_expect("extract_constant_flat_row requires Constant-backed storage"); + let single = ConstantArray::new(constant.scalar().clone(), 1).into_array(); + let fsl: FixedSizeListArray = single.execute(ctx)?; + let elems: PrimitiveArray = fsl.elements().clone().execute(ctx)?; + vortex_ensure!( + !elems.nullability().is_nullable(), + "tensor storage elements must be non-nullable, got {}", + elems.dtype(), + ); + Ok(FlatRow { elems }) +} + /// Extracts the `(normalized, norms)` children from an [`L2Denorm`] scalar function array. /// /// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm @@ -154,15 +226,17 @@ pub fn extract_l2_denorm_children(array: &ArrayRef) -> (ArrayRef, ArrayRef) { #[cfg(test)] pub mod test_helpers { use vortex_array::ArrayRef; + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; + use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; + use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; - use vortex_array::dtype::PType; use vortex_array::dtype::extension::ExtDType; - use vortex_array::extension::EmptyMetadata; + use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::validity::Validity; use vortex_buffer::Buffer; @@ -170,81 +244,86 @@ pub mod test_helpers { use crate::fixed_shape::FixedShapeTensor; use crate::fixed_shape::FixedShapeTensorMetadata; + use crate::scalar_fns::l2_denorm::L2Denorm; use crate::vector::Vector; - /// Builds a [`FixedShapeTensor`] extension array from flat f64 elements and a logical shape. + /// Builds a `FixedSizeList` storage array from flat `elements`. The row count is + /// inferred from `elements.len() / list_size`. + fn flat_fsl(elements: &[T], list_size: u32) -> ArrayRef { + let row_count = elements.len() / list_size as usize; + let elems: ArrayRef = Buffer::copy_from(elements).into_array(); + FixedSizeListArray::new(elems, list_size, Validity::NonNullable, row_count).into_array() + } + + /// Builds an FSL-valued [`Scalar`] from `elements` for use as a constant query. + fn fsl_scalar>(elements: &[T]) -> Scalar { + let element_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable); + let children: Vec = elements + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable) + } + + /// Builds a [`FixedShapeTensor`] extension array from flat `elements` and a logical shape. /// /// The number of rows is inferred from the total element count divided by the product of the /// shape dimensions. For 0-dimensional tensors (scalar), each element is one row. - pub fn tensor_array(shape: &[usize], elements: &[f64]) -> VortexResult { + pub fn tensor_array(shape: &[usize], elements: &[T]) -> VortexResult { let list_size: u32 = shape.iter().product::().max(1).try_into().unwrap(); - let row_count = elements.len() / list_size as usize; - - let elems: ArrayRef = Buffer::copy_from(elements).into_array(); - let fsl = FixedSizeListArray::new(elems, list_size, Validity::NonNullable, row_count); - + let storage = flat_fsl(elements, list_size); let metadata = FixedShapeTensorMetadata::new(shape.to_vec()); let ext_dtype = - ExtDType::::try_new(metadata, fsl.dtype().clone())?.erased(); - - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + ExtDType::::try_new(metadata, storage.dtype().clone())?.erased(); + Ok(ExtensionArray::new(ext_dtype, storage).into_array()) } - /// Builds a [`Vector`] extension array from flat f64 elements and a vector dimension size. - pub fn vector_array(dim: u32, elements: &[f64]) -> VortexResult { - let row_count = elements.len() / dim as usize; - - let elems: ArrayRef = Buffer::copy_from(elements).into_array(); - let fsl = FixedSizeListArray::new(elems, dim, Validity::NonNullable, row_count); - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) + /// Builds a [`Vector`] extension array from flat `elements` and a vector dimension size. + pub fn vector_array(dim: u32, elements: &[T]) -> VortexResult { + Vector::try_new_vector_array(flat_fsl(elements, dim)) } /// Builds a [`FixedShapeTensor`] extension array whose storage is a [`ConstantArray`], /// representing a single query tensor broadcast to `len` rows. - pub fn constant_tensor_array( + pub fn constant_tensor_array>( shape: &[usize], - elements: &[f64], + elements: &[T], len: usize, ) -> VortexResult { - let element_dtype = DType::Primitive(PType::F64, Nullability::NonNullable); - - let children: Vec = elements - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = - Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - - let storage = ConstantArray::new(storage_scalar, len).into_array(); - + let storage = ConstantArray::new(fsl_scalar(elements), len).into_array(); let metadata = FixedShapeTensorMetadata::new(shape.to_vec()); let ext_dtype = ExtDType::::try_new(metadata, storage.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, storage).into_array()) } - /// Builds a [`Vector`] extension array whose storage is a [`ConstantArray`], representing a - /// single query vector broadcast to `len` rows. - pub fn constant_vector_array(elements: &[f64], len: usize) -> VortexResult { - let element_dtype = DType::Primitive(PType::F64, Nullability::NonNullable); - - let children: Vec = elements - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = - Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - - let storage = ConstantArray::new(storage_scalar, len).into_array(); - - let ext_dtype = - ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); + /// Builds a [`ConstantArray`] whose scalar is itself a [`Vector`] extension scalar, broadcast + /// to `len` rows. This is the shape produced by an `lit(vector_scalar)` literal expression — + /// the constant lives at the extension level rather than inside the FSL storage, in contrast + /// to [`Vector::constant_array`]. + pub fn literal_vector_array>( + elements: &[T], + len: usize, + ) -> ArrayRef { + use vortex_array::extension::EmptyMetadata; + let ext_scalar = Scalar::extension::(EmptyMetadata, fsl_scalar(elements)); + ConstantArray::new(ext_scalar, len).into_array() + } - Ok(ExtensionArray::new(ext_dtype, storage).into_array()) + /// Creates an [`L2Denorm`] scalar function array from pre-normalized tensor elements and + /// matching norms. The caller must ensure every row of `normalized_elements` is unit-norm or + /// zero. + pub fn l2_denorm_array( + shape: &[usize], + normalized_elements: &[T], + norms: &[T], + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let len = norms.len(); + let normalized = tensor_array(shape, normalized_elements)?; + let norms = + PrimitiveArray::new(Buffer::copy_from(norms), Validity::NonNullable).into_array(); + Ok(L2Denorm::try_new_array(normalized, norms, len, ctx)?.into_array()) } /// Asserts that each element in `actual` is within `1e-10` of the corresponding `expected` diff --git a/vortex-tensor/src/vector/matcher.rs b/vortex-tensor/src/vector/matcher.rs index 0da5384f303..e4838b77426 100644 --- a/vortex-tensor/src/vector/matcher.rs +++ b/vortex-tensor/src/vector/matcher.rs @@ -49,7 +49,7 @@ impl Matcher for AnyVector { let dimensions = *list_size; - assert!(element_dtype.is_float(), "element dtype must be primitive"); + assert!(element_dtype.is_float(), "element dtype must be float"); assert!( !element_dtype.is_nullable(), "element dtype must be non-nullable" diff --git a/vortex-tensor/src/vector/mod.rs b/vortex-tensor/src/vector/mod.rs index 3c6a8a8c8cc..d077a183713 100644 --- a/vortex-tensor/src/vector/mod.rs +++ b/vortex-tensor/src/vector/mod.rs @@ -3,10 +3,54 @@ //! Vector extension type for fixed-length float vectors (e.g., embeddings). +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::dtype::DType; +use vortex_array::dtype::NativePType; +use vortex_array::dtype::Nullability; +use vortex_array::extension::EmptyMetadata; +use vortex_array::scalar::PValue; +use vortex_array::scalar::Scalar; +use vortex_error::VortexResult; + /// The Vector extension type. #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] pub struct Vector; +impl Vector { + /// Helper function for creating a new [`Vector`] [`ExtensionArray`]. + /// + /// # Errors + /// + /// Returns an error if the [`Vector`] extension dtype rejects the storage array. + pub(crate) fn try_new_vector_array(storage: ArrayRef) -> VortexResult { + ExtensionArray::try_new_from_vtable(Vector, EmptyMetadata, storage) + .map(|ext| ext.into_array()) + } + + /// Helper function to build a [`Vector`] [`ExtensionArray`] whose storage is a + /// [`ConstantArray`], broadcasting a single vector `elements` across `len` rows. + /// + /// # Errors + /// + /// Returns an error if the [`Vector`] extension dtype rejects the constructed storage dtype. + pub(crate) fn constant_array>( + elements: &[T], + len: usize, + ) -> VortexResult { + let element_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable); + let children: Vec = elements + .iter() + .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) + .collect(); + let storage_scalar = + Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); + Self::try_new_vector_array(ConstantArray::new(storage_scalar, len).into_array()) + } +} + mod matcher; pub use matcher::AnyVector; diff --git a/vortex-tensor/src/vector_search.rs b/vortex-tensor/src/vector_search.rs index 81a379683db..c5551c99f2d 100644 --- a/vortex-tensor/src/vector_search.rs +++ b/vortex-tensor/src/vector_search.rs @@ -4,23 +4,14 @@ //! Reusable helpers for building brute-force vector similarity search expressions over //! [`Vector`] extension arrays. //! -//! This module exposes three small building blocks that together make it straightforward to -//! stand up a cosine-similarity-plus-threshold scan on top of a prepared data array: +//! [`build_similarity_search_tree`] broadcasts the query into the shape expected by +//! [`CosineSimilarity`] via `Vector::constant_array` and returns a lazy +//! `Binary(Gt, [CosineSimilarity(data, query), threshold])` expression. The caller is responsible +//! for preparing `data` (e.g. by running it through [`turboquant_encode`]); this builder does not +//! compress. //! -//! - [`compress_turboquant`] applies the canonical TurboQuant encoding pipeline -//! (`L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms)`) to a raw -//! `Vector` array without requiring the caller to plumb the -//! `unstable_encodings` feature flag on the `vortex` facade. -//! - [`build_constant_query_vector`] wraps a single query vector into a -//! [`Vector`] extension array whose storage is a [`ConstantArray`] broadcast -//! across `num_rows` rows. This is the shape expected by -//! [`CosineSimilarity::try_new_array`] for the RHS of a database-vs-query scan. -//! - [`build_similarity_search_tree`] wires everything together into a lazy -//! `Binary(Gt, [CosineSimilarity(data, query), threshold])` expression. -//! -//! Executing the tree from [`build_similarity_search_tree`] into a -//! [`BoolArray`](vortex_array::arrays::BoolArray) yields one boolean per row indicating whether -//! that row's cosine similarity to the query exceeds `threshold`. +//! Executing the tree into a [`BoolArray`] yields one boolean per row indicating whether that row's +//! cosine similarity to the query exceeds `threshold`. //! //! # Example //! @@ -28,11 +19,12 @@ //! use vortex_array::{ArrayRef, VortexSessionExecute}; //! use vortex_array::arrays::BoolArray; //! use vortex_session::VortexSession; -//! use vortex_tensor::vector_search::{build_similarity_search_tree, compress_turboquant}; +//! use vortex_tensor::encodings::turboquant::{TurboQuantConfig, turboquant_encode}; +//! use vortex_tensor::vector_search::build_similarity_search_tree; //! //! fn run(session: &VortexSession, data: ArrayRef, query: &[f32]) -> anyhow::Result<()> { //! let mut ctx = session.create_execution_ctx(); -//! let data = compress_turboquant(data, &mut ctx)?; +//! let data = turboquant_encode(data, &TurboQuantConfig::default(), &mut ctx)?; //! let tree = build_similarity_search_tree(data, query, 0.8)?; //! let _matches: BoolArray = tree.execute(&mut ctx)?; //! Ok(()) @@ -40,98 +32,24 @@ //! ``` //! //! [`Vector`]: crate::vector::Vector -//! [`CosineSimilarity::try_new_array`]: crate::scalar_fns::cosine_similarity::CosineSimilarity::try_new_array +//! [`CosineSimilarity`]: crate::scalar_fns::cosine_similarity::CosineSimilarity +//! [`turboquant_encode`]: crate::encodings::turboquant::turboquant_encode +//! [`BoolArray`]: vortex_array::arrays::BoolArray use vortex_array::ArrayRef; -use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::ConstantArray; -use vortex_array::arrays::Extension; -use vortex_array::arrays::ExtensionArray; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::builtins::ArrayBuiltins; -use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::extension::ExtDType; -use vortex_array::extension::EmptyMetadata; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use crate::encodings::turboquant::TurboQuantConfig; -use crate::encodings::turboquant::turboquant_encode_unchecked; use crate::scalar_fns::cosine_similarity::CosineSimilarity; -use crate::scalar_fns::l2_denorm::L2Denorm; -use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::vector::Vector; -/// Apply the canonical TurboQuant encoding pipeline to a `Vector` array. -/// -/// The returned array has the shape -/// `L2Denorm(SorfTransform(FSL(Dict(codes, centroids))), norms)` — exactly what -/// [`crate::encodings::turboquant::TurboQuantScheme`] produces when invoked through -/// `BtrBlocksCompressorBuilder::with_turboquant()`, but without requiring callers to enable -/// the `unstable_encodings` feature on the `vortex` facade. -/// -/// The input `data` must be a [`Vector`] extension array whose element type is `f32` and whose -/// dimensionality is at least -/// [`turboquant::MIN_DIMENSION`](crate::encodings::turboquant::MIN_DIMENSION). The TurboQuant -/// configuration used is [`TurboQuantConfig::default()`] (8-bit codes, 3 SORF rounds, seed 42). -/// -/// # Errors -/// -/// Returns an error if `data` is not a [`Vector`] extension array, if normalization fails, or -/// if the underlying TurboQuant encoder rejects the input shape. -pub fn compress_turboquant(data: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - let l2_denorm = normalize_as_l2_denorm(data, ctx)?; - let normalized = l2_denorm.child_at(0).clone(); - let norms = l2_denorm.child_at(1).clone(); - let num_rows = l2_denorm.len(); - - let Some(normalized_ext) = normalized.as_opt::() else { - vortex_bail!("normalize_as_l2_denorm must produce an Extension array child"); - }; - - let config = TurboQuantConfig::default(); - // SAFETY: `normalize_as_l2_denorm` guarantees every row is unit-norm (or zero), which is - // the invariant `turboquant_encode_unchecked` expects. - let tq = unsafe { turboquant_encode_unchecked(normalized_ext, &config, ctx) }?; - - Ok(unsafe { L2Denorm::new_array_unchecked(tq, norms, num_rows) }?.into_array()) -} - -/// Build a [`Vector`] extension array whose storage is a [`ConstantArray`] broadcasting a single -/// query vector across `num_rows` rows. -/// -/// The element type is inferred from `T` (e.g. `f32` or `f64`). This is the shape expected for -/// the RHS of a database-vs-query [`CosineSimilarity`] scan: the `ScalarFnArray` contract -/// requires both children to have the same length, so rather than hand-rolling a 1-row input we -/// broadcast the query across the whole database. -/// -/// # Errors -/// -/// Returns an error if the [`Vector`] extension dtype rejects the constructed storage dtype. -pub fn build_constant_query_vector>( - query: &[T], - num_rows: usize, -) -> VortexResult { - let element_dtype = DType::Primitive(T::PTYPE, Nullability::NonNullable); - - let children: Vec = query - .iter() - .map(|&v| Scalar::primitive(v, Nullability::NonNullable)) - .collect(); - let storage_scalar = Scalar::fixed_size_list(element_dtype, children, Nullability::NonNullable); - - let storage = ConstantArray::new(storage_scalar, num_rows).into_array(); - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, storage).into_array()) -} - /// Build the lazy similarity-search expression tree for a prepared database array and a /// single query vector. /// @@ -163,7 +81,7 @@ pub fn build_similarity_search_tree>( threshold: T, ) -> VortexResult { let num_rows = data.len(); - let query_vec = build_constant_query_vector(query, num_rows)?; + let query_vec = Vector::constant_array(query, num_rows)?; let cosine = CosineSimilarity::try_new_array(data, query_vec, num_rows)?.into_array(); @@ -175,64 +93,16 @@ pub fn build_similarity_search_tree>( #[cfg(test)] mod tests { - use vortex_array::ArrayRef; - use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; - use vortex_array::arrays::Extension; - use vortex_array::arrays::ExtensionArray; - use vortex_array::arrays::FixedSizeListArray; - use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; - use vortex_array::dtype::extension::ExtDType; - use vortex_array::extension::EmptyMetadata; - use vortex_array::session::ArraySession; - use vortex_array::validity::Validity; - use vortex_buffer::BufferMut; use vortex_error::VortexResult; - use vortex_session::VortexSession; - use super::build_constant_query_vector; use super::build_similarity_search_tree; - use super::compress_turboquant; - use crate::vector::Vector; - - /// Build a `Vector` extension array from a flat f32 slice. Each contiguous - /// group of `DIM` values becomes one row. - fn vector_array(dim: u32, values: &[f32]) -> VortexResult { - let dim_usize = dim as usize; - assert_eq!(values.len() % dim_usize, 0); - let num_rows = values.len() / dim_usize; - - let mut buf = BufferMut::::with_capacity(values.len()); - for &v in values { - buf.push(v); - } - let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); - let fsl = FixedSizeListArray::try_new( - elements.into_array(), - dim, - Validity::NonNullable, - num_rows, - )?; - - let ext_dtype = ExtDType::::try_new(EmptyMetadata, fsl.dtype().clone())?.erased(); - Ok(ExtensionArray::new(ext_dtype, fsl.into_array()).into_array()) - } - - fn test_session() -> VortexSession { - VortexSession::empty().with::() - } - - #[test] - fn constant_query_vector_has_vector_extension_dtype() -> VortexResult<()> { - let query = vec![1.0f32, 0.0, 0.0, 0.0]; - let rhs = build_constant_query_vector(&query, 5)?; - - assert_eq!(rhs.len(), 5); - assert!(rhs.as_opt::().is_some()); - Ok(()) - } + use crate::encodings::turboquant::TurboQuantConfig; + use crate::encodings::turboquant::turboquant_encode; + use crate::tests::SESSION; + use crate::utils::test_helpers::vector_array; #[test] fn similarity_search_tree_executes_to_bool_array() -> VortexResult<()> { @@ -240,7 +110,7 @@ mod tests { let data = vector_array( 3, &[ - 1.0, 0.0, 0.0, // + 1.0f32, 0.0, 0.0, // 0.0, 1.0, 0.0, // 0.0, 0.0, 1.0, // 1.0, 0.0, 0.0, // @@ -249,7 +119,7 @@ mod tests { let query = [1.0f32, 0.0, 0.0]; let tree = build_similarity_search_tree(data, &query, 0.5)?; - let mut ctx = test_session().create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let result: BoolArray = tree.execute(&mut ctx)?; let bits = result.to_bit_buffer(); @@ -287,8 +157,8 @@ mod tests { } let data = vector_array(DIM, &values)?; - let mut ctx = test_session().create_execution_ctx(); - let compressed = compress_turboquant(data, &mut ctx)?; + let mut ctx = SESSION.create_execution_ctx(); + let compressed = turboquant_encode(data, &TurboQuantConfig::default(), &mut ctx)?; assert_eq!(compressed.len(), NUM_ROWS); // Build a tree with a low threshold so row 0 (cosine=1.0 exact) matches. diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 4e2ef9b1beb..e74301cea9f 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -531,7 +531,7 @@ mod turboquant_benches { macro_rules! turboquant_bench { (compress, $dim:literal, $bits:literal, $name:ident) => { paste! { - #[divan::bench(name = concat!("turboquant_compress_dim", stringify!($dim), "_", stringify!($bits), "bit"))] + #[divan::bench(name = concat!("turboquant_encode_dim", stringify!($dim), "_", stringify!($bits), "bit"))] fn $name(bencher: Bencher) { let normalized_ext = setup_normalized_vector_ext($dim); let config = turboquant_config($bits); From e8d64af112fd632f314f1f53ac8253233cdd9aa6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:34:12 +0000 Subject: [PATCH 137/250] Update Rust crate reqwest to 0.13.0 (#7552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [reqwest](https://redirect.github.com/seanmonstar/reqwest) | workspace.dependencies | minor | `0.12.4` → `0.13.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
seanmonstar/reqwest (reqwest) ### [`v0.13.2`](https://redirect.github.com/seanmonstar/reqwest/blob/HEAD/CHANGELOG.md#v0132) [Compare Source](https://redirect.github.com/seanmonstar/reqwest/compare/v0.13.1...v0.13.2) - Fix HTTP/2 and native-tls ALPN feature combinations. - Fix HTTP/3 to send h3 ALPN. - (wasm) fix `RequestBuilder::json()` from override previously set content-type. ### [`v0.13.1`](https://redirect.github.com/seanmonstar/reqwest/blob/HEAD/CHANGELOG.md#v0131) [Compare Source](https://redirect.github.com/seanmonstar/reqwest/compare/v0.13.0...v0.13.1) - Fixes compiling with rustls on Android targets. ### [`v0.13.0`](https://redirect.github.com/seanmonstar/reqwest/blob/HEAD/CHANGELOG.md#v0130) [Compare Source](https://redirect.github.com/seanmonstar/reqwest/compare/v0.12.28...v0.13.0) - **Breaking changes**: - `rustls` is now the default TLS backend, instead of `native-tls`. - `rustls` crypto provider defaults to aws-lc instead of *ring*. (`rustls-no-provider` exists if you want a different crypto provider) - `rustls-tls` has been renamed to `rustls`. - rustls roots features removed, `rustls-platform-verifier` is used by default. - To use different roots, call `tls_certs_only(your_roots)`. - `native-tls` now includes ALPN. To disable, use `native-tls-no-alpn`. - `query` and `form` are now crate features, disabled by default. - Long-deprecated methods and crate features have been removed (such as `trust-dns`, which was renamed `hickory-dns` a while ago). - Many TLS-related methods renamed to improve autocompletion and discovery, but previous name left in place with a "soft" deprecation. (just documented, no warnings) - For example, prefer `tls_backend_rustls()` over `use_rustls_tls()`. #### v0.12.28 - Fix compiling on Windows if TLS and SOCKS features are not enabled. #### v0.12.27 - Add `ClientBuilder::windows_named_pipe(name)` option that will force all requests over that Windows Named Piper. #### v0.12.26 - Fix sending `Accept-Encoding` header only with values configured with reqwest, regardless of underlying tower-http config. #### v0.12.25 - Add `Error::is_upgrade()` to determine if the error was from an HTTP upgrade. - Fix sending `Proxy-Authorization` if only username is configured. - Fix sending `Proxy-Authorization` to HTTPS proxies when the target is HTTP. - Refactor internal decompression handling to use tower-http. #### v0.12.24 - Refactor cookie handling to an internal middleware. - Refactor internal random generator. - Refactor base64 encoding to reduce a copy. - Documentation updates. #### v0.12.23 - Add `ClientBuilder::unix_socket(path)` option that will force all requests over that Unix Domain Socket. - Add `ClientBuilder::retry(policy)` and `reqwest::retry::Builder` to configure automatic retries. - Add `ClientBuilder::dns_resolver2()` with more ergonomic argument bounds, allowing more resolver implementations. - Add `http3_*` options to `blocking::ClientBuilder`. - Fix default TCP timeout values to enabled and faster. - Fix SOCKS proxies to default to port 1080 - (wasm) Add cache methods to `RequestBuilder`. #### v0.12.22 - Fix socks proxies when resolving IPv6 destinations. #### v0.12.21 - Fix socks proxy to use `socks4a://` instead of `socks4h://`. - Fix `Error::is_timeout()` to check for hyper and IO timeouts too. - Fix request `Error` to again include URLs when possible. - Fix socks connect error to include more context. - (wasm) implement `Default` for `Body`. #### v0.12.20 - Add `ClientBuilder::tcp_user_timeout(Duration)` option to set `TCP_USER_TIMEOUT`. - Fix proxy headers only using the first matched proxy. - (wasm) Fix re-adding `Error::is_status()`. #### v0.12.19 - Fix redirect that changes the method to GET should remove payload headers. - Fix redirect to only check the next scheme if the policy action is to follow. - (wasm) Fix compilation error if `cookies` feature is enabled (by the way, it's a noop feature in wasm). #### v0.12.18 - Fix compilation when `socks` enabled without TLS. #### v0.12.17 - Fix compilation on macOS. #### v0.12.16 - Add `ClientBuilder::http3_congestion_bbr()` to enable BBR congestion control. - Add `ClientBuilder::http3_send_grease()` to configure whether to send use QUIC grease. - Add `ClientBuilder::http3_max_field_section_size()` to configure the maximum response headers. - Add `ClientBuilder::tcp_keepalive_interval()` to configure TCP probe interval. - Add `ClientBuilder::tcp_keepalive_retries()` to configure TCP probe count. - Add `Proxy::headers()` to add extra headers that should be sent to a proxy. - Fix `redirect::Policy::limit()` which had an off-by-1 error, allowing 1 more redirect than specified. - Fix HTTP/3 to support streaming request bodies. - (wasm) Fix null bodies when calling `Response::bytes_stream()`. #### v0.12.15 - Fix Windows to support both `ProxyOverride` and `NO_PROXY`. - Fix http3 to support streaming response bodies. - Fix http3 dependency from public API misuse. #### v0.12.14 - Fix missing `fetch_mode_no_cors()`, marking as deprecated when not on WASM. #### v0.12.13 - Add `Form::into_reader()` for blocking `multipart` forms. - Add `Form::into_stream()` for async `multipart` forms. - Add support for SOCKS4a proxies. - Fix decoding responses with multiple zstd frames. - Fix `RequestBuilder::form()` from overwriting a previously set `Content-Type` header, like the other builder methods. - Fix cloning of request timeout in `blocking::Request`. - Fix http3 synchronization of connection creation, reducing unneccesary extra connections. - Fix Windows system proxy to use `ProxyOverride` as a `NO_PROXY` value. - Fix blocking read to correctly reserve and zero read buffer. - (wasm) Add support for request timeouts. - (wasm) Fix `Error::is_timeout()` to return true when from a request timeout. #### v0.12.12 - (wasm) Fix compilation by not compiler `tokio/time` on WASM. #### v0.12.11 - Fix decompression returning an error when HTTP/2 ends with an empty data frame. #### v0.12.10 - Add `ClientBuilder::connector_layer()` to allow customizing the connector stack. - Add `ClientBuilder::http2_max_header_list_size()` option. - Fix propagating body size hint (`content-length`) information when wrapping bodies. - Fix decompression of chunked bodies so the connections can be reused more often. #### v0.12.9 - Add `tls::CertificateRevocationLists` support. - Add crate features to enable webpki roots without selecting a rustls provider. - Fix `connection_verbose()` to output read logs. - Fix `multipart::Part::file()` to automatically include content-length. - Fix proxy to internally no longer cache system proxy settings. #### v0.12.8 - Add support for SOCKS4 proxies. - Add `multipart::Form::file()` method for adding files easily. - Add `Body::wrap()` to wrap any `http_body::Body` type. - Fix the pool configuration to use a timer to remove expired connections. #### v0.12.7 - Revert adding `impl Service>` for `Client`. #### v0.12.6 - Add support for `danger_accept_invalid_hostnames` for `rustls`. - Add `impl Service>` for `Client` and `&'_ Client`. - Add support for `!Sync` bodies in `Body::wrap_stream()`. - Enable happy eyeballs when `hickory-dns` is used. - Fix `Proxy` so that `HTTP(S)_PROXY` values take precedence over `ALL_PROXY`. - Fix `blocking::RequestBuilder::header()` from unsetting `sensitive` on passed header values. #### v0.12.5 - Add `blocking::ClientBuilder::dns_resolver()` method to change DNS resolver in blocking client. - Add `http3` feature back, still requiring `reqwest_unstable`. - Add `rustls-tls-no-provider` Cargo feature to use rustls without a crypto provider. - Fix `Accept-Encoding` header combinations. - Fix http3 resolving IPv6 addresses. - Internal: upgrade to rustls 0.23. #### v0.12.4 - Add `zstd` support, enabled with `zstd` Cargo feature. - Add `ClientBuilder::read_timeout(Duration)`, which applies the duration for each read operation. The timeout resets after a successful read. #### v0.12.3 - Add `FromStr` for `dns::Name`. - Add `ClientBuilder::built_in_webpki_certs(bool)` to enable them separately. - Add `ClientBuilder::built_in_native_certs(bool)` to enable them separately. - Fix sending `content-length: 0` for GET requests. - Fix response body `content_length()` to return value when timeout is configured. - Fix `ClientBuilder::resolve()` to use lowercase domain names. #### v0.12.2 - Fix missing ALPN when connecting to socks5 proxy with rustls. - Fix TLS version limits with rustls. - Fix not detected ALPN h2 from server with native-tls. #### v0.12.1 - Fix `ClientBuilder::interface()` when no TLS is enabled. - Fix `TlsInfo::peer_certificate()` being truncated with rustls. - Fix panic if `http2` feature disabled but TLS negotiated h2 in ALPN. - Fix `Display` for `Error` to not include its source error.
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- Cargo.lock | 74 +++++++++++++++++++++++++----------------------------- Cargo.toml | 10 ++++---- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb4668115b9..598d1f74f18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1932,12 +1932,12 @@ dependencies = [ [[package]] name = "cudarc" -version = "0.19.4" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f071cd6a7b5d51607df76aa2d426aaabc7a74bc6bdb885b8afa63a880572ad9b" +checksum = "3aa12038120eb13347a6ae2ffab1d34efe78150125108627fd85044dd4d6ff1e" dependencies = [ "half", - "libloading 0.9.0", + "libloading 0.8.9", ] [[package]] @@ -3534,7 +3534,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4563,7 +4563,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -4909,7 +4908,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4990,7 +4989,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5766,16 +5765,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "libloading" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" -dependencies = [ - "cfg-if", - "windows-link", -] - [[package]] name = "liblzma" version = "0.4.6" @@ -6326,7 +6315,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7947,9 +7936,8 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", - "webpki-roots", ] [[package]] @@ -7983,12 +7971,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -8193,7 +8183,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8251,7 +8241,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9327,10 +9317,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9349,7 +9339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10285,7 +10275,7 @@ dependencies = [ "parquet 58.1.0", "rand 0.10.1", "regex", - "reqwest 0.12.28", + "reqwest 0.13.2", "serde", "serde_json", "sysinfo", @@ -10379,7 +10369,7 @@ dependencies = [ "clap", "futures", "parquet 58.1.0", - "reqwest 0.12.28", + "reqwest 0.13.2", "serde", "serde_json", "sha2", @@ -10559,7 +10549,7 @@ dependencies = [ "object_store 0.13.2", "parking_lot", "paste", - "reqwest 0.12.28", + "reqwest 0.13.2", "rstest", "tempfile", "tracing", @@ -10864,7 +10854,7 @@ dependencies = [ "bindgen", "libloading 0.8.9", "liblzma", - "reqwest 0.12.28", + "reqwest 0.13.2", "tar", "vortex-cuda-macros", ] @@ -11290,6 +11280,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -11331,15 +11334,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "which" version = "8.0.2" @@ -11371,7 +11365,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 89d3e73c0d8..36cdbdb5a15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,7 +119,7 @@ chrono = "0.4.42" clap = "4.5" criterion = "0.8" crossterm = "0.29" -cudarc = { version = "0.19.0", features = [ +cudarc = { version = "0.18.2", features = [ # The NSight Compute version available on lambda.ai hosts does not inject a symbol for # several functions that cudarc expects to load when this is set to "cuda-12080". We # don't use any CUDA 12.8 specific features currently so this is fine. @@ -209,12 +209,12 @@ rand_distr = "0.6" ratatui = { version = "0.30", default-features = false } regex = "1.11.0" regex-automata = "0.4" -reqwest = { version = "0.12.4", features = [ +reqwest = { version = "0.13.0", features = [ + "blocking", "charset", "http2", - "macos-system-configuration", - "blocking", - "rustls-tls", + "rustls", + "system-proxy", ], default-features = false } roaring = "0.11.0" rstest = "0.26.1" From 6d499b9db131a2daf524f54dffe71c8cf8c295f7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:43:52 +0000 Subject: [PATCH 138/250] Update Rust crate sha2 to 0.11 (#7553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [sha2](https://redirect.github.com/RustCrypto/hashes) | workspace.dependencies | minor | `0.10` → `0.11` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
RustCrypto/hashes (sha2) ### [`v0.11.0`](https://redirect.github.com/RustCrypto/hashes/compare/sha2-v0.10.9...sha2-v0.11.0) [Compare Source](https://redirect.github.com/RustCrypto/hashes/compare/sha2-v0.10.9...sha2-v0.11.0)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- Cargo.lock | 92 +++++++++++++++++++++----- Cargo.toml | 3 +- vortex-test/compat-gen/Cargo.toml | 1 + vortex-test/compat-gen/src/generate.rs | 3 +- 4 files changed, 82 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 598d1f74f18..d67ebdebb97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,7 @@ dependencies = [ "bon", "bzip2", "crc32fast", - "digest", + "digest 0.10.7", "liblzma", "log", "miniz_oxide", @@ -900,6 +900,12 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" @@ -1014,7 +1020,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1040,6 +1046,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1393,7 +1408,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1698,6 +1713,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1909,6 +1930,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2766,7 +2796,7 @@ dependencies = [ "num-traits", "rand 0.9.2", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2798,7 +2828,7 @@ dependencies = [ "num-traits", "rand 0.9.2", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -3361,7 +3391,7 @@ dependencies = [ "rand 0.9.2", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "url", ] @@ -3511,11 +3541,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid", + "crypto-common 0.2.1", +] + [[package]] name = "dirs" version = "6.0.0" @@ -4463,7 +4504,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -4526,6 +4567,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -5967,7 +6017,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47bb1e988e6fb779cf720ad431242d3f03167c1b3f2b1aae7f1a94b2495b36ae" dependencies = [ - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6015,7 +6065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -6890,7 +6940,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", "hmac", ] @@ -8570,7 +8620,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -8581,7 +8631,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -10365,6 +10426,7 @@ version = "0.1.0" dependencies = [ "arrow-array 58.1.0", "arrow-select 58.1.0", + "base16ct", "bytes", "clap", "futures", @@ -10372,7 +10434,7 @@ dependencies = [ "reqwest 0.13.2", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "tempfile", "tokio", "tpchgen", diff --git a/Cargo.toml b/Cargo.toml index 36cdbdb5a15..12e0acb882d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,6 +105,7 @@ async-fs = "2.2.0" async-lock = "3.4" async-stream = "0.3.6" async-trait = "0.1.89" +base16ct = "1.0.0" bigdecimal = "0.4.8" bindgen = "0.72.0" bit-vec = "0.9.0" @@ -223,7 +224,7 @@ rustc-hash = "2.1" serde = "1.0.220" serde_json = "1.0.138" serde_test = "1.0.176" -sha2 = "0.10" +sha2 = "0.11.0" simdutf8 = "0.1.5" similar = "2.7.0" sketches-ddsketch = "0.4.0" diff --git a/vortex-test/compat-gen/Cargo.toml b/vortex-test/compat-gen/Cargo.toml index 62906843e8b..dd8bec4703a 100644 --- a/vortex-test/compat-gen/Cargo.toml +++ b/vortex-test/compat-gen/Cargo.toml @@ -45,6 +45,7 @@ tokio = { workspace = true, features = ["full"] } reqwest = { workspace = true } # CLI + serialization +base16ct = { workspace = true } clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/vortex-test/compat-gen/src/generate.rs b/vortex-test/compat-gen/src/generate.rs index 1393866c757..7490540f75e 100644 --- a/vortex-test/compat-gen/src/generate.rs +++ b/vortex-test/compat-gen/src/generate.rs @@ -3,6 +3,7 @@ use std::path::Path; +use base16ct::HexDisplay; use serde::Serialize; use sha2::Digest; use sha2::Sha256; @@ -50,7 +51,7 @@ pub fn write_fixtures(output_dir: &Path, exclude: &[String]) -> VortexResult Date: Mon, 20 Apr 2026 11:34:53 -0400 Subject: [PATCH 139/250] Remove weird file (#7568) ## Summary Remove weird file. ## Testing N/A Signed-off-by: Connor Tsui --- df-q6-parquet.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 df-q6-parquet.json diff --git a/df-q6-parquet.json b/df-q6-parquet.json deleted file mode 100644 index 0df39f7ed3a..00000000000 --- a/df-q6-parquet.json +++ /dev/null @@ -1 +0,0 @@ -{"all_runtimes":[140238709],"commit_id":"5b3726ee1725a1c21e3ab935d21ef48435e97197","dataset":{"tpch":{"scale_factor":"10.0"}},"env_triple":{"architecture":"aarch64","environment":"unknown","operating_system":"darwin"},"name":"tpch_q06/datafusion:parquet","storage":"nvme","target":{"engine":"datafusion","format":"parquet"},"unit":"ns","value":140238709} From ba90b10bde93321bce09fa22ce5a2cebf49929d3 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 20 Apr 2026 17:42:05 +0100 Subject: [PATCH 140/250] duckdb: define virtual columns on c++ side (#7571) Slightly reduce complexity. Virtual columns aren't dependent on anything, so we don't need to expose extra stuff Signed-off-by: Mikhail Kot --- .../cpp/include/duckdb_vx/table_function.h | 12 ------ vortex-duckdb/cpp/table_function.cpp | 32 ++------------ vortex-duckdb/src/datasource.rs | 15 ++----- .../src/duckdb/table_function/mod.rs | 8 ---- .../duckdb/table_function/virtual_columns.rs | 43 ------------------- 5 files changed, 8 insertions(+), 102 deletions(-) delete mode 100644 vortex-duckdb/src/duckdb/table_function/virtual_columns.rs diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h index b0e4532f7d0..7fe9eed251c 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h @@ -38,15 +38,6 @@ void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_result ffi_resu size_t name_len, duckdb_logical_type ffi_type); -// Opaque type for the result of get_virtual_columns -typedef struct duckdb_vx_tfunc_virtual_cols_result_ *duckdb_vx_tfunc_virtual_cols_result; -// Push a column into the get_virtual_columns result. -void duckdb_vx_tfunc_virtual_columns_push(duckdb_vx_tfunc_virtual_cols_result ffi_result, - idx_t column_idx, - const char *name_str, - size_t name_len, - duckdb_logical_type ffi_type); - // String map for to_string result typedef struct duckdb_vx_string_map_ *duckdb_vx_string_map; @@ -161,8 +152,6 @@ typedef struct { bool (*pushdown_complex_filter)(void *bind_data, duckdb_vx_expr expr, duckdb_vx_error *error_out); - void (*get_virtual_columns)(void *bind_data, duckdb_vx_tfunc_virtual_cols_result result_out); - void *pushdown_expression; duckdb_vx_string_map (*to_string)(void *bind_data); // void *dynamic_to_string; @@ -179,7 +168,6 @@ typedef struct { // void *supports_pushdown_type; // void *get_partition_info; // void *get_partition_stats; - // void *get_virtual_columns; // void *get_row_id_columns; bool projection_pushdown; diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index 45ab61a743e..5a114613623 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -377,33 +377,6 @@ extern "C" void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_resu result->return_types.emplace_back(*logical_type); } -virtual_column_map_t c_get_virtual_columns(ClientContext & /*context*/, - optional_ptr bind_data) { - auto &bind = bind_data->Cast(); - - auto result = virtual_column_map_t(); - bind.info->vtab.get_virtual_columns(bind_data->Cast().ffi_data->DataPtr(), - reinterpret_cast(&result)); - return result; -} - -extern "C" void duckdb_vx_tfunc_virtual_columns_push(duckdb_vx_tfunc_virtual_cols_result ffi_result, - idx_t column_idx, - const char *name_str, - size_t name_len, - duckdb_logical_type ffi_type) { - if (!ffi_result || !name_str || !ffi_type) { - return; - } - - auto result = reinterpret_cast(ffi_result); - const auto logical_type = reinterpret_cast(ffi_type); - const auto name = string(name_str, name_len); - - auto table_col = TableColumn(std::move(name), *logical_type); - result->emplace(column_idx, std::move(table_col)); -} - OperatorPartitionData c_get_partition_data(ClientContext & /*context*/, TableFunctionGetPartitionInput &input) { if (input.partition_info.RequiresPartitionColumns()) { @@ -463,11 +436,14 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d tf.late_materialization = vtab->late_materialization; tf.cardinality = c_cardinality; tf.get_partition_data = c_get_partition_data; - tf.get_virtual_columns = c_get_virtual_columns; tf.to_string = c_to_string; tf.table_scan_progress = c_table_scan_progress; tf.statistics = c_statistics; + tf.get_virtual_columns = [](auto &, auto) -> virtual_column_map_t { + return {{COLUMN_IDENTIFIER_EMPTY, TableColumn("", LogicalTypeId::BOOLEAN)}}; + }; + // Set up the parameters tf.arguments.reserve(vtab->parameter_count); for (size_t i = 0; i < vtab->parameter_count; i++) { diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index e7a795621cb..40ed1315e0b 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -5,7 +5,7 @@ //! //! Table functions that resolve to a [`DataSourceRef`] can implement [`DataSourceTableFunction`] //! to get a blanket [`TableFunction`] implementation covering init, scan, progress, filter -//! pushdown, cardinality, partitioning, and virtual columns. +//! pushdown, cardinality, and partitioning. use std::ffi::CString; use std::fmt::Debug; @@ -69,7 +69,6 @@ use crate::duckdb::LogicalType; use crate::duckdb::TableFilterSetRef; use crate::duckdb::TableFunction; use crate::duckdb::TableInitInput; -use crate::duckdb::VirtualColumnsResultRef; use crate::exporter::ArrayExporter; use crate::exporter::ConversionCache; @@ -83,17 +82,15 @@ use crate::exporter::ConversionCache; /// If you define COLUMN_IDENTIFIER_EMPTY, planner takes it, otherwise the /// first column. As we don't want to fill the output chunk and we can leave /// it uninitialized in this case, we define COLUMN_IDENTIFIER_EMPTY as a -/// virtual column in our table function vtab's get_virtual_columns. -/// See vortex-duckdb/cpp/include/duckdb_vx/table_function.h -/// See virtual_columns in this file +/// virtual column. +/// See vortex-duckdb/cpp/table_function.cpp static EMPTY_COLUMN_IDX: u64 = 18446744073709551614; -static EMPTY_COLUMN_NAME: &str = ""; /// A trait for table functions that resolve to a [`DataSourceRef`]. /// /// Implementors only need to define how parameters are declared and how binding produces a /// data source. All other [`TableFunction`] methods (init, scan, progress, filter pushdown, -/// cardinality, partitioning, virtual columns) are provided by a blanket implementation. +/// cardinality, partitioning) are provided by a blanket implementation. pub(crate) trait DataSourceTableFunction: Sized + Debug { /// Returns the positional parameters of the table function. fn parameters() -> Vec { @@ -555,10 +552,6 @@ impl TableFunction for T { Some(result) } - - fn virtual_columns(_bind_data: &Self::BindData, result: &mut VirtualColumnsResultRef) { - result.register(EMPTY_COLUMN_IDX, EMPTY_COLUMN_NAME, &LogicalType::bool()); - } } // --------------------------------------------------------------------------- diff --git a/vortex-duckdb/src/duckdb/table_function/mod.rs b/vortex-duckdb/src/duckdb/table_function/mod.rs index 36cd1464cc6..7b9f9592728 100644 --- a/vortex-duckdb/src/duckdb/table_function/mod.rs +++ b/vortex-duckdb/src/duckdb/table_function/mod.rs @@ -16,12 +16,9 @@ mod partition; mod pushdown_complex_filter; mod statistics; mod table_scan_progress; -mod virtual_columns; pub use bind::*; pub use init::*; -pub use virtual_columns::VirtualColumnsResult; -pub use virtual_columns::VirtualColumnsResultRef; use crate::cpp; use crate::cpp::duckdb_client_context; @@ -38,7 +35,6 @@ use crate::duckdb::table_function::partition::get_partition_data_callback; use crate::duckdb::table_function::pushdown_complex_filter::pushdown_complex_filter_callback; use crate::duckdb::table_function::statistics::statistics; use crate::duckdb::table_function::table_scan_progress::table_scan_progress_callback; -use crate::duckdb::table_function::virtual_columns::get_virtual_columns_callback; use crate::duckdb_try; #[derive(Debug, Default)] @@ -157,9 +153,6 @@ pub trait TableFunction: Sized + Debug { _local_init_data: &mut Self::LocalState, ) -> VortexResult; - /// Returns the virtual columns of the table function. - fn virtual_columns(_bind_data: &Self::BindData, _result: &mut VirtualColumnsResultRef) {} - /// Returns a vector of key-value pairs for EXPLAIN output fn to_string(_bind_data: &Self::BindData) -> Option> { None @@ -209,7 +202,6 @@ impl DatabaseRef { cardinality: Some(cardinality_callback::), pushdown_complex_filter: Some(pushdown_complex_filter_callback::), pushdown_expression: ptr::null_mut::(), - get_virtual_columns: Some(get_virtual_columns_callback::), to_string: Some(to_string_callback::), table_scan_progress: Some(table_scan_progress_callback::), get_partition_data: Some(get_partition_data_callback::), diff --git a/vortex-duckdb/src/duckdb/table_function/virtual_columns.rs b/vortex-duckdb/src/duckdb/table_function/virtual_columns.rs deleted file mode 100644 index 7e4b06e48b9..00000000000 --- a/vortex-duckdb/src/duckdb/table_function/virtual_columns.rs +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::ffi::c_void; - -use vortex::error::VortexExpect; - -use crate::cpp; -use crate::duckdb::LogicalTypeRef; -use crate::duckdb::TableFunction; -use crate::lifetime_wrapper; - -/// Native callback for the get_virtual_columns function. -pub(crate) unsafe extern "C-unwind" fn get_virtual_columns_callback( - bind_data: *mut c_void, - result: cpp::duckdb_vx_tfunc_virtual_cols_result, -) { - let bind_data = - unsafe { bind_data.cast::().as_ref() }.vortex_expect("bind_data null pointer"); - let result = unsafe { VirtualColumnsResult::borrow_mut(result) }; - - T::virtual_columns(bind_data, result); -} - -lifetime_wrapper!( - VirtualColumnsResult, - cpp::duckdb_vx_tfunc_virtual_cols_result, - |_| {} -); - -impl VirtualColumnsResultRef { - pub fn register(&self, column_idx: u64, name: &str, logical_type: &LogicalTypeRef) { - unsafe { - cpp::duckdb_vx_tfunc_virtual_columns_push( - self.as_ptr(), - column_idx as _, - name.as_ptr().cast(), - name.len() as _, - logical_type.as_ptr(), - ) - } - } -} From 58ed3c2a7a8d3eb306e4b01faf123ea207a4d514 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 20 Apr 2026 13:31:43 -0400 Subject: [PATCH 141/250] Fuzzer cloudflare bucket is configured only in store and persist (#7570) Looks like sccache wanted to pickup this config which is not what we want Signed-off-by: Robert Kruszewski --- .github/workflows/fuzz-coverage.yml | 10 +++++----- .../workflows/minimize_fuzz_corpus_workflow.yml | 15 ++++++++++----- .github/workflows/run-fuzzer.yml | 14 ++++++++++---- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index 0b3c2c14e54..de1e40891e6 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -11,11 +11,6 @@ env: jobs: coverage: name: "Coverage: ${{ matrix.fuzz_target }}" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" strategy: fail-fast: false matrix: @@ -61,6 +56,11 @@ jobs: - name: Download corpus from R2 shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ matrix.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ matrix.fuzz_target }}" diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index 9193b3d1d68..5e0fb20900a 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -34,11 +34,6 @@ env: jobs: minimize: name: "Minimize ${{ inputs.fuzz_target }}" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=arm64-medium/disk=large/tag={1}-minimize', github.run_id, inputs.fuzz_target) @@ -70,6 +65,11 @@ jobs: - name: Restore corpus shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" @@ -99,6 +99,11 @@ jobs: - name: Persist corpus shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index 959e6b345d7..bd6dcbd6976 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -61,10 +61,6 @@ jobs: name: "Run ${{ inputs.fuzz_name || inputs.fuzz_target }}" env: FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }} - AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" timeout-minutes: 240 # 4 hours runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -99,6 +95,11 @@ jobs: - name: Restore corpus shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" @@ -188,6 +189,11 @@ jobs: - name: Persist corpus shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" run: | CORPUS_KEY="${FUZZ_NAME}_corpus.tar.zst" CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" From 76ccbdfda0b48aef839824ccb71969bd7a466ae6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 20 Apr 2026 18:48:18 +0100 Subject: [PATCH 142/250] break: remove deprecated methods and plumb ExecutionCtx (#7512) ## Summary Remove deprecated compute methods that lack an `ExecutionCtx` parameter (e.g., `to_canonical`) and plumb `ExecutionCtx` through remaining callsites. Key changes: - Remove deprecated non-ctx methods across encodings - Add `&mut ExecutionCtx` parameter to `BitPacked::encode`, `RunEnd::new`, and other encoding methods - Fix all callsites in benchmarks, tests, CUDA code, compat-gen, and production code - Fix clippy warnings (redundant clones, too-many-arguments, auto-deref) - Regenerate public-api.lock files ## API Changes Breaking change: encoding methods like `BitPacked::encode` now require an `&mut ExecutionCtx` parameter. --------- Signed-off-by: Joe Isaacs Co-authored-by: Claude Opus 4.6 (1M context) --- encodings/alp/benches/alp_compress.rs | 9 +- encodings/alp/public-api.lock | 4 +- encodings/alp/src/alp/array.rs | 18 +- encodings/alp/src/alp/compress.rs | 162 ++++++------ encodings/alp/src/alp/compute/cast.rs | 53 ++-- encodings/alp/src/alp/compute/compare.rs | 62 +++-- encodings/alp/src/alp/compute/filter.rs | 13 +- encodings/alp/src/alp/compute/mask.rs | 30 +-- encodings/alp/src/alp/compute/take.rs | 13 +- encodings/alp/src/alp_rd/array.rs | 38 +-- encodings/alp/src/alp_rd/compute/cast.rs | 28 ++- encodings/alp/src/alp_rd/compute/filter.rs | 14 +- encodings/alp/src/alp_rd/compute/mask.rs | 15 +- encodings/alp/src/alp_rd/compute/mod.rs | 24 +- encodings/alp/src/alp_rd/compute/take.rs | 27 +- encodings/alp/src/alp_rd/mod.rs | 11 +- encodings/alp/src/alp_rd/ops.rs | 11 +- encodings/datetime-parts/public-api.lock | 12 +- encodings/datetime-parts/src/array.rs | 7 +- encodings/datetime-parts/src/canonical.rs | 14 +- encodings/datetime-parts/src/compress.rs | 52 +--- encodings/datetime-parts/src/compute/cast.rs | 49 ++-- .../datetime-parts/src/compute/compare.rs | 15 +- .../datetime-parts/src/compute/filter.rs | 7 +- encodings/datetime-parts/src/compute/mod.rs | 5 +- encodings/datetime-parts/src/compute/rules.rs | 8 +- encodings/datetime-parts/src/compute/take.rs | 19 +- .../src/decimal_byte_parts/compute/cast.rs | 16 +- .../fastlanes/benches/bitpacking_take.rs | 56 +++-- .../fastlanes/benches/canonicalize_bench.rs | 38 ++- .../fastlanes/benches/compute_between.rs | 23 +- encodings/fastlanes/public-api.lock | 22 +- .../src/bitpacking/array/bitpack_compress.rs | 131 +++++----- .../bitpacking/array/bitpack_decompress.rs | 69 +++--- .../fastlanes/src/bitpacking/array/mod.rs | 38 ++- .../src/bitpacking/array/unpack_iter.rs | 4 +- .../fastlanes/src/bitpacking/compute/cast.rs | 4 +- .../src/bitpacking/compute/filter.rs | 44 ++-- .../src/bitpacking/compute/is_constant.rs | 24 +- .../fastlanes/src/bitpacking/compute/mod.rs | 10 +- .../fastlanes/src/bitpacking/compute/slice.rs | 5 +- .../fastlanes/src/bitpacking/compute/take.rs | 60 +++-- encodings/fastlanes/src/bitpacking/plugin.rs | 7 +- .../fastlanes/src/bitpacking/vtable/mod.rs | 8 +- .../src/bitpacking/vtable/operations.rs | 8 +- .../src/delta/array/delta_compress.rs | 18 +- .../fastlanes/src/delta/vtable/operations.rs | 9 +- .../fastlanes/src/for/array/for_compress.rs | 28 +-- .../fastlanes/src/for/compute/is_sorted.rs | 5 +- encodings/fastlanes/src/lib.rs | 29 ++- encodings/fastlanes/src/rle/array/mod.rs | 73 +++--- .../fastlanes/src/rle/array/rle_compress.rs | 231 +++++++++++------- encodings/fastlanes/src/rle/compute/cast.rs | 20 +- encodings/fastlanes/src/rle/kernel.rs | 6 +- encodings/fastlanes/src/rle/vtable/mod.rs | 7 +- .../fastlanes/src/rle/vtable/operations.rs | 24 +- .../fsst/benches/chunked_dict_fsst_builder.rs | 3 +- encodings/fsst/benches/fsst_compress.rs | 33 ++- encodings/fsst/benches/fsst_like.rs | 21 +- encodings/fsst/benches/fsst_url_compare.rs | 48 +++- encodings/fsst/public-api.lock | 12 +- encodings/fsst/src/array.rs | 24 +- encodings/fsst/src/canonical.rs | 20 +- encodings/fsst/src/compress.rs | 22 +- encodings/fsst/src/compute/cast.rs | 13 +- encodings/fsst/src/compute/compare.rs | 15 +- encodings/fsst/src/compute/filter.rs | 1 + encodings/fsst/src/compute/like.rs | 16 +- encodings/fsst/src/compute/mod.rs | 44 ++-- encodings/fsst/src/dfa/tests.rs | 8 +- encodings/fsst/src/kernel.rs | 27 +- encodings/fsst/src/test_utils.rs | 45 ++-- encodings/fsst/src/tests.rs | 16 +- encodings/pco/public-api.lock | 6 +- encodings/pco/src/array.rs | 52 ++-- encodings/pco/src/compute/cast.rs | 17 +- encodings/pco/src/compute/mod.rs | 66 ++++- encodings/pco/src/tests.rs | 31 ++- encodings/runend/benches/run_end_compress.rs | 13 +- encodings/runend/benches/run_end_decode.rs | 26 +- .../runend/benches/run_end_null_count.rs | 2 +- encodings/runend/public-api.lock | 18 +- encodings/runend/src/arbitrary.rs | 8 +- encodings/runend/src/array.rs | 93 ++++--- encodings/runend/src/arrow.rs | 13 +- encodings/runend/src/compress.rs | 69 +++--- encodings/runend/src/compute/cast.rs | 75 +++--- encodings/runend/src/compute/compare.rs | 16 +- encodings/runend/src/compute/fill_null.rs | 5 + encodings/runend/src/compute/filter.rs | 17 +- encodings/runend/src/compute/mod.rs | 20 +- encodings/runend/src/compute/take.rs | 29 ++- encodings/runend/src/compute/take_from.rs | 39 +-- encodings/runend/src/decompress_bool.rs | 55 ++--- encodings/runend/src/kernel.rs | 17 +- encodings/runend/src/ops.rs | 25 +- encodings/runend/src/rules.rs | 5 + encodings/sequence/public-api.lock | 2 +- encodings/sequence/src/compress.rs | 46 ++-- encodings/sequence/src/compute/cast.rs | 16 +- encodings/sparse/public-api.lock | 4 +- encodings/sparse/src/canonical.rs | 188 ++++++++------ encodings/sparse/src/compute/cast.rs | 12 +- encodings/sparse/src/lib.rs | 43 ++-- encodings/sparse/src/ops.rs | 8 +- encodings/zigzag/src/array.rs | 8 +- encodings/zigzag/src/compress.rs | 20 +- encodings/zigzag/src/compute/mod.rs | 6 +- encodings/zstd/benches/listview_rebuild.rs | 4 +- encodings/zstd/public-api.lock | 18 +- encodings/zstd/src/array.rs | 72 +++--- encodings/zstd/src/compute/cast.rs | 28 ++- encodings/zstd/src/compute/mod.rs | 18 +- encodings/zstd/src/test.rs | 59 +++-- fuzz/fuzz_targets/file_io.rs | 16 +- fuzz/src/array/cast.rs | 33 +-- fuzz/src/array/compare.rs | 48 ++-- fuzz/src/array/fill_null.rs | 89 ++++--- fuzz/src/array/filter.rs | 35 ++- fuzz/src/array/mask.rs | 140 +++++------ fuzz/src/array/mod.rs | 63 ++--- fuzz/src/array/scalar_at.rs | 34 +-- fuzz/src/array/search_sorted.rs | 40 ++- fuzz/src/array/slice.rs | 46 ++-- fuzz/src/array/sort.rs | 45 +--- fuzz/src/array/take.rs | 38 ++- fuzz/src/compress.rs | 9 +- fuzz/src/fsst_like.rs | 10 +- vortex-bench/src/datasets/mod.rs | 3 +- .../src/datasets/struct_list_of_ints.rs | 6 +- vortex-bench/src/datasets/taxi_data.rs | 3 +- vortex-bench/src/datasets/tpch_l_comment.rs | 25 +- vortex-bench/src/downloadable_dataset.rs | 3 +- vortex-bench/src/public_bi.rs | 3 +- vortex-btrblocks/benches/compress.rs | 11 +- vortex-btrblocks/public-api.lock | 14 +- vortex-btrblocks/src/schemes/decimal.rs | 9 +- vortex-btrblocks/src/schemes/float.rs | 47 ++-- vortex-btrblocks/src/schemes/integer.rs | 161 ++++++++---- vortex-btrblocks/src/schemes/patches.rs | 28 ++- vortex-btrblocks/src/schemes/string.rs | 60 +++-- vortex-btrblocks/src/schemes/temporal.rs | 28 +-- vortex-compressor/benches/dict_encode.rs | 4 +- vortex-compressor/benches/stats_calc.rs | 29 ++- vortex-compressor/public-api.lock | 22 +- .../src/builtins/constant/bool.rs | 6 +- .../src/builtins/constant/float.rs | 7 +- .../src/builtins/constant/integer.rs | 6 +- .../src/builtins/constant/string.rs | 6 +- vortex-compressor/src/builtins/dict/float.rs | 31 ++- .../src/builtins/dict/integer.rs | 27 +- vortex-compressor/src/builtins/dict/string.rs | 6 +- vortex-compressor/src/compressor.rs | 26 +- vortex-compressor/src/stats/bool.rs | 24 +- vortex-compressor/src/stats/cache.rs | 44 ++-- vortex-compressor/src/stats/float.rs | 51 ++-- vortex-compressor/src/stats/integer.rs | 50 ++-- vortex-compressor/src/stats/string.rs | 18 +- vortex-cuda/benches/bitpacked_cuda.rs | 8 +- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 39 +-- vortex-cuda/benches/for_cuda.rs | 6 +- vortex-cuda/benches/runend_cuda.rs | 11 +- vortex-cuda/benches/zstd_cuda.rs | 18 +- vortex-cuda/gpu-scan-cli/src/main.rs | 7 +- vortex-cuda/src/arrow/canonical.rs | 9 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 149 ++++++++--- vortex-cuda/src/hybrid_dispatch/mod.rs | 11 +- vortex-cuda/src/kernel/encodings/bitpacked.rs | 72 ++++-- vortex-cuda/src/kernel/encodings/for_.rs | 6 +- vortex-cuda/src/kernel/encodings/runend.rs | 33 ++- vortex-cuda/src/kernel/encodings/zstd.rs | 8 +- vortex-cuda/src/kernel/patches/mod.rs | 32 ++- vortex-duckdb/src/convert/vector.rs | 120 +++++---- .../src/e2e_test/vortex_scan_test.rs | 6 +- vortex-duckdb/src/exporter/dict.rs | 3 +- vortex-ffi/src/array.rs | 14 +- vortex-ffi/src/expression.rs | 31 ++- vortex-ffi/src/struct_array.rs | 11 +- vortex-file/src/tests.rs | 181 +++++++++----- vortex-file/src/writer.rs | 1 + vortex-file/tests/test_write_table.rs | 15 +- vortex-jni/src/array.rs | 66 +++-- vortex-layout/public-api.lock | 4 +- vortex-layout/src/layouts/dict/reader.rs | 7 +- vortex-layout/src/layouts/file_stats.rs | 36 ++- vortex-layout/src/layouts/flat/writer.rs | 24 +- vortex-layout/src/layouts/repartition.rs | 23 +- vortex-layout/src/layouts/row_idx/mod.rs | 36 ++- vortex-layout/src/layouts/struct_/reader.rs | 17 +- vortex-layout/src/layouts/table.rs | 11 +- vortex-layout/src/layouts/zoned/reader.rs | 9 +- vortex-layout/src/layouts/zoned/zone_map.rs | 52 ++-- vortex-layout/src/scan/scan_builder.rs | 8 +- vortex-python/src/arrays/compressed.rs | 12 +- vortex-python/src/arrays/mod.rs | 27 +- vortex-python/src/dataset.rs | 23 +- vortex-python/src/file.rs | 25 +- .../arrays/synthetic/encodings/alprd.rs | 35 ++- .../arrays/synthetic/encodings/bitpacked.rs | 35 +-- .../synthetic/encodings/datetimeparts.rs | 4 +- .../arrays/synthetic/encodings/fsst.rs | 36 ++- .../arrays/synthetic/encodings/pco.rs | 21 +- .../arrays/synthetic/encodings/rle.rs | 22 +- .../arrays/synthetic/encodings/runend.rs | 23 +- .../arrays/synthetic/encodings/sparse.rs | 27 +- .../arrays/synthetic/encodings/zstd.rs | 19 +- vortex-tui/src/browse/ui/layouts.rs | 34 +-- .../common_encoding_tree_throughput.rs | 100 +++++--- vortex/benches/single_encoding_throughput.rs | 85 +++++-- 209 files changed, 3824 insertions(+), 2566 deletions(-) diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index d0a23b64b21..7fbe60d5997 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -143,8 +143,8 @@ fn compress_rd(bencher: Bencher, args: (usize, f64) let encoder = RDEncoder::new(primitive.as_slice::()); bencher - .with_inputs(|| (&primitive, &encoder)) - .bench_refs(|(primitive, encoder)| encoder.encode(primitive.as_view())) + .with_inputs(|| (&primitive, &encoder, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(primitive, encoder, ctx)| encoder.encode(primitive.as_view(), ctx)) } #[divan::bench(types = [f32, f64], args = RD_BENCH_ARGS)] @@ -152,7 +152,10 @@ fn decompress_rd(bencher: Bencher, args: (usize, f6 let (n, fraction_patch) = args; let primitive = make_rd_array::(n, fraction_patch); let encoder = RDEncoder::new(primitive.as_slice::()); - let encoded = encoder.encode(primitive.as_view()); + let encoded = encoder.encode( + primitive.as_view(), + &mut LEGACY_SESSION.create_execution_ctx(), + ); bencher .with_inputs(|| (&encoded, LEGACY_SESSION.create_execution_ctx())) diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index b4dcad31098..e41427eed3d 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -146,7 +146,7 @@ impl vortex_alp::ALPRD pub unsafe fn vortex_alp::ALPRD::new_unchecked(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option) -> vortex_alp::ALPRDArray -pub fn vortex_alp::ALPRD::try_new(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRD::try_new(dtype: vortex_array::dtype::DType, left_parts: vortex_array::array::erased::ArrayRef, left_parts_dictionary: vortex_buffer::buffer::Buffer, right_parts: vortex_array::array::erased::ArrayRef, right_bit_width: u8, left_parts_patches: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_alp::ALPRD @@ -380,7 +380,7 @@ pub struct vortex_alp::RDEncoder impl vortex_alp::RDEncoder -pub fn vortex_alp::RDEncoder::encode(&self, array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_alp::ALPRDArray +pub fn vortex_alp::RDEncoder::encode(&self, array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_alp::ALPRDArray pub fn vortex_alp::RDEncoder::from_parts(right_bit_width: u8, codes: alloc::vec::Vec) -> Self diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 887111299ab..dd11f3eb60a 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -526,8 +526,6 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -785,6 +783,7 @@ mod tests { #[case(1000, 200)] #[case(2048, 512)] fn test_sliced_to_primitive(#[case] size: usize, #[case] slice_start: usize) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Vec> = (0..size) .map(|i| { if i % 5 == 0 { @@ -798,19 +797,13 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; let sliced_encoded = encoded.slice(slice_start..slice_end).unwrap(); - #[expect(deprecated)] - let result_primitive = sliced_encoded.to_primitive(); + let result_primitive = sliced_encoded.execute::(&mut ctx).unwrap(); for idx in 0..slice_len { let expected_value = values[slice_start + idx]; @@ -819,10 +812,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( - result_primitive.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) + .to_mask(result_primitive.as_ref().len(), &mut ctx) .unwrap() .value(idx); assert_eq!( diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 7be9c0b6537..392118ad472 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -136,8 +136,6 @@ mod tests { use f64::consts::E; use f64::consts::PI; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; use vortex_array::dtype::NativePType; @@ -273,16 +271,14 @@ mod tests { #[test] fn roundtrips_all_null() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let original = PrimitiveArray::new(buffer![195.26274f64, PI, -48.815685], Validity::AllInvalid); - let alp_arr = alp_encode( - original.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); - #[expect(deprecated)] - let decompressed = alp_arr.into_array().to_primitive(); + let alp_arr = alp_encode(original.as_view(), None, &mut ctx).unwrap(); + let decompressed = alp_arr + .into_array() + .execute::(&mut ctx) + .unwrap(); assert_eq!( // The second and third values become exceptions and are replaced @@ -295,18 +291,17 @@ mod tests { #[test] fn non_finite_numbers() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let original = PrimitiveArray::new( buffer![0.0f32, -0.0, f32::NAN, f32::NEG_INFINITY, f32::INFINITY], Validity::NonNullable, ); - let encoded = alp_encode( - original.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); - #[expect(deprecated)] - let decoded = encoded.as_array().to_primitive(); + let encoded = alp_encode(original.as_view(), None, &mut ctx).unwrap(); + let decoded = encoded + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); for idx in 0..original.len() { let decoded_val = decoded.as_slice::()[idx]; let original_val = original.as_slice::()[idx]; @@ -319,6 +314,7 @@ mod tests { #[test] fn test_chunk_offsets() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut values = vec![1.0f64; 3072]; values[1023] = PI; @@ -326,118 +322,141 @@ mod tests { values[1025] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); let patches = encoded.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .clone() + .unwrap() + .execute::(&mut ctx) + .unwrap(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 3]); assert_arrays_eq!(chunk_offsets, expected_offsets); - #[expect(deprecated)] - let patch_indices = patches.indices().to_primitive(); + let patch_indices = patches + .indices() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_indices = PrimitiveArray::from_iter(vec![1023u64, 1024, 1025]); assert_arrays_eq!(patch_indices, expected_indices); - #[expect(deprecated)] - let patch_values = patches.values().to_primitive(); + let patch_values = patches + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_values = PrimitiveArray::from_iter(vec![PI, E, PI]); assert_arrays_eq!(patch_values, expected_values); } #[test] fn test_chunk_offsets_no_patches_in_middle() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut values = vec![1.0f64; 3072]; values[0] = PI; values[2048] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); let patches = encoded.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .clone() + .unwrap() + .execute::(&mut ctx) + .unwrap(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]); assert_arrays_eq!(chunk_offsets, expected_offsets); - #[expect(deprecated)] - let patch_indices = patches.indices().to_primitive(); + let patch_indices = patches + .indices() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_indices = PrimitiveArray::from_iter(vec![0u64, 2048]); assert_arrays_eq!(patch_indices, expected_indices); - #[expect(deprecated)] - let patch_values = patches.values().to_primitive(); + let patch_values = patches + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_values = PrimitiveArray::from_iter(vec![PI, E]); assert_arrays_eq!(patch_values, expected_values); } #[test] fn test_chunk_offsets_trailing_empty_chunks() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut values = vec![1.0f64; 3072]; values[0] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); let patches = encoded.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .clone() + .unwrap() + .execute::(&mut ctx) + .unwrap(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64, 1, 1]); assert_arrays_eq!(chunk_offsets, expected_offsets); - #[expect(deprecated)] - let patch_indices = patches.indices().to_primitive(); + let patch_indices = patches + .indices() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_indices = PrimitiveArray::from_iter(vec![0u64]); assert_arrays_eq!(patch_indices, expected_indices); - #[expect(deprecated)] - let patch_values = patches.values().to_primitive(); + let patch_values = patches + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_values = PrimitiveArray::from_iter(vec![PI]); assert_arrays_eq!(patch_values, expected_values); } #[test] fn test_chunk_offsets_single_chunk() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut values = vec![1.0f64; 512]; values[0] = PI; values[100] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); let patches = encoded.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .clone() + .unwrap() + .execute::(&mut ctx) + .unwrap(); let expected_offsets = PrimitiveArray::from_iter(vec![0u64]); assert_arrays_eq!(chunk_offsets, expected_offsets); - #[expect(deprecated)] - let patch_indices = patches.indices().to_primitive(); + let patch_indices = patches + .indices() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_indices = PrimitiveArray::from_iter(vec![0u64, 100]); assert_arrays_eq!(patch_indices, expected_indices); - #[expect(deprecated)] - let patch_values = patches.values().to_primitive(); + let patch_values = patches + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_values = PrimitiveArray::from_iter(vec![PI, E]); assert_arrays_eq!(patch_values, expected_values); } @@ -526,21 +545,16 @@ mod tests { #[test] fn test_slice_half_chunk_nullable_roundtrip() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = (0..1024) .map(|i| if i % 3 == 0 { None } else { Some(2.5f32) }) .collect::>(); let original = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode( - original.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(original.as_view(), None, &mut ctx).unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); - #[expect(deprecated)] - let decoded = sliced_alp.to_primitive(); + let decoded = sliced_alp.execute::(&mut ctx).unwrap(); let expected_slice = original.slice(512..1024).unwrap(); assert_arrays_eq!(decoded, expected_slice); diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index 99eaa09ff2c..45adf5a46cd 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -61,8 +61,6 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -80,14 +78,10 @@ mod tests { #[test] fn issue_5766_test_cast_alp_with_patches_to_nullable() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = buffer![1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456].into_array(); - #[expect(deprecated)] - let values_primitive = values.to_primitive(); - let alp = alp_encode( - values_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let values_primitive = values.clone().execute::(&mut ctx)?; + let alp = alp_encode(values_primitive.as_view(), None, &mut ctx)?; assert!( alp.patches().is_some(), @@ -99,8 +93,7 @@ mod tests { let expected = values.cast(nullable_dtype)?; - #[expect(deprecated)] - let casted_prim = casted.to_canonical()?.into_primitive(); + let casted_prim = casted.execute::(&mut ctx)?; assert_arrays_eq!(casted_prim, expected); Ok(()) @@ -108,14 +101,10 @@ mod tests { #[test] fn test_cast_alp_f32_to_f64() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = buffer![1.5f32, 2.5, 3.5, 4.5].into_array(); - #[expect(deprecated)] - let values_primitive = values.to_primitive(); - let alp = alp_encode( - values_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let values_primitive = values.execute::(&mut ctx)?; + let alp = alp_encode(values_primitive.as_view(), None, &mut ctx)?; let casted = alp .into_array() @@ -125,8 +114,7 @@ mod tests { &DType::Primitive(PType::F64, Nullability::NonNullable) ); - #[expect(deprecated)] - let decoded = casted.to_canonical()?.into_primitive(); + let decoded = casted.execute::(&mut ctx)?; let values = decoded.as_slice::(); assert_eq!(values.len(), 4); assert!((values[0] - 1.5).abs() < f64::EPSILON); @@ -137,14 +125,10 @@ mod tests { #[test] fn test_cast_alp_to_int() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = buffer![1.0f32, 2.0, 3.0, 4.0].into_array(); - #[expect(deprecated)] - let values_primitive = values.to_primitive(); - let alp = alp_encode( - values_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let values_primitive = values.execute::(&mut ctx)?; + let alp = alp_encode(values_primitive.as_view(), None, &mut ctx)?; let casted = alp .into_array() @@ -154,8 +138,7 @@ mod tests { &DType::Primitive(PType::I32, Nullability::NonNullable) ); - #[expect(deprecated)] - let decoded = casted.to_canonical()?.into_primitive(); + let decoded = casted.execute::(&mut ctx)?; assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i32, 2, 3, 4])); Ok(()) @@ -168,14 +151,10 @@ mod tests { #[case(buffer![42.42f64].into_array())] #[case(buffer![0.0f32, -1.5, 2.5, -3.5, 4.5].into_array())] fn test_cast_alp_conformance(#[case] array: vortex_array::ArrayRef) -> VortexResult<()> { - #[expect(deprecated)] - let array_primitive = array.to_primitive(); - let alp = alp_encode( - array_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("cannot fail"); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array_primitive = array.execute::(&mut ctx)?; + let alp = + alp_encode(array_primitive.as_view(), None, &mut ctx).vortex_expect("cannot fail"); test_cast_conformance(&alp.into_array()); Ok(()) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index d0eca850b8c..62fc092c284 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -153,8 +153,6 @@ mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; @@ -185,16 +183,15 @@ mod tests { #[test] fn basic_comparison_test() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(encoded.patches().is_none()); - #[expect(deprecated)] - let encoded_prim = encoded.encoded().to_primitive(); + let encoded_prim = encoded + .encoded() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(encoded_prim.as_slice::(), vec![1234; 1025]); let r = alp_scalar_compare(encoded.as_view(), 1.3_f32, CompareOperator::Eq) @@ -212,16 +209,15 @@ mod tests { #[test] fn comparison_with_unencodable_value() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(encoded.patches().is_none()); - #[expect(deprecated)] - let encoded_prim = encoded.encoded().to_primitive(); + let encoded_prim = encoded + .encoded() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(encoded_prim.as_slice::(), vec![1234; 1025]); let r_eq = alp_scalar_compare(encoded.as_view(), 1.234444_f32, CompareOperator::Eq) @@ -239,16 +235,15 @@ mod tests { #[test] fn comparison_range() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([0.0605_f32; 10]); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(encoded.patches().is_none()); - #[expect(deprecated)] - let encoded_prim = encoded.encoded().to_primitive(); + let encoded_prim = encoded + .encoded() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(encoded_prim.as_slice::(), vec![605; 10]); // !(0.0605_f32 >= 0.06051_f32); @@ -282,16 +277,15 @@ mod tests { #[test] fn comparison_zeroes() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([0.0_f32; 10]); - let encoded = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let encoded = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(encoded.patches().is_none()); - #[expect(deprecated)] - let encoded_prim = encoded.encoded().to_primitive(); + let encoded_prim = encoded + .encoded() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(encoded_prim.as_slice::(), vec![0; 10]); let r_gte = diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 2843e72f8e1..97b5b4af49e 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -45,8 +45,6 @@ mod test { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::filter::test_filter_conformance; @@ -64,14 +62,9 @@ mod test { 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 ].into_array())] fn test_filter_alp_conformance(#[case] array: ArrayRef) { - #[expect(deprecated)] - let array_primitive = array.to_primitive(); - let alp = alp_encode( - array_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array_primitive = array.execute::(&mut ctx).unwrap(); + let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); test_filter_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 2123e4d85f8..31f131f6e0c 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -57,8 +57,6 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -80,48 +78,34 @@ mod test { 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0 ].into_array())] fn test_mask_alp_conformance(#[case] array: vortex_array::ArrayRef) { - #[expect(deprecated)] - let array_primitive = array.to_primitive(); - let alp = alp_encode( - array_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array_primitive = array.execute::(&mut ctx).unwrap(); + let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); test_mask_conformance(&alp.into_array()); } #[test] fn test_mask_alp_with_patches() { use std::f64::consts::PI; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // PI doesn't encode cleanly with ALP, so it creates patches. let values: Vec = (0..100) .map(|i| if i % 4 == 3 { PI } else { 1.0 }) .collect(); let array = PrimitiveArray::from_iter(values); - let alp = alp_encode( - array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let alp = alp_encode(array.as_view(), None, &mut ctx).unwrap(); assert!(alp.patches().is_some(), "expected patches"); test_mask_conformance(&alp.into_array()); } #[test] fn test_mask_alp_with_patches_casts_surviving_patch_values_to_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456]); - let alp = alp_encode( - values.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let alp = alp_encode(values.as_view(), None, &mut ctx).unwrap(); assert!(alp.patches().is_some(), "expected patches"); let keep_mask = BoolArray::from_iter([false, true, true, true, true]).into_array(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let masked = ::mask(alp.as_view(), &keep_mask, &mut ctx) .unwrap() .unwrap(); diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index daeead2a4d0..3b2b35ceaa9 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -43,8 +43,6 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::take::test_take_conformance; @@ -58,14 +56,9 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).into_array())] #[case(buffer![42.42f64].into_array())] fn test_take_alp_conformance(#[case] array: vortex_array::ArrayRef) { - #[expect(deprecated)] - let array_primitive = array.to_primitive(); - let alp = alp_encode( - array_primitive.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array_primitive = array.execute::(&mut ctx).unwrap(); + let alp = alp_encode(array_primitive.as_view(), None, &mut ctx).unwrap(); test_take_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index a9bce5c7f78..5573c43d694 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -16,6 +16,7 @@ use vortex_array::ArrayId; use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -213,7 +214,13 @@ impl VTable for ALPRD { ) }) .transpose()?; - let left_parts_patches = ALPRDData::canonicalize_patches(&left_parts, left_parts_patches)?; + // NOTE: `VTable::deserialize` has a fixed trait signature without `ExecutionCtx`, so we + // cannot plumb a ctx in here. We construct a legacy ctx locally at this trait boundary. + let left_parts_patches = ALPRDData::canonicalize_patches( + &left_parts, + left_parts_patches, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let slots = ALPRDData::make_slots(&left_parts, &right_parts, &left_parts_patches); let data = ALPRDData::new( left_parts_dictionary, @@ -371,9 +378,11 @@ impl ALPRD { right_parts: ArrayRef, right_bit_width: u8, left_parts_patches: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult { let len = left_parts.len(); - let left_parts_patches = ALPRDData::canonicalize_patches(&left_parts, left_parts_patches)?; + let left_parts_patches = + ALPRDData::canonicalize_patches(&left_parts, left_parts_patches, ctx)?; let slots = ALPRDData::make_slots(&left_parts, &right_parts, &left_parts_patches); let data = ALPRDData::new(left_parts_dictionary, right_bit_width, left_parts_patches); Array::try_from_parts(ArrayParts::new(ALPRD, dtype, len, data).with_slots(slots)) @@ -404,13 +413,11 @@ impl ALPRDData { fn canonicalize_patches( left_parts: &ArrayRef, left_parts_patches: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult> { left_parts_patches .map(|patches| { - if !patches - .values() - .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? - { + if !patches.values().all_valid(ctx)? { vortex_bail!("patches must be all valid: {}", patches.values()); } // TODO(ngates): assert the DType, don't cast it. @@ -418,9 +425,8 @@ impl ALPRDData { let mut patches = patches.cast_values(&left_parts.dtype().as_nonnullable())?; // Force execution of the lazy cast so patch values are materialized // before serialization. - #[expect(deprecated)] - let canonical = patches.values().to_canonical()?.into_array(); - *patches.values_mut() = canonical; + let canonical = patches.values().clone().execute::(ctx)?; + *patches.values_mut() = canonical.into_array(); Ok(patches) }) .transpose() @@ -662,8 +668,8 @@ impl ValidityChild for ALPRD { mod test { use prost::Message; use rstest::rstest; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::PType; @@ -681,6 +687,7 @@ mod test { #[case] reals: Vec, #[case] seed: T, ) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_eq!(reals.len(), 1024, "test expects 1024-length fixture"); // Null out some of the values. let mut reals: Vec> = reals.into_iter().map(Some).collect(); @@ -694,10 +701,13 @@ mod test { // Pick a seed that we know will trigger lots of patches. let encoder: alp_rd::RDEncoder = alp_rd::RDEncoder::new(&[seed.powi(-2)]); - let rd_array = encoder.encode(real_array.as_view()); + let rd_array = encoder.encode(real_array.as_view(), &mut ctx); - #[expect(deprecated)] - let decoded = rd_array.as_array().to_primitive(); + let decoded = rd_array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(decoded, PrimitiveArray::from_option_iter(reals)); } diff --git a/encodings/alp/src/alp_rd/compute/cast.rs b/encodings/alp/src/alp_rd/compute/cast.rs index 69d3252ba89..9086877c2ce 100644 --- a/encodings/alp/src/alp_rd/compute/cast.rs +++ b/encodings/alp/src/alp_rd/compute/cast.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; @@ -28,6 +30,8 @@ impl CastReduce for ALPRD { .with_nullability(dtype.nullability()), )?; + // NOTE: `CastReduce::cast` has a fixed trait signature without `ExecutionCtx`, so we + // construct a legacy ctx locally at this trait boundary. return Ok(Some( ALPRD::try_new( dtype.clone(), @@ -36,6 +40,7 @@ impl CastReduce for ALPRD { array.right_parts().clone(), array.right_bit_width(), array.left_parts_patches(), + &mut LEGACY_SESSION.create_execution_ctx(), )? .into_array(), )); @@ -50,8 +55,8 @@ impl CastReduce for ALPRD { mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::compute::conformance::cast::test_cast_conformance; @@ -63,10 +68,11 @@ mod tests { #[test] fn test_cast_alprd_f32_to_f64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - let alprd = encoder.encode(arr.as_view()); + let alprd = encoder.encode(arr.as_view(), &mut ctx); let casted = alprd .into_array() @@ -77,8 +83,7 @@ mod tests { &DType::Primitive(PType::F64, Nullability::NonNullable) ); - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); let f64_values = decoded.as_slice::(); assert_eq!(f64_values.len(), 5); assert!((f64_values[0] - 1.0).abs() < f64::EPSILON); @@ -87,11 +92,12 @@ mod tests { #[test] fn test_cast_alprd_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = PrimitiveArray::from_option_iter([Some(10.0f64), None, Some(10.1), Some(10.2), None]); let values = vec![10.0f64, 10.1, 10.2]; let encoder = RDEncoder::new(&values); - let alprd = encoder.encode(arr.as_view()); + let alprd = encoder.encode(arr.as_view(), &mut ctx); // Cast to NonNullable should fail since we have nulls let result = alprd @@ -116,31 +122,31 @@ mod tests { let values = vec![1.23f32, 4.56, 7.89, 10.11, 12.13]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::f64({ let values = vec![100.1f64, 200.2, 300.3, 400.4, 500.5]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::single({ let values = vec![42.42f64]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::negative({ let values = vec![0.0f32, -1.5, 2.5, -3.5, 4.5]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::nullable({ let arr = PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]); let values = vec![1.1f32, 2.2, 3.3]; let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] fn test_cast_alprd_conformance(#[case] alprd: crate::alp_rd::ALPRDArray) { test_cast_conformance(&alprd.into_array()); diff --git a/encodings/alp/src/alp_rd/compute/filter.rs b/encodings/alp/src/alp_rd/compute/filter.rs index dfd82695933..5deaa7c466b 100644 --- a/encodings/alp/src/alp_rd/compute/filter.rs +++ b/encodings/alp/src/alp_rd/compute/filter.rs @@ -32,6 +32,7 @@ impl FilterKernel for ALPRD { array.right_parts().filter(mask.clone())?, array.right_bit_width(), left_parts_exceptions, + ctx, )? .into_array(), )) @@ -42,6 +43,8 @@ impl FilterKernel for ALPRD { mod test { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::filter::test_filter_conformance; @@ -57,8 +60,9 @@ mod test { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn test_filter(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::new(buffer![a, b, outlier], Validity::NonNullable); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); // Make sure that we're testing the exception pathway. assert!(encoded.left_parts_patches().is_some()); @@ -74,9 +78,13 @@ mod test { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn test_filter_simple(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_filter_conformance( &RDEncoder::new(&[a, b]) - .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) + .encode( + PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view(), + &mut ctx, + ) .into_array(), ); } @@ -85,11 +93,13 @@ mod test { #[case(0.1f32, 3e25f32)] #[case(0.5f64, 1e100f64)] fn test_filter_with_nulls(#[case] a: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_filter_conformance( &RDEncoder::new(&[a]) .encode( PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) .as_view(), + &mut ctx, ) .into_array(), ); diff --git a/encodings/alp/src/alp_rd/compute/mask.rs b/encodings/alp/src/alp_rd/compute/mask.rs index 2773387c738..47a2f2a6b67 100644 --- a/encodings/alp/src/alp_rd/compute/mask.rs +++ b/encodings/alp/src/alp_rd/compute/mask.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; @@ -20,6 +22,8 @@ impl MaskReduce for ALPRD { EmptyOptions, [array.left_parts().clone(), mask.clone()], )?; + // NOTE: `MaskReduce::mask` has a fixed trait signature without `ExecutionCtx`, so we + // construct a legacy ctx locally at this trait boundary. Ok(Some( ALPRD::try_new( array.dtype().as_nullable(), @@ -28,6 +32,7 @@ impl MaskReduce for ALPRD { array.right_parts().clone(), array.right_bit_width(), array.left_parts_patches(), + &mut LEGACY_SESSION.create_execution_ctx(), )? .into_array(), )) @@ -38,6 +43,8 @@ impl MaskReduce for ALPRD { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::mask::test_mask_conformance; @@ -48,9 +55,13 @@ mod tests { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn test_mask_simple(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_mask_conformance( &RDEncoder::new(&[a, b]) - .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) + .encode( + PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view(), + &mut ctx, + ) .into_array(), ); } @@ -59,11 +70,13 @@ mod tests { #[case(0.1f32, 3e25f32)] #[case(0.5f64, 1e100f64)] fn test_mask_with_nulls(#[case] a: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_mask_conformance( &RDEncoder::new(&[a]) .encode( PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) .as_view(), + &mut ctx, ) .into_array(), ); diff --git a/encodings/alp/src/alp_rd/compute/mod.rs b/encodings/alp/src/alp_rd/compute/mod.rs index 7984a9ce02a..7d03eddaf40 100644 --- a/encodings/alp/src/alp_rd/compute/mod.rs +++ b/encodings/alp/src/alp_rd/compute/mod.rs @@ -10,6 +10,8 @@ mod take; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -23,47 +25,47 @@ mod tests { let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::f64_array({ let values = vec![100.0f64, 100.01, 100.02, 100.03, 100.04]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] // Nullable arrays #[case::nullable_f32({ let values = vec![1.0f32, 1.2, 1.3]; let arr = PrimitiveArray::from_option_iter([Some(1.0f32), None, Some(1.2), Some(1.3), None]); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::nullable_f64({ let values = vec![10.0f64, 10.2, 10.3]; let arr = PrimitiveArray::from_option_iter([Some(10.0f64), None, Some(10.2), Some(10.3), None]); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] // Edge cases #[case::single_element({ let values = vec![42.42f64]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] // Arrays with small deltas (good for RD encoding) #[case::small_deltas({ let values = vec![1000.0f32, 1000.001, 1000.002, 1000.003, 1000.004]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] // Large arrays #[case::large_f32({ let values: Vec = (0..1000).map(|i| 100.0 + i as f32 * 0.01).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] fn test_alp_rd_consistency(#[case] array: ALPRDArray) { test_array_consistency(&array.into_array()); @@ -74,25 +76,25 @@ mod tests { let values = vec![1.0f32, 1.1, 1.2, 1.3, 1.4]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::f64_basic({ let values = vec![100.0f64, 100.01, 100.02, 100.03, 100.04]; let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::f32_large({ let values: Vec = (0..100).map(|i| 50.0 + i as f32 * 0.1).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] #[case::f64_large({ let values: Vec = (0..100).map(|i| 1000.0 + i as f64 * 0.01).collect(); let arr = PrimitiveArray::from_iter(values.clone()); let encoder = RDEncoder::new(&values); - encoder.encode(arr.as_view()) + encoder.encode(arr.as_view(), &mut LEGACY_SESSION.create_execution_ctx()) })] fn test_alp_rd_binary_numeric(#[case] array: ALPRDArray) { test_binary_numeric_array(array.into_array()); diff --git a/encodings/alp/src/alp_rd/compute/take.rs b/encodings/alp/src/alp_rd/compute/take.rs index fac46e74d02..0d980e0abfc 100644 --- a/encodings/alp/src/alp_rd/compute/take.rs +++ b/encodings/alp/src/alp_rd/compute/take.rs @@ -48,6 +48,7 @@ impl TakeExecute for ALPRD { right_parts, array.right_bit_width(), left_parts_exceptions, + ctx, )? .into_array(), )) @@ -58,8 +59,8 @@ impl TakeExecute for ALPRD { mod test { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::take::test_take_conformance; @@ -75,8 +76,9 @@ mod test { use vortex_array::IntoArray as _; use vortex_buffer::buffer; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); assert!(encoded.left_parts_patches().is_some()); assert!( @@ -87,11 +89,11 @@ mod test { .is_unsigned_int() ); - #[expect(deprecated)] let taken = encoded .take(buffer![0, 2].into_array()) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(taken, PrimitiveArray::from_iter([a, outlier])); } @@ -100,8 +102,9 @@ mod test { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn take_with_nulls(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); assert!(encoded.left_parts_patches().is_some()); assert!( @@ -112,11 +115,11 @@ mod test { .is_unsigned_int() ); - #[expect(deprecated)] let taken = encoded .take(PrimitiveArray::from_option_iter([Some(0), Some(2), None]).into_array()) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( taken, @@ -128,9 +131,13 @@ mod test { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn test_take_conformance_alprd(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_take_conformance( &RDEncoder::new(&[a, b]) - .encode(PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view()) + .encode( + PrimitiveArray::from_iter([a, b, outlier, b, outlier]).as_view(), + &mut ctx, + ) .into_array(), ); } @@ -139,11 +146,13 @@ mod test { #[case(0.1f32, 3e25f32)] #[case(0.5f64, 1e100f64)] fn test_take_with_nulls_conformance(#[case] a: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); test_take_conformance( &RDEncoder::new(&[a]) .encode( PrimitiveArray::from_option_iter([Some(a), None, Some(outlier), Some(a), None]) .as_view(), + &mut ctx, ) .into_array(), ); diff --git a/encodings/alp/src/alp_rd/mod.rs b/encodings/alp/src/alp_rd/mod.rs index 29356ad812f..5f2e703cc0a 100644 --- a/encodings/alp/src/alp_rd/mod.rs +++ b/encodings/alp/src/alp_rd/mod.rs @@ -182,11 +182,15 @@ impl RDEncoder { /// /// Each value will be split into a left and right component, which are compressed individually. // TODO(joe): make fallible - pub fn encode(&self, array: ArrayView<'_, Primitive>) -> ALPRDArray { - match_each_alp_float_ptype!(array.ptype(), |P| { self.encode_generic::

(array) }) + pub fn encode(&self, array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx) -> ALPRDArray { + match_each_alp_float_ptype!(array.ptype(), |P| { self.encode_generic::

(array, ctx) }) } - fn encode_generic(&self, array: ArrayView<'_, Primitive>) -> ALPRDArray + fn encode_generic( + &self, + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> ALPRDArray where T: ALPRDFloat + NativePType, T::UINT: NativePType, @@ -285,6 +289,7 @@ impl RDEncoder { packed_right, self.right_bit_width, exceptions, + ctx, ) .vortex_expect("ALPRDArray construction in encode") } diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index be30a3f7f03..0351d446424 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -66,6 +66,8 @@ impl OperationsVTable for ALPRD { #[cfg(test)] mod test { use rstest::rstest; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::scalar::Scalar; @@ -78,8 +80,9 @@ mod test { #[case(0.1f32, 0.2f32, 3e25f32)] #[case(0.1f64, 0.2f64, 3e100f64)] fn test_slice(#[case] a: T, #[case] b: T, #[case] outlier: T) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!(encoded, array); @@ -93,19 +96,21 @@ mod test { #[case] b: T, #[case] outlier: T, ) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([a, b, outlier]); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!(encoded, array); } #[test] fn nullable_scalar_at() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let a = 0.1f64; let b = 0.2f64; let outlier = 3e100f64; let array = PrimitiveArray::from_option_iter([Some(a), Some(b), Some(outlier)]); - let encoded = RDEncoder::new(&[a, b]).encode(array.as_view()); + let encoded = RDEncoder::new(&[a, b]).encode(array.as_view(), &mut ctx); assert!(encoded.left_parts_patches().is_some()); assert_arrays_eq!( encoded, diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index e5935cf15a8..f9d63d86197 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -4,7 +4,7 @@ pub struct vortex_datetime_parts::DateTimeParts impl vortex_datetime_parts::DateTimeParts -pub fn vortex_datetime_parts::DateTimeParts::try_from_temporal(temporal: vortex_array::arrays::datetime::TemporalArray) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimeParts::try_from_temporal(temporal: vortex_array::arrays::datetime::TemporalArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_datetime_parts::DateTimeParts::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult @@ -56,7 +56,7 @@ pub fn vortex_datetime_parts::DateTimeParts::validity_child(array: vortex_array: impl vortex_array::arrays::dict::take::TakeExecute for vortex_datetime_parts::DateTimeParts -pub fn vortex_datetime_parts::DateTimeParts::take(array: vortex_array::array::view::ArrayView<'_, Self>, indices: &vortex_array::array::erased::ArrayRef, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_datetime_parts::DateTimeParts::take(array: vortex_array::array::view::ArrayView<'_, Self>, indices: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::arrays::filter::kernel::FilterReduce for vortex_datetime_parts::DateTimeParts @@ -88,12 +88,6 @@ impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::clone(&self) -> vortex_datetime_parts::DateTimePartsData -impl core::convert::TryFrom for vortex_datetime_parts::DateTimePartsData - -pub type vortex_datetime_parts::DateTimePartsData::Error = vortex_error::VortexError - -pub fn vortex_datetime_parts::DateTimePartsData::try_from(array: vortex_array::arrays::datetime::TemporalArray) -> core::result::Result - impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -184,6 +178,6 @@ pub fn T::subseconds(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_datetime_parts::initialize(session: &vortex_session::VortexSession) -pub fn vortex_datetime_parts::split_temporal(array: vortex_array::arrays::datetime::TemporalArray) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::split_temporal(array: vortex_array::arrays::datetime::TemporalArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub type vortex_datetime_parts::DateTimePartsArray = vortex_array::array::typed::Array diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 67edc3dfc20..74a58b9418e 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -271,13 +271,16 @@ impl DateTimeParts { } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. - pub fn try_from_temporal(temporal: TemporalArray) -> VortexResult { + pub fn try_from_temporal( + temporal: TemporalArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let dtype = temporal.dtype().clone(); let TemporalParts { days, seconds, subseconds, - } = split_temporal(temporal)?; + } = split_temporal(temporal, ctx)?; Self::try_new(dtype, days, seconds, subseconds) } } diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index f1d1aaf6914..0314b669a12 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -140,13 +140,15 @@ mod test { ], validity.clone(), ); - let date_times = DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( - milliseconds.clone().into_array(), - TimeUnit::Milliseconds, - Some("UTC".into()), - ))?; - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let date_times = DateTimeParts::try_from_temporal( + TemporalArray::new_timestamp( + milliseconds.clone().into_array(), + TimeUnit::Milliseconds, + Some("UTC".into()), + ), + &mut ctx, + )?; assert!( date_times diff --git a/encodings/datetime-parts/src/compress.rs b/encodings/datetime-parts/src/compress.rs index bc25f0eb999..676e7bbfd43 100644 --- a/encodings/datetime-parts/src/compress.rs +++ b/encodings/datetime-parts/src/compress.rs @@ -2,19 +2,16 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_buffer::BufferMut; -use vortex_error::VortexError; use vortex_error::VortexResult; -use crate::DateTimePartsData; use crate::timestamp; pub struct TemporalParts { pub days: ArrayRef, @@ -26,12 +23,13 @@ pub struct TemporalParts { /// /// Splitting the components by granularity creates more small values, which enables better /// cascading compression. -pub fn split_temporal(array: TemporalArray) -> VortexResult { - #[expect(deprecated)] - let temporal_values = array.temporal_values().to_primitive(); +pub fn split_temporal(array: TemporalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let temporal_values = array + .temporal_values() + .clone() + .execute::(ctx)?; // After this operation, timestamps will be a PrimitiveArray - #[expect(deprecated)] let timestamps = temporal_values .clone() .into_array() @@ -39,7 +37,7 @@ pub fn split_temporal(array: TemporalArray) -> VortexResult { PType::I64, temporal_values.dtype().nullability(), ))? - .to_primitive(); + .execute::(ctx)?; let length = timestamps.len(); let mut days = BufferMut::with_capacity(length); @@ -60,34 +58,11 @@ pub fn split_temporal(array: TemporalArray) -> VortexResult { }) } -impl TryFrom for DateTimePartsData { - type Error = VortexError; - - fn try_from(array: TemporalArray) -> Result { - let ext_dtype = array.ext_dtype(); - let TemporalParts { - days, - seconds, - subseconds, - } = split_temporal(array)?; - DateTimePartsData::validate( - &DType::Extension(ext_dtype), - &days, - &seconds, - &subseconds, - days.len(), - )?; - Ok(DateTimePartsData {}) - } -} - #[cfg(test)] mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; @@ -105,6 +80,7 @@ mod tests { #[case(Validity::AllInvalid)] #[case(Validity::from_iter([true, false, true]))] fn test_split_temporal(#[case] validity: Validity) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let milliseconds = PrimitiveArray::new( buffer![ 86_400i64, // element with only day component @@ -120,11 +96,9 @@ mod tests { days, seconds, subseconds, - } = split_temporal(temporal_array).unwrap(); + } = split_temporal(temporal_array, &mut ctx).unwrap(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - #[expect(deprecated)] - let days_prim = days.to_primitive(); + let days_prim = days.execute::(&mut ctx).unwrap(); assert!( days_prim .validity() @@ -132,16 +106,14 @@ mod tests { .mask_eq(&validity, &mut ctx) .unwrap() ); - #[expect(deprecated)] - let seconds_prim = seconds.to_primitive(); + let seconds_prim = seconds.execute::(&mut ctx).unwrap(); assert!(matches!( seconds_prim .validity() .vortex_expect("seconds validity should be derivable"), Validity::NonNullable )); - #[expect(deprecated)] - let subseconds_prim = subseconds.to_primitive(); + let subseconds_prim = subseconds.execute::(&mut ctx).unwrap(); assert!(matches!( subseconds_prim .validity() diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index 573a0af48d7..3124d77a2d0 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -35,7 +35,10 @@ impl CastReduce for DateTimeParts { mod tests { use rstest::rstest; use vortex_array::ArrayRef; + use vortex_array::Canonical; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::builtins::ArrayBuiltins; @@ -49,19 +52,22 @@ mod tests { use crate::DateTimePartsArray; fn date_time_array(validity: Validity) -> ArrayRef { - DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( - PrimitiveArray::new( - buffer![ - 86_400i64, // element with only day component - 86_400i64 + 1000, // element with day + second components - 86_400i64 + 1000 + 1, // element with day + second + sub-second components - ], - validity, - ) - .into_array(), - TimeUnit::Milliseconds, - Some("UTC".into()), - )) + DateTimeParts::try_from_temporal( + TemporalArray::new_timestamp( + PrimitiveArray::new( + buffer![ + 86_400i64, // element with only day component + 86_400i64 + 1000, // element with day + second components + 86_400i64 + 1000 + 1, // element with day + second + sub-second components + ], + validity, + ) + .into_array(), + TimeUnit::Milliseconds, + Some("UTC".into()), + ), + &mut LEGACY_SESSION.create_execution_ctx(), + ) .unwrap() .into_array() } @@ -89,19 +95,18 @@ mod tests { #[case(Validity::AllInvalid)] #[case(Validity::from_iter([true, false, true]))] fn test_bad_cast_fails(#[case] validity: Validity) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = date_time_array(validity); - // Cast to incompatible type - force evaluation via to_canonical - #[expect(deprecated)] + // Cast to incompatible type - force evaluation via execute:: let result = array .cast(DType::Bool(Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err(), "Expected error, got: {result:?}"); - // Cast nullable with nulls to non-nullable - force evaluation via to_canonical - #[expect(deprecated)] + // Cast nullable with nulls to non-nullable - force evaluation via execute:: let result = array .cast(array.dtype().with_nullability(Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err(), "Expected error, got: {result:?}"); } @@ -116,7 +121,7 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), @@ -127,12 +132,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000_000_000i64].into_array(), // 1 day in ns TimeUnit::Nanoseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_cast_datetime_parts_conformance(#[case] array: DateTimePartsArray) { use vortex_array::compute::conformance::cast::test_cast_conformance; test_cast_conformance(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 3d2c76e81e3..bbeda194636 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -202,6 +202,8 @@ fn compare_dtp( #[cfg(test)] mod test { use rstest::rstest; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::dtype::IntegerPType; @@ -217,11 +219,14 @@ mod test { value: T, validity: Validity, ) -> DateTimePartsArray { - DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( - PrimitiveArray::new(buffer![value], validity).into_array(), - TimeUnit::Seconds, - Some("UTC".into()), - )) + DateTimeParts::try_from_temporal( + TemporalArray::new_timestamp( + PrimitiveArray::new(buffer![value], validity).into_array(), + TimeUnit::Seconds, + Some("UTC".into()), + ), + &mut LEGACY_SESSION.create_execution_ctx(), + ) .expect("Failed to construct DateTimePartsArray from TemporalArray") } diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index d717d3d3500..c0042806264 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -27,6 +27,8 @@ impl FilterReduce for DateTimeParts { #[cfg(test)] mod test { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::compute::conformance::filter::test_filter_conformance; @@ -37,6 +39,7 @@ mod test { #[test] fn test_filter_datetime_parts() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create temporal arrays and convert to DateTimePartsArray let timestamps = buffer![ 0i64, @@ -50,7 +53,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimeParts::try_from_temporal(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal, &mut ctx).unwrap(); test_filter_conformance(&array.into_array()); // Test with nullable values @@ -66,7 +69,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimeParts::try_from_temporal(temporal).unwrap(); + let array = DateTimeParts::try_from_temporal(temporal, &mut ctx).unwrap(); test_filter_conformance(&array.into_array()); } } diff --git a/encodings/datetime-parts/src/compute/mod.rs b/encodings/datetime-parts/src/compute/mod.rs index 53717a52b0c..5d25757c20d 100644 --- a/encodings/datetime-parts/src/compute/mod.rs +++ b/encodings/datetime-parts/src/compute/mod.rs @@ -15,6 +15,8 @@ mod take; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -25,7 +27,8 @@ mod tests { use crate::DateTimePartsArray; fn dtp_from_temporal(temporal: TemporalArray) -> DateTimePartsArray { - DateTimeParts::try_from_temporal(temporal).unwrap() + DateTimeParts::try_from_temporal(temporal, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() } #[rstest] diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index c18d959cd7e..533de399081 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -179,6 +179,8 @@ fn is_constant_zero(array: &ArrayRef) -> bool { #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; @@ -217,7 +219,7 @@ mod tests { time_unit, None, ); - DateTimeParts::try_from_temporal(temporal) + DateTimeParts::try_from_temporal(temporal, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("TemporalArray must produce valid DateTimeParts") } @@ -350,7 +352,9 @@ mod tests { TimeUnit::Seconds, None, ); - let dtp = DateTimeParts::try_from_temporal(temporal).unwrap(); + let dtp = + DateTimeParts::try_from_temporal(temporal, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let len = dtp.len(); // Compare against midnight constant diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index f5f67836e56..a9cb88b449c 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -5,8 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::TakeExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::Nullability; @@ -21,10 +20,10 @@ use crate::array::DateTimePartsArrayExt; fn take_datetime_parts( array: ArrayView, indices: &ArrayRef, + ctx: &mut ExecutionCtx, ) -> VortexResult { // we go ahead and canonicalize here to avoid worst-case canonicalizing 3 separate times - #[expect(deprecated)] - let indices = indices.to_primitive(); + let indices = indices.clone().execute::(ctx)?; let taken_days = array.days().take(indices.clone().into_array())?; let taken_seconds = array.seconds().take(indices.clone().into_array())?; @@ -87,9 +86,9 @@ impl TakeExecute for DateTimeParts { fn take( array: ArrayView<'_, Self>, indices: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { - take_datetime_parts(array, indices).map(Some) + take_datetime_parts(array, indices, ctx).map(Some) } } @@ -97,6 +96,8 @@ impl TakeExecute for DateTimeParts { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::compute::conformance::take::test_take_conformance; @@ -117,7 +118,7 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), @@ -128,12 +129,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( buffer![86_400_000i64].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + ), &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_take_datetime_parts_conformance(#[case] array: DateTimePartsArray) { test_take_conformance(&array.into_array()); } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index 30d7bcd9a5f..b313a165b79 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -46,9 +46,11 @@ impl CastReduce for DecimalByteParts { #[cfg(test)] mod tests { use rstest::rstest; + use vortex_array::Canonical; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::compute::conformance::cast::test_cast_conformance; @@ -62,6 +64,7 @@ mod tests { #[test] fn test_cast_decimal_byte_parts_nullability() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decimal_dtype = DecimalDType::new(10, 2); let array = DecimalByteParts::try_new(buffer![100i32, 200, 300, 400].into_array(), decimal_dtype) @@ -78,13 +81,13 @@ mod tests { ); // Verify the values are preserved - #[expect(deprecated)] - let decoded = casted.to_decimal(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_eq!(decoded.len(), 4); } #[test] fn test_cast_decimal_byte_parts_nullable_to_non_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decimal_dtype = DecimalDType::new(10, 2); let array = DecimalByteParts::try_new( PrimitiveArray::from_option_iter([Some(100i32), None, Some(300)]).into_array(), @@ -92,12 +95,11 @@ mod tests { ) .unwrap(); - // Cast to non-nullable should fail due to nulls - force evaluation via to_canonical - #[expect(deprecated)] + // Cast to non-nullable should fail due to nulls - force evaluation via execute:: let result = array .into_array() .cast(DType::Decimal(decimal_dtype, Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); assert!(result.is_err()); } diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index 9298a426d99..a5a7803bcaf 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -28,7 +28,9 @@ fn main() { fn take_10_stratified(bencher: Bencher) { let values = fixture(65_536, 8); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0..10).map(|i| i * 6_553)); bencher @@ -46,7 +48,9 @@ fn take_10_stratified(bencher: Bencher) { fn take_10_contiguous(bencher: Bencher) { let values = fixture(65_536, 8); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = buffer![0..10].into_array(); bencher @@ -65,7 +69,9 @@ fn take_10k_random(bencher: Bencher) { let values = fixture(65_536, 8); let range = Uniform::new(0, values.len()).unwrap(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let rng = StdRng::seed_from_u64(0); let indices = PrimitiveArray::from_iter(rng.sample_iter(range).take(10_000).map(|i| i as u32)); @@ -85,7 +91,9 @@ fn take_10k_random(bencher: Bencher) { fn take_10k_contiguous(bencher: Bencher) { let values = fixture(65_536, 8); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter(0..10_000); bencher @@ -103,7 +111,9 @@ fn take_10k_contiguous(bencher: Bencher) { fn take_10k_dispersed(bencher: Bencher) { let values = fixture(65_536, 8); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0..10_000).map(|i| (i * 42) % values.len() as u64)); bencher @@ -121,7 +131,9 @@ fn take_10k_dispersed(bencher: Bencher) { fn take_10k_first_chunk_only(bencher: Bencher) { let values = fixture(65_536, 8); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0..10_000).map(|i| ((i * 42) % 1024) as u64)); bencher @@ -159,7 +171,9 @@ const NUM_EXCEPTIONS: u32 = 1024; fn patched_take_10_stratified(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(packed.patches().is_some()); assert_eq!( @@ -184,7 +198,9 @@ fn patched_take_10_stratified(bencher: Bencher) { fn patched_take_10_contiguous(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(packed.patches().is_some()); assert_eq!( @@ -209,7 +225,9 @@ fn patched_take_10_contiguous(bencher: Bencher) { fn patched_take_10k_random(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let rng = StdRng::seed_from_u64(0); let range = Uniform::new(0, values.len()).unwrap(); @@ -230,7 +248,9 @@ fn patched_take_10k_random(bencher: Bencher) { fn patched_take_10k_contiguous_not_patches(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0u32..NUM_EXCEPTIONS).cycle().take(10000)); bencher @@ -248,7 +268,9 @@ fn patched_take_10k_contiguous_not_patches(bencher: Bencher) { fn patched_take_10k_contiguous_patches(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); assert!(packed.patches().is_some()); assert_eq!( @@ -274,7 +296,9 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) { fn patched_take_10k_dispersed(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0..10_000).map(|i| (i * 42) % values.len() as u64)); bencher @@ -292,7 +316,9 @@ fn patched_take_10k_dispersed(bencher: Bencher) { fn patched_take_10k_first_chunk_only(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let indices = PrimitiveArray::from_iter((0..10_000).map(|i| ((i * 42) % 1024) as u64)); bencher @@ -310,7 +336,9 @@ fn patched_take_10k_first_chunk_only(bencher: Bencher) { fn patched_take_10k_adversarial(bencher: Bencher) { let values = (0u32..BIG_BASE2 + NUM_EXCEPTIONS).collect::>(); let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); - let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); + let packed = + bitpack_to_best_bit_width(&uncompressed, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let per_chunk_count = 100; let indices = PrimitiveArray::from_iter( (0..(NUM_EXCEPTIONS + 1024) / 1024) diff --git a/encodings/fastlanes/benches/canonicalize_bench.rs b/encodings/fastlanes/benches/canonicalize_bench.rs index bbcdf6a72c1..7b0ee2453f5 100644 --- a/encodings/fastlanes/benches/canonicalize_bench.rs +++ b/encodings/fastlanes/benches/canonicalize_bench.rs @@ -48,7 +48,14 @@ fn into_canonical_non_nullable( let chunks = (0..chunk_count) .map(|_| { - make_array(&mut rng, chunk_len, fraction_patched, 0.0).vortex_expect("make_array works") + make_array( + &mut rng, + chunk_len, + fraction_patched, + 0.0, + &mut SESSION.create_execution_ctx(), + ) + .vortex_expect("make_array works") }) .collect::>(); @@ -72,7 +79,14 @@ fn canonical_into_non_nullable( let chunks = (0..chunk_count) .map(|_| { - make_array(&mut rng, chunk_len, fraction_patched, 0.0).vortex_expect("make_array works") + make_array( + &mut rng, + chunk_len, + fraction_patched, + 0.0, + &mut SESSION.create_execution_ctx(), + ) + .vortex_expect("make_array works") }) .collect::>(); @@ -113,8 +127,14 @@ fn into_canonical_nullable( let chunks = (0..chunk_count) .map(|_| { - make_array(&mut rng, chunk_len, fraction_patched, 0.05) - .vortex_expect("make_array works") + make_array( + &mut rng, + chunk_len, + fraction_patched, + 0.05, + &mut SESSION.create_execution_ctx(), + ) + .vortex_expect("make_array works") }) .collect::>(); @@ -138,8 +158,14 @@ fn canonical_into_nullable( let chunks = (0..chunk_count) .map(|_| { - make_array(&mut rng, chunk_len, fraction_patched, 0.05) - .vortex_expect("make_array works") + make_array( + &mut rng, + chunk_len, + fraction_patched, + 0.05, + &mut SESSION.create_execution_ctx(), + ) + .vortex_expect("make_array works") }) .collect::>(); diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index d43a2d91639..945dd9f0d5c 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -12,8 +12,6 @@ use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; @@ -41,7 +39,9 @@ fn generate_bit_pack_primitive_array( .map(|_| T::from_usize(rng.random_range(0..10_000)).vortex_expect("")) .collect::(); - bitpack_to_best_bit_width(&a).vortex_expect("").into_array() + bitpack_to_best_bit_width(&a, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("") + .into_array() } fn generate_alp_bit_pack_primitive_array( @@ -52,17 +52,16 @@ fn generate_alp_bit_pack_primitive_array( .map(|_| T::from_usize(rng.random_range(0..10_000)).vortex_expect("")) .collect::(); - let alp = alp_encode( - a.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect(""); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let alp = alp_encode(a.as_view(), None, &mut ctx).vortex_expect(""); - #[expect(deprecated)] - let encoded = alp.encoded().to_primitive(); + let encoded = alp + .encoded() + .clone() + .execute::(&mut ctx) + .vortex_expect(""); - let bp = bitpack_to_best_bit_width(&encoded) + let bp = bitpack_to_best_bit_width(&encoded, &mut ctx) .vortex_expect("") .into_array(); ALP::new(bp, alp.exponents(), None).into_array() diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 4f8e8f67cfe..59dfbb4ea4f 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -16,21 +16,21 @@ pub fn vortex_fastlanes::bit_transpose::untranspose_validity(validity: &vortex_a pub mod vortex_fastlanes::bitpack_compress -pub fn vortex_fastlanes::bitpack_compress::bit_width_histogram(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::bitpack_compress::bit_width_histogram(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::bitpack_compress::bitpack_encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8, bit_width_freq: core::option::Option<&[usize]>) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_compress::bitpack_encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8, bit_width_freq: core::option::Option<&[usize]>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub unsafe fn vortex_fastlanes::bitpack_compress::bitpack_encode_unchecked(array: vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8) -> vortex_error::VortexResult pub fn vortex_fastlanes::bitpack_compress::bitpack_primitive(array: &[T], bit_width: u8) -> vortex_buffer::buffer::Buffer -pub fn vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub unsafe fn vortex_fastlanes::bitpack_compress::bitpack_unchecked(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8) -> vortex_buffer::ByteBuffer pub fn vortex_fastlanes::bitpack_compress::find_best_bit_width(ptype: vortex_array::dtype::ptype::PType, bit_width_freq: &[usize]) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_compress::gather_patches(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8, num_exceptions_hint: usize) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::bitpack_compress::gather_patches(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, bit_width: u8, num_exceptions_hint: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> pub mod vortex_fastlanes::bitpack_decompress @@ -120,7 +120,7 @@ pub struct vortex_fastlanes::BitPacked impl vortex_fastlanes::BitPacked -pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedDataParts @@ -196,7 +196,7 @@ impl vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 -pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize @@ -468,7 +468,7 @@ pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray @@ -522,7 +522,7 @@ pub fn vortex_fastlanes::RLE::validity(array: vortex_array::array::view::ArrayVi impl vortex_array::arrays::slice::SliceKernel for vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::RLE::slice(array: vortex_array::array::view::ArrayView<'_, Self>, range: core::ops::range::Range, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes::RLE @@ -532,7 +532,7 @@ pub struct vortex_fastlanes::RLEData impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLEData::encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_fastlanes::RLEData @@ -638,7 +638,7 @@ pub fn vortex_fastlanes::RLEArrayExt::offset(&self) -> usize pub fn vortex_fastlanes::RLEArrayExt::values(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_fastlanes::RLEArrayExt::values_idx_offset(&self, chunk_idx: usize) -> usize +pub fn vortex_fastlanes::RLEArrayExt::values_idx_offset(&self, chunk_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> usize pub fn vortex_fastlanes::RLEArrayExt::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef @@ -650,7 +650,7 @@ pub fn T::offset(&self) -> usize pub fn T::values(&self) -> &vortex_array::array::erased::ArrayRef -pub fn T::values_idx_offset(&self, chunk_idx: usize) -> usize +pub fn T::values_idx_offset(&self, chunk_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> usize pub fn T::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 537e7c07a94..624280ddb75 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -5,9 +5,8 @@ use fastlanes::BitPacking; use itertools::Itertools; use num_traits::PrimInt; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -32,10 +31,13 @@ use crate::BitPacked; use crate::BitPackedArray; use crate::bitpack_decompress; -pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult { - let bit_width_freq = bit_width_histogram(array.as_view())?; +pub fn bitpack_to_best_bit_width( + array: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let bit_width_freq = bit_width_histogram(array.as_view(), ctx)?; let best_bit_width = find_best_bit_width(array.ptype(), &bit_width_freq)?; - bitpack_encode(array, best_bit_width, Some(&bit_width_freq)) + bitpack_encode(array, best_bit_width, Some(&bit_width_freq), ctx) } #[expect(unused_comparisons, clippy::absurd_extreme_comparisons)] @@ -43,20 +45,17 @@ pub fn bitpack_encode( array: &PrimitiveArray, bit_width: u8, bit_width_freq: Option<&[usize]>, + ctx: &mut ExecutionCtx, ) -> VortexResult { let bit_width_freq = match bit_width_freq { Some(freq) => freq, - None => &bit_width_histogram(array.as_view())?, + None => &bit_width_histogram(array.as_view(), ctx)?, }; // Check array contains no negative values. if array.ptype().is_signed_int() { let has_negative_values = match_each_integer_ptype!(array.ptype(), |P| { - array - .statistics() - .compute_min::

(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap_or_default() - < 0 + array.statistics().compute_min::

(ctx).unwrap_or_default() < 0 }); if has_negative_values { vortex_bail!(InvalidArgument: "cannot bitpack_encode array containing negative integers") @@ -76,7 +75,7 @@ pub fn bitpack_encode( // SAFETY: we check that array only contains non-negative values. let packed = unsafe { bitpack_unchecked(array, bit_width) }; let patches = (num_exceptions > 0) - .then(|| gather_patches(array, bit_width, num_exceptions)) + .then(|| gather_patches(array, bit_width, num_exceptions, ctx)) .transpose()? .flatten(); @@ -199,6 +198,7 @@ pub fn gather_patches( parray: &PrimitiveArray, bit_width: u8, num_exceptions_hint: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let patch_validity = match parray.validity()? { Validity::NonNullable => Validity::NonNullable, @@ -206,10 +206,7 @@ pub fn gather_patches( }; let array_len = parray.len(); - let validity_mask = parray - .as_ref() - .validity()? - .to_mask(parray.len(), &mut LEGACY_SESSION.create_execution_ctx())?; + let validity_mask = parray.as_ref().validity()?.to_mask(parray.len(), ctx)?; let patches = if array_len < u8::MAX as usize { match_each_integer_ptype!(parray.ptype(), |T| { @@ -300,12 +297,18 @@ where } } -pub fn bit_width_histogram(array: ArrayView<'_, Primitive>) -> VortexResult> { - match_each_integer_ptype!(array.ptype(), |P| { bit_width_histogram_typed::

(array) }) +pub fn bit_width_histogram( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + match_each_integer_ptype!(array.ptype(), |P| { + bit_width_histogram_typed::

(array, ctx) + }) } fn bit_width_histogram_typed( array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let bit_width: fn(T) -> usize = |v: T| (8 * size_of::()) - (PrimInt::leading_zeros(v) as usize); @@ -313,10 +316,7 @@ fn bit_width_histogram_typed( let mut bit_widths = vec![0usize; size_of::() * 8 + 1]; match array .validity()? - .to_mask( - array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(array.as_ref().len(), ctx)? .bit_buffer() { AllOr::All => { @@ -388,9 +388,8 @@ pub mod test_harness { use rand::RngExt; use rand::rngs::StdRng; use vortex_array::ArrayRef; + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; @@ -403,6 +402,7 @@ pub mod test_harness { len: usize, fraction_patches: f64, fraction_null: f64, + ctx: &mut ExecutionCtx, ) -> VortexResult { let values = (0..len) .map(|_| { @@ -414,15 +414,14 @@ pub mod test_harness { }) .collect::>(); - #[expect(deprecated)] let values = if fraction_null == 0.0 { - values.into_array().to_primitive() + values.into_array().execute::(ctx)? } else { let validity = Validity::from_iter((0..len).map(|_| !rng.random_bool(fraction_null))); PrimitiveArray::new(values, validity) }; - bitpack_encode(&values, 12, None).map(|a| a.into_array()) + bitpack_encode(&values, 12, None, ctx).map(|a| a.into_array()) } } @@ -432,8 +431,6 @@ mod test { use rand::SeedableRng; use rand::rngs::StdRng; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::assert_arrays_eq; @@ -465,13 +462,14 @@ mod test { #[test] fn null_patches() { + let mut ctx = SESSION.create_execution_ctx(); let valid_values = (0..24).map(|v| v < 1 << 4).collect::>(); let values = PrimitiveArray::new( (0u32..24).collect::>(), Validity::from_iter(valid_values), ); assert!(values.ptype().is_unsigned_int()); - let compressed = BitPackedData::encode(&values.into_array(), 4).unwrap(); + let compressed = BitPackedData::encode(&values.into_array(), 4, &mut ctx).unwrap(); assert!(compressed.patches().is_none()); assert_eq!( (0..(1 << 4)).collect::>(), @@ -479,10 +477,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( - compressed.as_ref().len(), - &mut SESSION.create_execution_ctx() - ) + .to_mask(compressed.as_ref().len(), &mut ctx) .unwrap() .to_bit_buffer() .set_indices() @@ -492,28 +487,30 @@ mod test { #[test] fn compress_signed_fails() { + let mut ctx = SESSION.create_execution_ctx(); let values: Buffer = (-500..500).collect(); let array = PrimitiveArray::new(values, Validity::AllValid); assert!(array.ptype().is_signed_int()); - let err = BitPackedData::encode(&array.into_array(), 1024u32.ilog2() as u8).unwrap_err(); + let err = BitPackedData::encode(&array.into_array(), 1024u32.ilog2() as u8, &mut ctx) + .unwrap_err(); assert!(matches!(err, VortexError::InvalidArgument(_, _))); } #[test] fn canonicalize_chunked_of_bitpacked() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let mut rng = StdRng::seed_from_u64(0); let chunks = (0..10) - .map(|_| make_array(&mut rng, 100, 0.25, 0.25).unwrap()) + .map(|_| make_array(&mut rng, 100, 0.25, 0.25, &mut ctx).unwrap()) .collect::>(); let chunked = ChunkedArray::from_iter(chunks).into_array(); - #[expect(deprecated)] - let into_ca = chunked.to_primitive(); + let into_ca = chunked.clone().execute::(&mut ctx)?; let mut primitive_builder = PrimitiveBuilder::::with_capacity(chunked.dtype().nullability(), 10 * 100); - chunked.append_to_builder(&mut primitive_builder, &mut SESSION.create_execution_ctx())?; + chunked.append_to_builder(&mut primitive_builder, &mut ctx)?; let ca_into = primitive_builder.finish(); assert_arrays_eq!(into_ca, ca_into); @@ -529,7 +526,8 @@ mod test { } #[test] - fn test_chunk_offsets() { + fn test_chunk_offsets() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let patch_value = 1u32 << 20; let patch_indices = [100usize, 200, 3000, 3100]; let mut values = vec![0u32; 4096usize]; @@ -539,21 +537,27 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); let patches = bitpacked.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .as_ref() + .unwrap() + .clone() + .execute::(&mut ctx)?; // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 // chunk 1 (1024-2047): no patches -> points to patch index 2 // chunk 2 (2048-3071): patch at 3000 -> starts at patch index 2 // chunk 3 (3072-4095): patch at 3100 -> starts at patch index 3 assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2, 3])); + Ok(()) } #[test] - fn test_chunk_offsets_no_patches_in_middle() { + fn test_chunk_offsets_no_patches_in_middle() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let patch_value = 1u32 << 20; let patch_indices = [100usize, 200, 2500]; let mut values = vec![0u32; 3072usize]; @@ -563,17 +567,23 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); let patches = bitpacked.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .as_ref() + .unwrap() + .clone() + .execute::(&mut ctx)?; assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); + Ok(()) } #[test] - fn test_chunk_offsets_trailing_empty_chunks() { + fn test_chunk_offsets_trailing_empty_chunks() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let patch_value = 1u32 << 20; let patch_indices = [100usize, 200, 1500]; let mut values = vec![0u32; 5120usize]; @@ -583,11 +593,15 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); let patches = bitpacked.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .as_ref() + .unwrap() + .clone() + .execute::(&mut ctx)?; // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 // chunk 1 (1024-2047): patch at 1500 -> starts at patch index 2 @@ -595,10 +609,12 @@ mod test { // chunk 3 (3072-4095): no patches -> points to patch index 3 (remaining chunks filled) // chunk 4 (4096-5119): no patches -> points to patch index 3 (remaining chunks filled) assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 3, 3, 3])); + Ok(()) } #[test] - fn test_chunk_offsets_single_chunk() { + fn test_chunk_offsets_single_chunk() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let patch_value = 1u32 << 20; let patch_indices = [100usize, 200]; let mut values = vec![0u32; 500usize]; @@ -608,13 +624,18 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); let patches = bitpacked.patches().unwrap(); - #[expect(deprecated)] - let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); + let chunk_offsets = patches + .chunk_offsets() + .as_ref() + .unwrap() + .clone() + .execute::(&mut ctx)?; // Single chunk starting at patch index 0. assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64])); + Ok(()) } } diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 4ddbd1832cc..1bd3e388672 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -160,8 +160,6 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; @@ -178,7 +176,7 @@ mod tests { use crate::bitpack_compress::bitpack_encode; fn encode(array: &PrimitiveArray, bit_width: u8) -> BitPackedArray { - bitpack_encode(array, bit_width, None).unwrap() + bitpack_encode(array, bit_width, None, &mut SESSION.create_execution_ctx()).unwrap() } static SESSION: LazyLock = @@ -189,8 +187,9 @@ mod tests { } fn compression_roundtrip(n: usize) { + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter((0..n).map(|i| (i % 2047) as u16)); - let compressed = BitPackedData::encode(&values.clone().into_array(), 11).unwrap(); + let compressed = BitPackedData::encode(&values.clone().into_array(), 11, &mut ctx).unwrap(); assert_arrays_eq!(compressed, values); values @@ -220,8 +219,10 @@ mod tests { #[test] fn test_all_zeros() -> VortexResult<()> { - #[expect(deprecated)] - let zeros = buffer![0u16, 0, 0, 0].into_array().to_primitive(); + let mut ctx = SESSION.create_execution_ctx(); + let zeros = buffer![0u16, 0, 0, 0] + .into_array() + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 0); let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 0, 0, 0])); @@ -230,8 +231,10 @@ mod tests { #[test] fn test_simple_patches() -> VortexResult<()> { - #[expect(deprecated)] - let zeros = buffer![0u16, 1, 0, 1].into_array().to_primitive(); + let mut ctx = SESSION.create_execution_ctx(); + let zeros = buffer![0u16, 1, 0, 1] + .into_array() + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 0); let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 1, 0, 1])); @@ -240,8 +243,10 @@ mod tests { #[test] fn test_one_full_chunk() -> VortexResult<()> { - #[expect(deprecated)] - let zeros = BufferMut::from_iter(0u16..1024).into_array().to_primitive(); + let mut ctx = SESSION.create_execution_ctx(); + let zeros = BufferMut::from_iter(0u16..1024) + .into_array() + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 10); let actual = unpack(&bitpacked)?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1024)); @@ -250,10 +255,10 @@ mod tests { #[test] fn test_three_full_chunks_with_patches() -> VortexResult<()> { - #[expect(deprecated)] + let mut ctx = SESSION.create_execution_ctx(); let zeros = BufferMut::from_iter((5u16..1029).chain(5u16..1029).chain(5u16..1029)) .into_array() - .to_primitive(); + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 10); assert!(bitpacked.patches().is_some()); let actual = unpack(&bitpacked)?; @@ -266,8 +271,10 @@ mod tests { #[test] fn test_one_full_chunk_and_one_short_chunk_no_patch() -> VortexResult<()> { - #[expect(deprecated)] - let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); + let mut ctx = SESSION.create_execution_ctx(); + let zeros = BufferMut::from_iter(0u16..1025) + .into_array() + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 11); assert!(bitpacked.patches().is_none()); let actual = unpack(&bitpacked)?; @@ -277,10 +284,10 @@ mod tests { #[test] fn test_one_full_chunk_and_one_short_chunk_with_patches() -> VortexResult<()> { - #[expect(deprecated)] + let mut ctx = SESSION.create_execution_ctx(); let zeros = BufferMut::from_iter(512u16..1537) .into_array() - .to_primitive(); + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); assert!(bitpacked.patches().is_some()); @@ -291,42 +298,36 @@ mod tests { #[test] fn test_offset_and_short_chunk_and_patches() -> VortexResult<()> { - #[expect(deprecated)] + let mut ctx = SESSION.create_execution_ctx(); let zeros = BufferMut::from_iter(512u16..1537) .into_array() - .to_primitive(); + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); assert!(bitpacked.patches().is_some()); let slice_ref = bitpacked.into_array().slice(1023..1025).unwrap(); - let actual = { - let mut ctx = SESSION.create_execution_ctx(); - slice_ref - .execute::(&mut ctx) - .unwrap() - .into_primitive() - }; + let actual = slice_ref + .execute::(&mut ctx) + .unwrap() + .into_primitive(); assert_arrays_eq!(actual, PrimitiveArray::from_iter([1535u16, 1536])); Ok(()) } #[test] fn test_offset_and_short_chunk_with_chunks_between_and_patches() -> VortexResult<()> { - #[expect(deprecated)] + let mut ctx = SESSION.create_execution_ctx(); let zeros = BufferMut::from_iter(512u16..2741) .into_array() - .to_primitive(); + .execute::(&mut ctx)?; let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 2229); assert!(bitpacked.patches().is_some()); let slice_ref = bitpacked.into_array().slice(1023..2049).unwrap(); - let actual = { - let mut ctx = SESSION.create_execution_ctx(); - slice_ref - .execute::(&mut ctx) - .unwrap() - .into_primitive() - }; + let actual = slice_ref + .execute::(&mut ctx) + .unwrap() + .into_primitive(); assert_arrays_eq!( actual, PrimitiveArray::from_iter((1023u16..2049).map(|x| x + 512)) diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index ba5029260a7..9cadc8ccf93 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -6,6 +6,7 @@ use std::fmt::Formatter; use fastlanes::BitPacking; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::TypedArrayRef; use vortex_array::array_slots; use vortex_array::arrays::Primitive; @@ -254,12 +255,16 @@ impl BitPackedData { /// /// If the requested bit-width for packing is larger than the array's native width, an /// error will be returned. - pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { + pub fn encode( + array: &ArrayRef, + bit_width: u8, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let parray: PrimitiveArray = array .clone() .try_downcast::() .map_err(|a| vortex_err!(InvalidArgument: "Bitpacking can only encode primitive arrays, got {}", a.encoding_id()))?; - bitpack_encode(&parray, bit_width, None) + bitpack_encode(&parray, bit_width, None, ctx) } /// Calculate the maximum value that **can** be contained by this array, given its bit-width. @@ -333,8 +338,8 @@ impl> BitPackedArrayExt for T {} #[cfg(test)] mod test { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_buffer::Buffer; @@ -344,6 +349,7 @@ mod test { #[test] fn test_encode() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = [ Some(1u64), None, @@ -354,32 +360,40 @@ mod test { Some(u64::MAX), ]; let uncompressed = PrimitiveArray::from_option_iter(values); - let packed = BitPackedData::encode(&uncompressed.into_array(), 1).unwrap(); + let packed = BitPackedData::encode(&uncompressed.into_array(), 1, &mut ctx).unwrap(); let expected = PrimitiveArray::from_option_iter(values); - #[expect(deprecated)] - let packed_primitive = packed.as_array().to_primitive(); + let packed_primitive = packed + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(packed_primitive, expected); } #[test] fn test_encode_too_wide() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = [Some(1u8), None, Some(1), None, Some(1), None]; let uncompressed = PrimitiveArray::from_option_iter(values); - let _packed = BitPackedData::encode(&uncompressed.clone().into_array(), 8) + let _packed = BitPackedData::encode(&uncompressed.clone().into_array(), 8, &mut ctx) .expect_err("Cannot pack value into the same width"); - let _packed = BitPackedData::encode(&uncompressed.into_array(), 9) + let _packed = BitPackedData::encode(&uncompressed.into_array(), 9, &mut ctx) .expect_err("Cannot pack value into larger width"); } #[test] fn signed_with_patches() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = (0i32..=512).collect(); let parray = values.clone().into_array(); - let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); + let packed_with_patches = BitPackedData::encode(&parray, 9, &mut ctx).unwrap(); assert!(packed_with_patches.patches().is_some()); - #[expect(deprecated)] - let packed_primitive = packed_with_patches.as_array().to_primitive(); + let packed_primitive = packed_with_patches + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( packed_primitive, PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 80a73f82678..2f7187d26f1 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -55,12 +55,14 @@ impl> UnpackStrategy for BitPackingStr /// #[gat(Item)] /// use lending_iterator::prelude::LendingIterator; /// use vortex_array::IntoArray; +/// use vortex_array::VortexSessionExecute; /// use vortex_buffer::buffer; /// use vortex_fastlanes::BitPackedData; /// use vortex_fastlanes::BitPackedArrayExt; /// use vortex_fastlanes::unpack_iter::BitUnpackedChunks; /// -/// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2).unwrap(); +/// let mut ctx = vortex_array::LEGACY_SESSION.create_execution_ctx(); +/// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2, &mut ctx).unwrap(); /// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks().unwrap(); /// /// if let Some(header) = unpacked_chunks.initial() { diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index fbcd48ce75b..8843c8e3013 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -53,6 +53,8 @@ mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -66,7 +68,7 @@ mod tests { use crate::BitPackedData; fn bp(array: &ArrayRef, bit_width: u8) -> BitPackedArray { - BitPackedData::encode(array, bit_width).unwrap() + BitPackedData::encode(array, bit_width, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } #[test] diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 1b161889560..aab597e9ff3 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -178,8 +178,8 @@ fn filter_with_indices( #[cfg(test)] mod test { use vortex_array::IntoArray as _; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::filter::test_filter_conformance; @@ -193,9 +193,10 @@ mod test { #[test] fn take_indices() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a u8 array modulo 63. let unpacked = PrimitiveArray::from_iter((0..4096).map(|i| (i % 63) as u8)); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6, &mut ctx).unwrap(); let mask = Mask::from_indices(bitpacked.len(), vec![0, 125, 2047, 2049, 2151, 2790]); @@ -208,9 +209,10 @@ mod test { #[test] fn take_sliced_indices() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a u8 array modulo 63. let unpacked = PrimitiveArray::from_iter((0..4096).map(|i| (i % 63) as u8)); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6, &mut ctx).unwrap(); let sliced = bitpacked.slice(128..2050).unwrap(); let mask = Mask::from_indices(sliced.len(), vec![1919, 1921]); @@ -221,13 +223,13 @@ mod test { #[test] fn filter_bitpacked() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unpacked = PrimitiveArray::from_iter((0..4096).map(|i| (i % 63) as u8)); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6, &mut ctx).unwrap(); let filtered = bitpacked .filter(Mask::from_indices(4096, (0..1024).collect())) .unwrap(); - #[expect(deprecated)] - let filtered_prim = filtered.to_primitive(); + let filtered_prim = filtered.execute::(&mut ctx).unwrap(); assert_arrays_eq!( filtered_prim, PrimitiveArray::from_iter((0..1024).map(|i| (i % 63) as u8)) @@ -236,14 +238,15 @@ mod test { #[test] fn filter_bitpacked_signed() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = (0..500).collect(); let unpacked = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 9).unwrap(); - #[expect(deprecated)] + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 9, &mut ctx).unwrap(); let filtered = bitpacked .filter(Mask::from_indices(values.len(), (0..250).collect())) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!( filtered, @@ -253,19 +256,20 @@ mod test { #[test] fn test_filter_bitpacked_conformance() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test with u8 values let unpacked = buffer![1u8, 2, 3, 4, 5].into_array(); - let bitpacked = BitPackedData::encode(&unpacked, 3).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked, 3, &mut ctx).unwrap(); test_filter_conformance(&bitpacked.into_array()); // Test with u32 values let unpacked = buffer![100u32, 200, 300, 400, 500].into_array(); - let bitpacked = BitPackedData::encode(&unpacked, 9).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked, 9, &mut ctx).unwrap(); test_filter_conformance(&bitpacked.into_array()); // Test with nullable values let unpacked = PrimitiveArray::from_option_iter([Some(1u16), None, Some(3), Some(4), None]); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 3).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 3, &mut ctx).unwrap(); test_filter_conformance(&bitpacked.into_array()); } @@ -276,22 +280,23 @@ mod test { /// This test ensures that the type handling is correct. #[test] fn filter_bitpacked_signed_with_patches() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create signed integer values where some exceed the bit width (causing patches). // Values 0-127 fit in 7 bits, but 1000 and 2000 do not. let values: Vec = vec![0, 10, 1000, 20, 30, 2000, 40, 50, 60, 70]; let unpacked = PrimitiveArray::from_iter(values.clone()); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7, &mut ctx).unwrap(); assert!( bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" ); // Filter to include some patched and some non-patched values. - #[expect(deprecated)] let filtered = bitpacked .filter(Mask::from_indices(values.len(), vec![0, 2, 5, 9])) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(filtered, PrimitiveArray::from_iter([0i32, 1000, 2000, 70])); } @@ -302,6 +307,7 @@ mod test { /// that doesn't fully decompress the array first. #[test] fn filter_bitpacked_signed_with_patches_low_selectivity() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a larger array with signed integers and some patches. let values: Vec = (0..1000) .map(|i| { @@ -313,7 +319,7 @@ mod test { }) .collect(); let unpacked = PrimitiveArray::from_iter(values.clone()); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7, &mut ctx).unwrap(); assert!( bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" @@ -321,11 +327,11 @@ mod test { // Use low selectivity (only select 2% of values) to avoid full decompression. let indices: Vec = (0..20).collect(); - #[expect(deprecated)] let filtered = bitpacked .filter(Mask::from_indices(values.len(), indices)) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); let expected: Vec = values[0..20].to_vec(); assert_arrays_eq!(filtered, PrimitiveArray::from_iter(expected)); diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index cdf27dade75..0a743f9269b 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -8,8 +8,6 @@ use lending_iterator::LendingIterator; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::aggregate_fn::fns::is_constant::IsConstant; use vortex_array::aggregate_fn::fns::is_constant::primitive::IS_CONST_LANE_WIDTH; @@ -46,7 +44,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { }; let result = match_each_integer_ptype!(array.dtype().as_ptype(), |P| { - bitpacked_is_constant::() }>(array)? + bitpacked_is_constant::() }>(array, ctx)? }); Ok(Some(IsConstant::make_partial(batch, result, ctx)?)) @@ -55,16 +53,18 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { fn bitpacked_is_constant( array: ArrayView<'_, BitPacked>, + ctx: &mut ExecutionCtx, ) -> VortexResult { let mut bit_unpack_iterator = array.unpacked_chunks::()?; - let patches = array.patches().map(|p| { - #[expect(deprecated)] - let values = p.values().to_primitive(); - #[expect(deprecated)] - let indices = p.indices().to_primitive(); - let offset = p.offset(); - (indices, values, offset) - }); + let patches = array + .patches() + .map(|p| -> VortexResult<_> { + let values = p.values().clone().execute::(ctx)?; + let indices = p.indices().clone().execute::(ctx)?; + let offset = p.offset(); + Ok((indices, values, offset)) + }) + .transpose()?; let mut header_constant_value = None; let mut current_idx = 0; @@ -195,8 +195,8 @@ mod tests { #[test] fn is_constant_with_patches() -> VortexResult<()> { - let array = BitPackedData::encode(&buffer![4; 1025].into_array(), 2)?; let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = BitPackedData::encode(&buffer![4; 1025].into_array(), 2, &mut ctx)?; assert!(is_constant(&array.into_array(), &mut ctx)?); Ok(()) } diff --git a/encodings/fastlanes/src/bitpacking/compute/mod.rs b/encodings/fastlanes/src/bitpacking/compute/mod.rs index f404102c019..2501d952356 100644 --- a/encodings/fastlanes/src/bitpacking/compute/mod.rs +++ b/encodings/fastlanes/src/bitpacking/compute/mod.rs @@ -42,6 +42,8 @@ fn chunked_indices( mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -51,7 +53,13 @@ mod tests { use crate::bitpacking::compute::chunked_indices; fn bp(array: &PrimitiveArray, bit_width: u8) -> BitPackedArray { - bitpack_encode(array, bit_width, None).unwrap() + bitpack_encode( + array, + bit_width, + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } #[test] diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index 93cb60353c6..e6e51c57591 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -46,6 +46,8 @@ impl SliceReduce for BitPacked { #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::SliceArray; use vortex_error::VortexResult; @@ -55,8 +57,9 @@ mod tests { #[test] fn test_reduce_parent_returns_bitpacked_slice() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter(0u32..2048); - let bitpacked = bitpack_encode(&values, 11, None)?; + let bitpacked = bitpack_encode(&values, 11, None, &mut ctx)?; let slice_array = SliceArray::new(bitpacked.clone().into_array(), 500..1500); diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index b7ea83513c7..a4c148c5717 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -166,8 +166,6 @@ mod test { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -182,11 +180,12 @@ mod test { #[test] fn take_indices() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let indices = buffer![0, 125, 2047, 2049, 2151, 2790].into_array(); // Create a u8 array modulo 63. let unpacked = PrimitiveArray::from_iter((0..4096).map(|i| (i % 63) as u8)); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6, &mut ctx).unwrap(); let primitive_result = bitpacked.take(indices).unwrap(); assert_arrays_eq!( @@ -197,8 +196,9 @@ mod test { #[test] fn take_with_patches() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unpacked = Buffer::from_iter(0u32..1024).into_array(); - let bitpacked = BitPackedData::encode(&unpacked, 2).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked, 2, &mut ctx).unwrap(); let indices = buffer![0, 2, 4, 6].into_array(); @@ -208,11 +208,12 @@ mod test { #[test] fn take_sliced_indices() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let indices = buffer![1919, 1921].into_array(); // Create a u8 array modulo 63. let unpacked = PrimitiveArray::from_iter((0..4096).map(|i| (i % 63) as u8)); - let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6).unwrap(); + let bitpacked = BitPackedData::encode(&unpacked.into_array(), 6, &mut ctx).unwrap(); let sliced = bitpacked.slice(128..2050).unwrap(); let primitive_result = sliced.take(indices).unwrap(); @@ -222,10 +223,11 @@ mod test { #[test] #[cfg_attr(miri, ignore)] // This test is too slow on miri fn take_random_indices() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let num_patches: usize = 128; let values = (0..u16::MAX as u32 + num_patches as u32).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let packed = BitPackedData::encode(&uncompressed.into_array(), 16).unwrap(); + let packed = BitPackedData::encode(&uncompressed.into_array(), 16, &mut ctx).unwrap(); assert!(packed.patches().is_some()); let rng = rng(); @@ -235,7 +237,6 @@ mod test { let taken = packed.take(random_indices.clone().into_array()).unwrap(); // sanity check - let mut ctx = LEGACY_SESSION.create_execution_ctx(); random_indices .as_slice::() .iter() @@ -255,14 +256,16 @@ mod test { #[test] #[cfg_attr(miri, ignore)] fn take_signed_with_patches() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let start = - BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); + BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1, &mut ctx) + .unwrap(); let taken_primitive = take_primitive::( start.as_view(), &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap(); assert_arrays_eq!(taken_primitive, PrimitiveArray::from_iter([1i32, 2, 3, 4])); @@ -270,8 +273,10 @@ mod test { #[test] fn take_nullable_with_nullables() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let start = - BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); + BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1, &mut ctx) + .unwrap(); let taken_primitive = start .take( @@ -282,26 +287,29 @@ mod test { taken_primitive, PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(4)]) ); - #[expect(deprecated)] - let taken_primitive_prim = taken_primitive.to_primitive(); - assert_eq!( - taken_primitive_prim - .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(), - 1 - ); + let taken_primitive_prim = taken_primitive.execute::(&mut ctx).unwrap(); + assert_eq!(taken_primitive_prim.invalid_count(&mut ctx).unwrap(), 1); + } + + fn bp(array: vortex_array::ArrayRef, bit_width: u8) -> BitPackedArray { + BitPackedData::encode( + &array, + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } #[rstest] - #[case(BitPackedData::encode(&PrimitiveArray::from_iter((0..100).map(|i| (i % 63) as u8)).into_array(), 6).unwrap())] - #[case(BitPackedData::encode(&PrimitiveArray::from_iter((0..256).map(|i| i as u32)).into_array(), 8).unwrap())] - #[case(BitPackedData::encode(&buffer![1i32, 2, 3, 4, 5, 6, 7, 8].into_array(), 3).unwrap())] - #[case(BitPackedData::encode( - &PrimitiveArray::from_option_iter([Some(10u16), None, Some(20), Some(30), None]).into_array(), + #[case(bp(PrimitiveArray::from_iter((0..100).map(|i| (i % 63) as u8)).into_array(), 6))] + #[case(bp(PrimitiveArray::from_iter((0..256).map(|i| i as u32)).into_array(), 8))] + #[case(bp(buffer![1i32, 2, 3, 4, 5, 6, 7, 8].into_array(), 3))] + #[case(bp( + PrimitiveArray::from_option_iter([Some(10u16), None, Some(20), Some(30), None]).into_array(), 5 - ).unwrap())] - #[case(BitPackedData::encode(&buffer![42u32].into_array(), 6).unwrap())] - #[case(BitPackedData::encode(&PrimitiveArray::from_iter((0..1024).map(|i| i as u32)).into_array(), 8).unwrap())] + ))] + #[case(bp(buffer![42u32].into_array(), 6))] + #[case(bp(PrimitiveArray::from_iter((0..1024).map(|i| i as u32)).into_array(), 8))] fn test_take_bitpacked_conformance(#[case] bitpacked: BitPackedArray) { use vortex_array::compute::conformance::take::test_take_conformance; test_take_conformance(&bitpacked.into_array()); diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index 49511a10748..d5ecdefd0c7 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -95,6 +95,7 @@ mod tests { use vortex_array::ArrayPlugin; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PatchedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::PatchedArraySlotsExt; @@ -120,11 +121,12 @@ mod tests { #[test] fn test_decode_bitpacked_patches() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); // Create values where some exceed the bit width, causing patches. // With bit_width=9, max value is 511. Values >=512 become patches. let values: Buffer = (0i32..=512).collect(); let parray = values.into_array(); - let bitpacked = BitPackedData::encode(&parray, 9)?; + let bitpacked = BitPackedData::encode(&parray, 9, &mut ctx)?; assert!( bitpacked.patches().is_some(), @@ -170,10 +172,11 @@ mod tests { #[test] fn bitpacked_without_patches_stays_bitpacked() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); // With bit_width=16, max value is 65535. All values 0..100 fit. let values: Buffer = (0i32..100).collect(); let parray = values.into_array(); - let bitpacked = BitPackedData::encode(&parray, 16)?; + let bitpacked = BitPackedData::encode(&parray, 16, &mut ctx)?; assert!( bitpacked.patches().is_none(), diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 9c1875468a1..8c9e4d214c6 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -353,7 +353,11 @@ impl BitPacked { } /// Encode an array into a bitpacked representation with the given bit width. - pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { - BitPackedData::encode(array, bit_width) + pub fn encode( + array: &ArrayRef, + bit_width: u8, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + BitPackedData::encode(array, bit_width, ctx) } } diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 74c5baa53a4..6b82343d318 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -58,7 +58,7 @@ mod test { use crate::bitpacking::array::BitPackedArrayExt; fn bp(array: &ArrayRef, bit_width: u8) -> BitPackedArray { - BitPackedData::encode(array, bit_width).unwrap() + BitPackedData::encode(array, bit_width, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn slice_via_reduce(array: &BitPackedArray, range: Range) -> BitPackedArray { @@ -141,8 +141,9 @@ mod test { #[test] fn slice_empty_patches() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // We create an array that has 1 element that does not fit in the 6-bit range. - let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6).unwrap(); + let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6, &mut ctx).unwrap(); assert!(array.patches().is_some()); @@ -213,9 +214,10 @@ mod test { #[test] fn scalar_at() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = (0u32..257).collect::>(); let uncompressed = values.clone().into_array(); - let packed = BitPackedData::encode(&uncompressed, 8).unwrap(); + let packed = BitPackedData::encode(&uncompressed, 8, &mut ctx).unwrap(); assert!(packed.patches().is_some()); let patches = packed.patches().unwrap().indices().clone(); diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index c13e9f60d4c..65f8b853aa6 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -27,7 +27,7 @@ pub fn delta_compress( // Fill-forward null values so that transposed deltas at null positions remain // small. Without this, bitpacking may skip patches for null positions, and the // corrupted delta values propagate through the cumulative sum during decompression. - let filled = fill_forward_nulls(array.to_buffer::(), &validity); + let filled = fill_forward_nulls(array.to_buffer::(), &validity, ctx)?; let (bases, deltas) = compress_primitive::(&filled); // TODO(robert): This can be avoided if we add TransposedBoolArray that performs index translation when necessary. let validity = transpose_validity(&validity, ctx)?; @@ -96,8 +96,6 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -132,12 +130,13 @@ mod tests { /// where null positions contain arbitrary values. Without fill-forward, the delta cumulative /// sum propagates corrupted values from null positions. #[test] - fn delta_bitpacked_trailing_nulls() { + fn delta_bitpacked_trailing_nulls() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter( (0u8..200).map(|i| (!(50..100).contains(&i)).then_some(i)), ); - let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap(); - let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap(); + let (bases, deltas) = delta_compress(&array, &mut ctx).unwrap(); + let bitpacked_deltas = bitpack_encode(&deltas, 1, None, &mut ctx).unwrap(); let packed_delta = Delta::try_new( bases.into_array(), bitpacked_deltas.into_array(), @@ -145,8 +144,11 @@ mod tests { array.len(), ) .vortex_expect("Delta array construction should succeed"); - #[expect(deprecated)] - let packed_delta_prim = packed_delta.as_array().to_primitive(); + let packed_delta_prim = packed_delta + .as_array() + .clone() + .execute::(&mut ctx)?; assert_arrays_eq!(packed_delta_prim, array); + Ok(()) } } diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index 72e8c2ab835..943f11379ac 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -4,8 +4,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexResult; @@ -17,8 +16,10 @@ impl OperationsVTable for Delta { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - #[expect(deprecated)] - let decompressed = array.array().slice(index..index + 1)?.to_primitive(); + let decompressed = array + .array() + .slice(index..index + 1)? + .execute::(ctx)?; decompressed.into_array().execute_scalar(0, ctx) } } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 0be93fa955d..358eff2a4e9 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -52,8 +52,6 @@ mod test { use std::sync::LazyLock; use itertools::Itertools; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::assert_arrays_eq; @@ -130,32 +128,31 @@ mod test { #[test] fn test_decompress_fused() { + let mut ctx = SESSION.create_execution_ctx(); // Create a range offset by a million. let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); - let bp = BitPackedData::encode(&array.into_array(), 3).unwrap(); + let bp = BitPackedData::encode(&array.into_array(), 3, &mut ctx).unwrap(); let compressed = FoR::try_new(bp.into_array(), 10u32.into()).unwrap(); assert_arrays_eq!(compressed, expect); } #[test] fn test_decompress_fused_patches() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); // Create a range offset by a million. let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); - let bp = BitPackedData::encode(&array.into_array(), 2).unwrap(); + let bp = BitPackedData::encode(&array.into_array(), 2, &mut ctx).unwrap(); let compressed = FoR::try_new(bp.clone().into_array(), 10u32.into())?; - let decompressed = fused_decompress::( - &compressed, - bp.as_view(), - &mut SESSION.create_execution_ctx(), - )?; + let decompressed = fused_decompress::(&compressed, bp.as_view(), &mut ctx)?; assert_arrays_eq!(decompressed, expect); Ok(()) } #[test] fn test_overflow() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter(i8::MIN..=i8::MAX); let compressed = FoRData::encode(array.clone()).unwrap(); assert_eq!( @@ -167,16 +164,16 @@ mod test { .unwrap() ); - #[expect(deprecated)] let encoded = compressed .encoded() - .to_primitive() + .clone() + .execute::(&mut ctx)? .reinterpret_cast(PType::U8); let unsigned: Vec = (0..=u8::MAX).collect_vec(); let expected_unsigned = PrimitiveArray::from_iter(unsigned); assert_eq!(encoded.as_slice::(), expected_unsigned.as_slice::()); - let decompressed = decompress(&compressed, &mut SESSION.create_execution_ctx())?; + let decompressed = decompress(&compressed, &mut ctx)?; array .as_slice::() .iter() @@ -184,12 +181,7 @@ mod test { .for_each(|(i, v)| { assert_eq!( *v, - i8::try_from( - &compressed - .execute_scalar(i, &mut SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap() + i8::try_from(&compressed.execute_scalar(i, &mut ctx).unwrap()).unwrap() ); }); assert_arrays_eq!(decompressed, array); diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index eba41b07927..6b72af5902b 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -4,8 +4,6 @@ use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::aggregate_fn::AggregateFnRef; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::fns::is_sorted::is_sorted; @@ -36,8 +34,7 @@ impl DynAggregateKernel for FoRIsSortedKernel { return Ok(None); }; - #[expect(deprecated)] - let encoded = array.encoded().to_primitive(); + let encoded = array.encoded().clone().execute::(ctx)?; let unsigned_array = PrimitiveArray::from_buffer_handle( encoded.buffer_handle().clone(), encoded.ptype().to_unsigned(), diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 0041cf99b15..9022b7c4e2b 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -7,12 +7,13 @@ pub use bitpacking::*; pub use delta::*; pub use r#for::*; pub use rle::*; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; +use vortex_error::VortexResult; pub mod bit_transpose; mod bitpacking; @@ -78,13 +79,16 @@ pub fn initialize(session: &VortexSession) { pub(crate) fn fill_forward_nulls( values: Buffer, validity: &Validity, -) -> Buffer { + ctx: &mut ExecutionCtx, +) -> VortexResult> { match validity { - Validity::NonNullable | Validity::AllValid => values, - Validity::AllInvalid => Buffer::zeroed(values.len()), + Validity::NonNullable | Validity::AllValid => Ok(values), + Validity::AllInvalid => Ok(Buffer::zeroed(values.len())), Validity::Array(validity_array) => { - #[expect(deprecated)] - let bit_buffer = validity_array.to_bool().to_bit_buffer(); + let bit_buffer = validity_array + .clone() + .execute::(ctx)? + .to_bit_buffer(); let mut last_valid = T::default(); match values.try_into_mut() { Ok(mut to_fill_mut) => { @@ -99,7 +103,7 @@ pub(crate) fn fill_forward_nulls( *v = last_valid; } } - to_fill_mut.freeze() + Ok(to_fill_mut.freeze()) } Err(to_fill) => { let mut to_fill_mut = BufferMut::::with_capacity(to_fill.len()); @@ -121,7 +125,7 @@ pub(crate) fn fill_forward_nulls( out.write(last_valid); } unsafe { to_fill_mut.set_len(to_fill.len()) }; - to_fill_mut.freeze() + Ok(to_fill_mut.freeze()) } } } @@ -132,6 +136,7 @@ pub(crate) fn fill_forward_nulls( mod test { use std::sync::LazyLock; + use vortex_array::VortexSessionExecute; use vortex_array::session::ArraySessionExt; use vortex_buffer::BitBufferMut; use vortex_session::VortexSession; @@ -148,7 +153,8 @@ mod test { }); #[test] - fn fill_forward_nulls_resets_at_chunk_boundary() { + fn fill_forward_nulls_resets_at_chunk_boundary() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); // Build a buffer spanning two chunks where the last valid value in chunk 0 // is non-zero. Null positions at the start of chunk 1 must get T::default() // (0), not the carry-over from chunk 0. @@ -160,7 +166,7 @@ mod test { validity_bits.set(FL_CHUNK_SIZE - 1); // only this position is valid let validity = Validity::from(validity_bits.freeze()); - let result = fill_forward_nulls(values.freeze(), &validity); + let result = fill_forward_nulls(values.freeze(), &validity, &mut ctx)?; // Within chunk 0, nulls before the valid element get 0 (default), and the // valid element itself is 42. @@ -174,5 +180,6 @@ mod test { "position {i} should be 0, not carried from chunk 0" ); } + Ok(()) } } diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index d8f822c19ff..743b18ec7f0 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -5,9 +5,8 @@ use std::fmt::Display; use std::fmt::Formatter; use vortex_array::ArrayRef; -use vortex_array::LEGACY_SESSION; +use vortex_array::ExecutionCtx; use vortex_array::TypedArrayRef; -use vortex_array::VortexSessionExecute; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -111,16 +110,16 @@ pub trait RLEArrayExt: TypedArrayRef { clippy::expect_used, reason = "expect is safe here as scalar_at returns a valid primitive" )] - fn values_idx_offset(&self, chunk_idx: usize) -> usize { + fn values_idx_offset(&self, chunk_idx: usize, ctx: &mut ExecutionCtx) -> usize { self.values_idx_offsets() - .execute_scalar(chunk_idx, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(chunk_idx, ctx) .expect("index must be in bounds") .as_primitive() .as_::() .expect("index must be of type usize") - self .values_idx_offsets() - .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(0, ctx) .expect("index must be in bounds") .as_primitive() .as_::() @@ -142,8 +141,6 @@ mod tests { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -237,10 +234,13 @@ mod tests { let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) .vortex_expect("RLEData is always valid"); - #[expect(deprecated)] - let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); - // TODO(joe): replace with compute null count let mut ctx = SESSION.create_execution_ctx(); + let valid_slice = rle_array + .slice(0..3) + .unwrap() + .execute::(&mut ctx) + .unwrap(); + // TODO(joe): replace with compute null count assert!(valid_slice.all_valid(&mut ctx).unwrap()); let mixed_slice = rle_array.slice(1..5).unwrap(); @@ -347,6 +347,7 @@ mod tests { #[test] fn test_multi_chunk_two_chunks() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]).into_array(); let indices = PrimitiveArray::from_iter([0u16, 1].repeat(1024)).into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64, 2]).into_array(); @@ -356,18 +357,21 @@ mod tests { assert_eq!(rle_array.len(), 2048); assert_eq!(rle_array.values().len(), 4); - assert_eq!(rle_array.values_idx_offset(0), 0); - assert_eq!(rle_array.values_idx_offset(1), 2); + assert_eq!(rle_array.values_idx_offset(0, &mut ctx), 0); + assert_eq!(rle_array.values_idx_offset(1, &mut ctx), 2); } #[test] - fn test_rle_serialization() { + fn test_rle_serialization() -> VortexResult<()> { + let mut exec_ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(primitive.as_view()).unwrap(); + let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx).unwrap(); assert_eq!(rle_array.len(), 2048); - #[expect(deprecated)] - let original_data = rle_array.as_array().to_primitive(); + let original_data = rle_array + .as_array() + .clone() + .execute::(&mut exec_ctx)?; let ctx = ArrayContext::empty(); let serialized = rle_array @@ -391,16 +395,17 @@ mod tests { ) .unwrap(); - #[expect(deprecated)] - let decoded_data = decoded.to_primitive(); + let decoded_data = decoded.execute::(&mut exec_ctx)?; assert_arrays_eq!(original_data, decoded_data); + Ok(()) } #[test] - fn test_rle_serialization_slice() { + fn test_rle_serialization_slice() -> VortexResult<()> { + let mut exec_ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(primitive.as_view()).unwrap(); + let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx).unwrap(); let sliced = RLE::try_new( rle_array.values().clone(), @@ -435,12 +440,14 @@ mod tests { ) .unwrap(); - #[expect(deprecated)] - let original_data = sliced.as_array().to_primitive(); - #[expect(deprecated)] - let decoded_data = decoded.to_primitive(); + let original_data = sliced + .as_array() + .clone() + .execute::(&mut exec_ctx)?; + let decoded_data = decoded.execute::(&mut exec_ctx)?; assert_arrays_eq!(original_data, decoded_data); + Ok(()) } /// Regression test: re-encoding RLE indices with RLE must not corrupt @@ -453,6 +460,7 @@ mod tests { /// because chunk 1 only had 1 unique value and index 1 was out of bounds. #[test] fn test_recompress_indices_no_cross_chunk_leak() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let len = FL_CHUNK_SIZE + 100; let mut values: Vec> = vec![None; len]; // Two distinct values in chunk 0 → indices 0 and 1. @@ -461,13 +469,16 @@ mod tests { // Chunk 1 (positions 1024..) is all-null. let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; // Simulate cascading compression: narrow u16->u8 then re-encode with RLE, // matching the path taken by the BtrBlocks compressor. - #[expect(deprecated)] - let indices_prim = rle.indices().to_primitive().narrow()?; - let re_encoded = RLEData::encode(indices_prim.as_view())?; + let indices_prim = rle + .indices() + .clone() + .execute::(&mut ctx)? + .narrow()?; + let re_encoded = RLEData::encode(indices_prim.as_view(), &mut ctx)?; // Reconstruct the outer RLE with re-encoded indices. // SAFETY: we only replace the indices child; all other invariants hold. @@ -482,8 +493,10 @@ mod tests { }; // Decompress — panicked before the fill_forward_nulls chunk-boundary fix. - #[expect(deprecated)] - let decoded = reconstructed.as_array().to_primitive(); + let decoded = reconstructed + .as_array() + .clone() + .execute::(&mut ctx)?; assert_arrays_eq!(decoded, original); Ok(()) } diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index a73e8ac10de..f566e1f101b 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -5,9 +5,9 @@ use std::mem; use fastlanes::RLE as FastLanesRLE; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::arrays::BoolArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -28,23 +28,26 @@ use crate::fill_forward_nulls; impl RLEData { /// Encodes a primitive array of unsigned integers using FastLanes RLE. - pub fn encode(array: ArrayView<'_, Primitive>) -> VortexResult { + pub fn encode( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let array = array.into_owned(); - match_each_native_ptype!(array.ptype(), |T| { rle_encode_typed::(&array) }) + match_each_native_ptype!(array.ptype(), |T| { rle_encode_typed::(&array, ctx) }) } } /// Encodes a primitive array of unsigned integers using FastLanes RLE. /// /// In case the input array length is % 1024 != 0, the last chunk is padded. -fn rle_encode_typed(array: &PrimitiveArray) -> VortexResult +fn rle_encode_typed(array: &PrimitiveArray, ctx: &mut ExecutionCtx) -> VortexResult where T: NativePType + FastLanesRLE, NativeValue: FastLanesRLE, { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. - let values = fill_forward_nulls(array.to_buffer::(), &array.validity()?); + let values = fill_forward_nulls(array.to_buffer::(), &array.validity()?, ctx)?; let len = values.len(); let padded_len = len.next_multiple_of(FL_CHUNK_SIZE); @@ -113,7 +116,7 @@ where RLE::try_new( values_buf.into_array(), - PrimitiveArray::new(indices_buf.freeze(), padded_validity(array)).into_array(), + PrimitiveArray::new(indices_buf.freeze(), padded_validity(array, ctx)?).into_array(), values_idx_offsets.into_array(), 0, array.len(), @@ -121,30 +124,29 @@ where } /// Returns validity padded to the next 1024 chunk for a given array. -fn padded_validity(array: &PrimitiveArray) -> Validity { +fn padded_validity(array: &PrimitiveArray, ctx: &mut ExecutionCtx) -> VortexResult { match array .validity() .vortex_expect("RLE validity should be derivable") { - Validity::NonNullable => Validity::NonNullable, - Validity::AllValid => Validity::AllValid, - Validity::AllInvalid => Validity::AllInvalid, + Validity::NonNullable => Ok(Validity::NonNullable), + Validity::AllValid => Ok(Validity::AllValid), + Validity::AllInvalid => Ok(Validity::AllInvalid), Validity::Array(validity_array) => { let len = array.len(); let padded_len = len.next_multiple_of(FL_CHUNK_SIZE); if len == padded_len { - return Validity::Array(validity_array); + return Ok(Validity::Array(validity_array)); } let mut builder = BitBufferMut::with_capacity(padded_len); - #[expect(deprecated)] - let bool_array = validity_array.to_bool(); + let bool_array = validity_array.execute::(ctx)?; builder.append_buffer(&bool_array.to_bit_buffer()); builder.append_n(false, padded_len - len); - Validity::from(builder.freeze()) + Ok(Validity::from(builder.freeze())) } } } @@ -152,9 +154,10 @@ fn padded_validity(array: &PrimitiveArray) -> Validity { #[cfg(test)] mod tests { use rstest::rstest; + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; use vortex_array::arrays::PrimitiveArray; @@ -168,91 +171,128 @@ mod tests { use crate::rle::array::RLEArrayExt; #[test] - fn test_encode_decode() { + fn test_encode_decode() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // u8 let array_u8: Buffer = buffer![1, 1, 2, 2, 3, 3]; - let encoded_u8 = - RLEData::encode(PrimitiveArray::new(array_u8, Validity::NonNullable).as_view()) - .unwrap(); - #[expect(deprecated)] - let decoded_u8 = encoded_u8.as_array().to_primitive(); + let encoded_u8 = RLEData::encode( + PrimitiveArray::new(array_u8, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); + let decoded_u8 = encoded_u8 + .as_array() + .clone() + .execute::(&mut ctx)?; let expected_u8 = PrimitiveArray::from_iter(vec![1u8, 1, 2, 2, 3, 3]); assert_arrays_eq!(decoded_u8, expected_u8); // u16 let array_u16: Buffer = buffer![100, 100, 200, 200]; - let encoded_u16 = - RLEData::encode(PrimitiveArray::new(array_u16, Validity::NonNullable).as_view()) - .unwrap(); - #[expect(deprecated)] - let decoded_u16 = encoded_u16.as_array().to_primitive(); + let encoded_u16 = RLEData::encode( + PrimitiveArray::new(array_u16, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); + let decoded_u16 = encoded_u16 + .as_array() + .clone() + .execute::(&mut ctx)?; let expected_u16 = PrimitiveArray::from_iter(vec![100u16, 100, 200, 200]); assert_arrays_eq!(decoded_u16, expected_u16); // u64 let array_u64: Buffer = buffer![1000, 1000, 2000]; - let encoded_u64 = - RLEData::encode(PrimitiveArray::new(array_u64, Validity::NonNullable).as_view()) - .unwrap(); - #[expect(deprecated)] - let decoded_u64 = encoded_u64.as_array().to_primitive(); + let encoded_u64 = RLEData::encode( + PrimitiveArray::new(array_u64, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); + let decoded_u64 = encoded_u64 + .as_array() + .clone() + .execute::(&mut ctx)?; let expected_u64 = PrimitiveArray::from_iter(vec![1000u64, 1000, 2000]); assert_arrays_eq!(decoded_u64, expected_u64); + Ok(()) } #[test] fn test_length() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = buffer![1, 1, 2, 2, 2, 3]; - let encoded = - RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); + let encoded = RLEData::encode( + PrimitiveArray::new(values, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); assert_eq!(encoded.len(), 6); } #[test] fn test_empty_length() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = Buffer::empty(); - let encoded = - RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); + let encoded = RLEData::encode( + PrimitiveArray::new(values, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); assert_eq!(encoded.len(), 0); assert_eq!(encoded.values().len(), 0); } #[test] - fn test_single_value() { + fn test_single_value() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = vec![42; 2000].into_iter().collect(); - let encoded = - RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); + let encoded = RLEData::encode( + PrimitiveArray::new(values, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); assert_eq!(encoded.values().len(), 2); // 2 chunks, each storing value 42 - #[expect(deprecated)] - let decoded = encoded.as_array().to_primitive(); // Verify round-trip + let decoded = encoded + .as_array() + .clone() + .execute::(&mut ctx)?; // Verify round-trip let expected = PrimitiveArray::from_iter(vec![42u16; 2000]); assert_arrays_eq!(decoded, expected); + Ok(()) } #[test] - fn test_all_different() { + fn test_all_different() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = (0u8..=255).collect(); - let encoded = - RLEData::encode(PrimitiveArray::new(values, Validity::NonNullable).as_view()).unwrap(); + let encoded = RLEData::encode( + PrimitiveArray::new(values, Validity::NonNullable).as_view(), + &mut ctx, + ) + .unwrap(); assert_eq!(encoded.values().len(), 256); - #[expect(deprecated)] - let decoded = encoded.as_array().to_primitive(); // Verify round-trip + let decoded = encoded + .as_array() + .clone() + .execute::(&mut ctx)?; // Verify round-trip let expected = PrimitiveArray::from_iter((0u8..=255).collect::>()); assert_arrays_eq!(decoded, expected); + Ok(()) } #[test] fn test_partial_last_chunk() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test array with partial last chunk (not divisible by 1024) let values: Buffer = (0..1500).map(|i| (i / 100) as u32).collect(); let array = PrimitiveArray::new(values, Validity::NonNullable); - let encoded = RLEData::encode(array.as_view()).unwrap(); + let encoded = RLEData::encode(array.as_view(), &mut ctx).unwrap(); assert_eq!(encoded.len(), 1500); assert_arrays_eq!(encoded, array); @@ -262,11 +302,12 @@ mod tests { #[test] fn test_two_full_chunks() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Array that spans exactly 2 chunks (2048 elements) let values: Buffer = (0..2048).map(|i| (i / 100) as u32).collect(); let array = PrimitiveArray::new(values, Validity::NonNullable); - let encoded = RLEData::encode(array.as_view()).unwrap(); + let encoded = RLEData::encode(array.as_view(), &mut ctx).unwrap(); assert_eq!(encoded.len(), 2048); assert_arrays_eq!(encoded, array); @@ -285,12 +326,19 @@ mod tests { #[case::f16((-2000..2000).map(|i| f16::from_f32(i as f32)).collect::>())] #[case::f32((-2000..2000).map(|i| i as f32).collect::>())] #[case::f64((-2000..2000).map(|i| i as f64).collect::>())] - fn test_roundtrip_primitive_types(#[case] values: Buffer) { - #[expect(deprecated)] - let primitive = values.clone().into_array().to_primitive(); - let result = RLEData::encode(primitive.as_view()).unwrap(); - #[expect(deprecated)] - let decoded = result.as_array().to_primitive(); + fn test_roundtrip_primitive_types( + #[case] values: Buffer, + ) -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let primitive = values + .clone() + .into_array() + .execute::(&mut ctx)?; + let result = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); + let decoded = result + .as_array() + .clone() + .execute::(&mut ctx)?; let expected = PrimitiveArray::new( values, primitive @@ -298,6 +346,7 @@ mod tests { .vortex_expect("primitive validity should be derivable"), ); assert_arrays_eq!(decoded, expected); + Ok(()) } /// Replaces the indices of an RLE array with MaskedArray(ConstantArray(1u16), validity). @@ -306,9 +355,11 @@ mod tests { /// Valid when every chunk has at least two RLE dictionary entries (the /// fill-forward default at index 0 and the actual value at index 1), which /// holds whenever the first position of each chunk is null. - fn with_masked_constant_indices(rle: &RLEArray) -> VortexResult { - #[expect(deprecated)] - let indices_prim = rle.indices().to_primitive(); + fn with_masked_constant_indices( + rle: &RLEArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let indices_prim = rle.indices().clone().execute::(ctx)?; let masked_indices = MaskedArray::try_new( ConstantArray::new(1u16, indices_prim.len()).into_array(), indices_prim.validity()?, @@ -325,40 +376,44 @@ mod tests { #[test] fn test_encode_all_null_chunk() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Vec> = vec![None; FL_CHUNK_SIZE]; let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let decoded = with_masked_constant_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let decoded = with_masked_constant_indices(&rle, &mut ctx)?; assert_arrays_eq!(decoded, original); Ok(()) } #[test] fn test_encode_all_null_chunk_then_value_chunk() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // First chunk is entirely null, second chunk has a value preceded by nulls. let mut values: Vec> = vec![None; 2 * FL_CHUNK_SIZE]; values[FL_CHUNK_SIZE + 100] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let decoded = with_masked_constant_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let decoded = with_masked_constant_indices(&rle, &mut ctx)?; assert_arrays_eq!(decoded, original); Ok(()) } #[test] fn test_encode_one_value_near_end() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Single distinct value near the end of the chunk. let mut values: Vec> = vec![None; FL_CHUNK_SIZE]; values[1000] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let decoded = with_masked_constant_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let decoded = with_masked_constant_indices(&rle, &mut ctx)?; assert_arrays_eq!(decoded, original); Ok(()) } #[test] fn test_encode_value_chunk_then_all_null_remainder() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // 1085 elements (2 chunks: 1024 + 61 padded to 1024). // Chunk 0 has -1i16 at scattered positions (273..=366), rest null. // Chunk 1 (the remainder) is entirely null. @@ -373,8 +428,8 @@ mod tests { values[pos] = Some(-1); } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let decoded = with_masked_constant_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let decoded = with_masked_constant_indices(&rle, &mut ctx)?; assert_arrays_eq!(decoded, original); Ok(()) } @@ -384,9 +439,11 @@ mod tests { /// This simulates a compressor that doesn't preserve index values at null /// positions, which can happen when indices are further compressed and the /// compressor clobbers invalid entries with arbitrary data. - fn with_random_invalid_indices(rle: &RLEArray) -> VortexResult { - #[expect(deprecated)] - let indices_prim = rle.indices().to_primitive(); + fn with_random_invalid_indices( + rle: &RLEArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let indices_prim = rle.indices().clone().execute::(ctx)?; let mut indices_data: Vec = indices_prim.as_slice::().to_vec(); // Use a simple deterministic "random" sequence. @@ -416,43 +473,47 @@ mod tests { #[test] fn test_random_invalid_indices_all_null_chunk() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Vec> = vec![None; FL_CHUNK_SIZE]; let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let clobbered = with_random_invalid_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let clobbered = with_random_invalid_indices(&rle, &mut ctx)?; assert_arrays_eq!(clobbered, original); Ok(()) } #[test] fn test_random_invalid_indices_sparse_values() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut values: Vec> = vec![None; FL_CHUNK_SIZE]; values[0] = Some(10); values[500] = Some(20); values[1000] = Some(30); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let clobbered = with_random_invalid_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let clobbered = with_random_invalid_indices(&rle, &mut ctx)?; assert_arrays_eq!(clobbered, original); Ok(()) } #[test] fn test_random_invalid_indices_multi_chunk() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Two chunks: first has scattered values, second is all null. let mut values: Vec> = vec![None; 2 * FL_CHUNK_SIZE]; values[0] = Some(10); values[500] = Some(20); values[FL_CHUNK_SIZE + 100] = Some(42); let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let clobbered = with_random_invalid_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let clobbered = with_random_invalid_indices(&rle, &mut ctx)?; assert_arrays_eq!(clobbered, original); Ok(()) } #[test] fn test_random_invalid_indices_partial_last_chunk() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // 1085 elements: chunk 0 has values at scattered positions, chunk 1 is // a partial (61 elements padded to 1024) that is entirely null. let mut values: Vec> = vec![None; 1085]; @@ -460,14 +521,15 @@ mod tests { values[i] = Some(i as u32); } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let clobbered = with_random_invalid_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let clobbered = with_random_invalid_indices(&rle, &mut ctx)?; assert_arrays_eq!(clobbered, original); Ok(()) } #[test] fn test_random_invalid_indices_mostly_valid() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Most positions are valid, only a few are null with garbage indices. let mut values: Vec> = (0..FL_CHUNK_SIZE).map(|i| Some((i / 100) as u64)).collect(); @@ -476,8 +538,8 @@ mod tests { values[i] = None; } let original = PrimitiveArray::from_option_iter(values); - let rle = RLEData::encode(original.as_view())?; - let clobbered = with_random_invalid_indices(&rle)?; + let rle = RLEData::encode(original.as_view(), &mut ctx)?; + let clobbered = with_random_invalid_indices(&rle, &mut ctx)?; assert_arrays_eq!(clobbered, original); Ok(()) } @@ -488,11 +550,14 @@ mod tests { #[case(vec![f16::ZERO, f16::NEG_ZERO])] #[case(vec![0f32, -0f32])] #[case(vec![0f64, -0f64])] - fn test_float_zeros(#[case] values: Vec) { + fn test_float_zeros( + #[case] values: Vec, + ) -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter(values); - let rle = RLEData::encode(primitive.as_view()).unwrap(); - #[expect(deprecated)] - let decoded = rle.as_array().to_primitive(); + let rle = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); + let decoded = rle.as_array().clone().execute::(&mut ctx)?; assert_arrays_eq!(primitive, decoded); + Ok(()) } } diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index fc63f293c3a..23231b10b12 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -45,7 +45,11 @@ impl CastReduce for RLE { #[cfg(test)] mod tests { use rstest::rstest; + use vortex_array::Canonical; + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -59,17 +63,18 @@ mod tests { use crate::RLEData; use crate::rle::RLEArray; - fn rle(primitive: &PrimitiveArray) -> RLEArray { - RLEData::encode(primitive.as_view()).unwrap() + fn rle(primitive: &PrimitiveArray, ctx: &mut ExecutionCtx) -> RLEArray { + RLEData::encode(primitive.as_view(), ctx).unwrap() } #[test] fn try_cast_rle_success() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive = PrimitiveArray::new( Buffer::from_iter([10u8, 20, 30, 40, 50]), Validity::from_iter([true, true, true, true, true]), ); - let encoded = rle(&primitive); + let encoded = rle(&primitive, &mut ctx); let casted = encoded .into_array() @@ -81,16 +86,16 @@ mod tests { #[test] #[should_panic] fn try_cast_rle_fail() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive = PrimitiveArray::new( Buffer::from_iter([10u8, 20, 30, 40, 50]), Validity::from_iter([true, false, true, true, false]), ); - let encoded = rle(&primitive); - #[expect(deprecated)] + let encoded = rle(&primitive, &mut ctx); let result = encoded .into_array() .cast(DType::Primitive(PType::U8, Nullability::NonNullable)) - .and_then(|a| a.to_canonical().map(|c| c.into_array())); + .and_then(|a| a.execute::(&mut ctx).map(|c| c.into_array())); result.unwrap(); } @@ -144,7 +149,8 @@ mod tests { ) )] fn test_cast_rle_conformance(#[case] primitive: PrimitiveArray) { - let rle_array = rle(&primitive); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let rle_array = rle(&primitive, &mut ctx); test_cast_conformance(&rle_array.into_array()); } } diff --git a/encodings/fastlanes/src/rle/kernel.rs b/encodings/fastlanes/src/rle/kernel.rs index 14eaf6dda3d..f5b654706cd 100644 --- a/encodings/fastlanes/src/rle/kernel.rs +++ b/encodings/fastlanes/src/rle/kernel.rs @@ -23,15 +23,15 @@ impl SliceKernel for RLE { fn slice( array: ArrayView<'_, Self>, range: Range, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let offset_in_chunk = array.offset(); let chunk_start_idx = (offset_in_chunk + range.start) / FL_CHUNK_SIZE; let chunk_end_idx = (offset_in_chunk + range.end).div_ceil(FL_CHUNK_SIZE); - let values_start_idx = array.values_idx_offset(chunk_start_idx); + let values_start_idx = array.values_idx_offset(chunk_start_idx, ctx); let values_end_idx = if chunk_end_idx < array.values_idx_offsets().len() { - array.values_idx_offset(chunk_end_idx) + array.values_idx_offset(chunk_end_idx, ctx) } else { array.values().len() }; diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 6d37aad85fd..ed549301433 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -244,8 +244,11 @@ impl RLE { } /// Encode a primitive array using FastLanes RLE. - pub fn encode(array: ArrayView<'_, Primitive>) -> VortexResult { - RLEData::encode(array) + pub fn encode( + array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + RLEData::encode(array, ctx) } } diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index 1273e019dec..0441a1e61b9 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -29,7 +29,7 @@ impl OperationsVTable for RLE { .vortex_expect("Index must not be null"); let chunk_id = (offset_in_chunk + index) / FL_CHUNK_SIZE; - let value_idx_offset = array.values_idx_offset(chunk_id); + let value_idx_offset = array.values_idx_offset(chunk_id, ctx); let scalar = array .values() @@ -43,8 +43,6 @@ impl OperationsVTable for RLE { mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -170,20 +168,21 @@ mod tests { #[test] fn test_scalar_at_multiple_chunks() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test accessing elements around chunk boundaries let values: Buffer = (0..3000).map(|i| (i / 50) as u16).collect(); let expected: Vec = (0..3000).map(|i| (i / 50) as u16).collect(); let array = values.into_array(); - #[expect(deprecated)] - let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); + let primitive = array.execute::(&mut ctx).unwrap(); + let encoded = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); // Access scalars from multiple chunks. for &idx in &[1023, 1024, 1025, 2047, 2048, 2049] { if idx < encoded.len() { let original_value = expected[idx]; let encoded_value = encoded - .execute_scalar(idx, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(idx, &mut ctx) .unwrap() .as_primitive() .as_::() @@ -265,9 +264,13 @@ mod tests { #[test] fn test_slice_decode_with_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = fixture::rle_array_with_nulls(); - #[expect(deprecated)] - let sliced = array.slice(1..4).unwrap().to_primitive(); // [null, 20, 20] + let sliced = array + .slice(1..4) + .unwrap() + .execute::(&mut ctx) + .unwrap(); // [null, 20, 20] let expected = PrimitiveArray::from_option_iter([Option::::None, Some(20), Some(20)]); assert_arrays_eq!(sliced.into_array(), expected.into_array()); @@ -283,12 +286,13 @@ mod tests { #[test] fn test_slice_across_chunk_boundaries() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values: Buffer = (0..2100).map(|i| (i / 100) as u32).collect(); let expected: Vec = (0..2100).map(|i| (i / 100) as u32).collect(); let array = values.into_array(); - #[expect(deprecated)] - let encoded = RLEData::encode(array.to_primitive().as_view()).unwrap(); + let primitive = array.execute::(&mut ctx).unwrap(); + let encoded = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); // Slice across first and second chunk. let slice = encoded.slice(500..1500).unwrap(); diff --git a/encodings/fsst/benches/chunked_dict_fsst_builder.rs b/encodings/fsst/benches/chunked_dict_fsst_builder.rs index ff1c61f0b80..22131631a6e 100644 --- a/encodings/fsst/benches/chunked_dict_fsst_builder.rs +++ b/encodings/fsst/benches/chunked_dict_fsst_builder.rs @@ -37,8 +37,9 @@ fn make_dict_fsst_chunks( unique_values: usize, chunk_count: usize, ) -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); (0..chunk_count) - .map(|_| gen_dict_fsst_test_data::(len, unique_values, 20, 30).into_array()) + .map(|_| gen_dict_fsst_test_data::(len, unique_values, 20, 30, &mut ctx).into_array()) .collect::() .into_array() } diff --git a/encodings/fsst/benches/fsst_compress.rs b/encodings/fsst/benches/fsst_compress.rs index 6e42f5a39cf..45aedfa464f 100644 --- a/encodings/fsst/benches/fsst_compress.rs +++ b/encodings/fsst/benches/fsst_compress.rs @@ -57,9 +57,9 @@ fn compress_fsst(bencher: Bencher, (string_count, avg_len, unique_chars): (usize let array = generate_test_data(string_count, avg_len, unique_chars); let compressor = fsst_train_compressor(&array); bencher - .with_inputs(|| (&array, &compressor)) - .bench_refs(|(array, compressor)| { - fsst_compress(*array, array.len(), array.dtype(), compressor) + .with_inputs(|| (&array, &compressor, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(array, compressor, ctx)| { + fsst_compress(*array, array.len(), array.dtype(), compressor, ctx) }) } @@ -69,7 +69,13 @@ fn decompress_fsst(bencher: Bencher, (string_count, avg_len, unique_chars): (usi let compressor = fsst_train_compressor(&array); let len = array.len(); let dtype = array.dtype().clone(); - let encoded = fsst_compress(array, len, &dtype, &compressor); + let encoded = fsst_compress( + array, + len, + &dtype, + &compressor, + &mut LEGACY_SESSION.create_execution_ctx(), + ); bencher .with_inputs(|| (&encoded, LEGACY_SESSION.create_execution_ctx())) @@ -88,7 +94,13 @@ fn train_compressor(bencher: Bencher, (string_count, avg_len, unique_chars): (us fn pushdown_compare(bencher: Bencher, (string_count, avg_len, unique_chars): (usize, usize, u8)) { let array = generate_test_data(string_count, avg_len, unique_chars); let compressor = fsst_train_compressor(&array); - let fsst_array = fsst_compress(&array, array.len(), array.dtype(), &compressor); + let fsst_array = fsst_compress( + &array, + array.len(), + array.dtype(), + &compressor, + &mut LEGACY_SESSION.create_execution_ctx(), + ); let constant = ConstantArray::new(Scalar::from(&b"const"[..]), array.len()); bencher @@ -117,7 +129,13 @@ fn canonicalize_compare( ) { let array = generate_test_data(string_count, avg_len, unique_chars); let compressor = fsst_train_compressor(&array); - let fsst_array = fsst_compress(&array, array.len(), array.dtype(), &compressor); + let fsst_array = fsst_compress( + &array, + array.len(), + array.dtype(), + &compressor, + &mut LEGACY_SESSION.create_execution_ctx(), + ); let constant = ConstantArray::new(Scalar::from(&b"const"[..]), array.len()); bencher @@ -217,13 +235,14 @@ fn generate_chunked_test_data( avg_len: usize, unique_chars: u8, ) -> ChunkedArray { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); (0..chunk_size) .map(|_| { let array = generate_test_data(string_count, avg_len, unique_chars); let compressor = fsst_train_compressor(&array); let len = array.len(); let dtype = array.dtype().clone(); - fsst_compress(array, len, &dtype, &compressor).into_array() + fsst_compress(array, len, &dtype, &compressor, &mut ctx).into_array() }) .collect::() } diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index 2771b558ccc..28aa3b109ae 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -35,13 +35,20 @@ static SESSION: LazyLock = const N: usize = NUM_STRINGS; -static FSST_URLS: LazyLock = LazyLock::new(|| make_fsst_short_urls(N)); -static FSST_CB_URLS: LazyLock = LazyLock::new(|| make_fsst_clickbench_urls(N)); -static FSST_LOG_LINES: LazyLock = LazyLock::new(|| make_fsst_log_lines(N)); -static FSST_JSON_STRINGS: LazyLock = LazyLock::new(|| make_fsst_json_strings(N)); -static FSST_FILE_PATHS: LazyLock = LazyLock::new(|| make_fsst_file_paths(N)); -static FSST_EMAILS: LazyLock = LazyLock::new(|| make_fsst_emails(N)); -static FSST_RARE_MATCH: LazyLock = LazyLock::new(|| make_fsst_rare_match(N)); +static FSST_URLS: LazyLock = + LazyLock::new(|| make_fsst_short_urls(N, &mut SESSION.create_execution_ctx())); +static FSST_CB_URLS: LazyLock = + LazyLock::new(|| make_fsst_clickbench_urls(N, &mut SESSION.create_execution_ctx())); +static FSST_LOG_LINES: LazyLock = + LazyLock::new(|| make_fsst_log_lines(N, &mut SESSION.create_execution_ctx())); +static FSST_JSON_STRINGS: LazyLock = + LazyLock::new(|| make_fsst_json_strings(N, &mut SESSION.create_execution_ctx())); +static FSST_FILE_PATHS: LazyLock = + LazyLock::new(|| make_fsst_file_paths(N, &mut SESSION.create_execution_ctx())); +static FSST_EMAILS: LazyLock = + LazyLock::new(|| make_fsst_emails(N, &mut SESSION.create_execution_ctx())); +static FSST_RARE_MATCH: LazyLock = + LazyLock::new(|| make_fsst_rare_match(N, &mut SESSION.create_execution_ctx())); enum Dataset { Urls, diff --git a/encodings/fsst/benches/fsst_url_compare.rs b/encodings/fsst/benches/fsst_url_compare.rs index 6514d691350..656cd9f1866 100644 --- a/encodings/fsst/benches/fsst_url_compare.rs +++ b/encodings/fsst/benches/fsst_url_compare.rs @@ -57,7 +57,13 @@ fn pick_url_with_domain(data: &VarBinArray, domain: &str) -> String { fn eq_pushdown_high_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let match_url = pick_url_with_domain(data, HIGH_MATCH_DOMAIN); let constant = ConstantArray::new(Scalar::from(match_url.as_str()), NUM_URLS); @@ -78,7 +84,13 @@ fn eq_pushdown_high_match(bencher: Bencher) { fn eq_pushdown_low_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let match_url = pick_url_with_domain(data, LOW_MATCH_DOMAIN); let constant = ConstantArray::new(Scalar::from(match_url.as_str()), NUM_URLS); @@ -99,7 +111,13 @@ fn eq_pushdown_low_match(bencher: Bencher) { fn eq_canonicalize_high_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let match_url = pick_url_with_domain(data, HIGH_MATCH_DOMAIN); let constant = ConstantArray::new(Scalar::from(match_url.as_str()), NUM_URLS); @@ -123,7 +141,13 @@ fn eq_canonicalize_high_match(bencher: Bencher) { fn eq_canonicalize_low_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let match_url = pick_url_with_domain(data, LOW_MATCH_DOMAIN); let constant = ConstantArray::new(Scalar::from(match_url.as_str()), NUM_URLS); @@ -151,7 +175,13 @@ fn eq_canonicalize_low_match(bencher: Bencher) { fn like_substr_high_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let pattern = format!("%{HIGH_MATCH_DOMAIN}%"); let expr = like(root(), lit(pattern.as_str())); @@ -172,7 +202,13 @@ fn like_substr_high_match(bencher: Bencher) { fn like_substr_low_match(bencher: Bencher) { let data = &*URL_DATA; let compressor = fsst_train_compressor(data); - let fsst_array = fsst_compress(data, data.len(), data.dtype(), &compressor); + let fsst_array = fsst_compress( + data, + data.len(), + data.dtype(), + &compressor, + &mut SESSION.create_execution_ctx(), + ); let pattern = format!("%{LOW_MATCH_DOMAIN}%"); let expr = like(root(), lit(pattern.as_str())); diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 412fa8302af..6e598e26c18 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -4,7 +4,7 @@ pub struct vortex_fsst::FSST impl vortex_fsst::FSST -pub fn vortex_fsst::FSST::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_fsst::FSST::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fsst::FSST @@ -56,7 +56,7 @@ pub fn vortex_fsst::FSST::validity(array: vortex_array::array::view::ArrayView<' impl vortex_array::arrays::dict::take::TakeExecute for vortex_fsst::FSST -pub fn vortex_fsst::FSST::take(array: vortex_array::array::view::ArrayView<'_, Self>, indices: &vortex_array::array::erased::ArrayRef, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_fsst::FSST::take(array: vortex_array::array::view::ArrayView<'_, Self>, indices: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> impl vortex_array::arrays::filter::kernel::FilterKernel for vortex_fsst::FSST @@ -76,7 +76,7 @@ pub fn vortex_fsst::FSST::cast(array: vortex_array::array::view::ArrayView<'_, S impl vortex_array::scalar_fn::fns::like::kernel::LikeKernel for vortex_fsst::FSST -pub fn vortex_fsst::FSST::like(array: vortex_array::array::view::ArrayView<'_, Self>, pattern: &vortex_array::array::erased::ArrayRef, options: vortex_array::scalar_fn::fns::like::LikeOptions, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_fsst::FSST::like(array: vortex_array::array::view::ArrayView<'_, Self>, pattern: &vortex_array::array::erased::ArrayRef, options: vortex_array::scalar_fn::fns::like::LikeOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> pub struct vortex_fsst::FSSTData @@ -100,7 +100,7 @@ pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes_bytes: vortex_array::buffer::BufferHandle, len: usize) -> vortex_error::VortexResult -pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option], ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> impl core::clone::Clone for vortex_fsst::FSSTData @@ -176,9 +176,9 @@ pub fn T::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef pub fn T::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_fsst::fsst_compress>(strings: A, len: usize, dtype: &vortex_array::dtype::DType, compressor: &fsst::Compressor) -> vortex_fsst::FSSTArray +pub fn vortex_fsst::fsst_compress>(strings: A, len: usize, dtype: &vortex_array::dtype::DType, compressor: &fsst::Compressor, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_fsst::FSSTArray -pub fn vortex_fsst::fsst_compress_iter<'a, I>(iter: I, len: usize, dtype: vortex_array::dtype::DType, compressor: &fsst::Compressor) -> vortex_fsst::FSSTArray where I: core::iter::traits::iterator::Iterator> +pub fn vortex_fsst::fsst_compress_iter<'a, I>(iter: I, len: usize, dtype: vortex_array::dtype::DType, compressor: &fsst::Compressor, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_fsst::FSSTArray where I: core::iter::traits::iterator::Iterator> pub fn vortex_fsst::fsst_train_compressor>(array: &A) -> fsst::Compressor diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index b510c0412ec..719a13a8507 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -115,7 +115,9 @@ impl VTable for FSST { len: usize, slots: &[Option], ) -> VortexResult<()> { - data.validate(dtype, len, slots) + // TODO(ctx): trait fixes - VTable::validate has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + data.validate(dtype, len, slots, &mut ctx) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -235,6 +237,8 @@ impl VTable for FSST { vortex_bail!("Expected 2 or 3 children, got {}", children.len()); }; + // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); FSSTData::validate_parts( &symbols, &symbol_lengths, @@ -244,6 +248,7 @@ impl VTable for FSST { &uncompressed_lengths, dtype, len, + &mut ctx, )?; let slots = vec![ Some(uncompressed_lengths), @@ -385,6 +390,7 @@ impl FSST { symbol_lengths: Buffer, codes: VarBinArray, uncompressed_lengths: ArrayRef, + ctx: &mut ExecutionCtx, ) -> VortexResult { let len = codes.len(); FSSTData::validate_parts_from_codes( @@ -394,6 +400,7 @@ impl FSST { &uncompressed_lengths, &dtype, len, + ctx, )?; let slots = FSSTData::make_slots(&codes, &uncompressed_lengths); let codes_bytes = codes.bytes_handle().clone(); @@ -437,6 +444,8 @@ impl FSST { len, )?; + // TODO(ctx): trait fixes - VTable::deserialize has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); FSSTData::validate_parts_from_codes( symbols, symbol_lengths, @@ -444,6 +453,7 @@ impl FSST { &uncompressed_lengths, dtype, len, + &mut ctx, )?; let slots = FSSTData::make_slots(&codes, &uncompressed_lengths); let codes_bytes = codes.bytes_handle().clone(); @@ -516,6 +526,7 @@ impl FSSTData { dtype: &DType, len: usize, slots: &[Option], + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { let codes_offsets = slots[CODES_OFFSETS_SLOT] .as_ref() @@ -529,6 +540,7 @@ impl FSSTData { uncompressed_lengths_from_slots(slots), dtype, len, + ctx, ) } @@ -543,6 +555,7 @@ impl FSSTData { uncompressed_lengths: &ArrayRef, dtype: &DType, len: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { vortex_ensure!( matches!(dtype, DType::Binary(_) | DType::Utf8(_)), @@ -582,10 +595,7 @@ impl FSSTData { // Validate that last offset doesn't exceed bytes length (when host-resident). if codes_bytes.is_on_host() && codes_offsets.is_host() && !codes_offsets.is_empty() { let last_offset: usize = (&codes_offsets - .execute_scalar( - codes_offsets.len() - 1, - &mut LEGACY_SESSION.create_execution_ctx(), - ) + .execute_scalar(codes_offsets.len() - 1, ctx) .vortex_expect("offsets must support scalar_at")) .try_into() .vortex_expect("Failed to convert offset to usize"); @@ -608,6 +618,7 @@ impl FSSTData { uncompressed_lengths: &ArrayRef, dtype: &DType, len: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { Self::validate_parts( symbols, @@ -618,6 +629,7 @@ impl FSSTData { uncompressed_lengths, dtype, len, + ctx, ) } @@ -792,11 +804,13 @@ mod test { let symbol_lengths = Buffer::::copy_from([3, 8]); let compressor = Compressor::rebuild_from(symbols.as_slice(), symbol_lengths.as_slice()); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let fsst_array = fsst_compress_iter( [Some(b"abcabcab".as_ref()), Some(b"defghijk".as_ref())].into_iter(), 2, DType::Utf8(Nullability::NonNullable), &compressor, + &mut ctx, ); let compressed_codes = fsst_array.codes(); diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 9853cd3956e..a8b8171b043 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -95,12 +95,11 @@ mod tests { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::VarBinArray; + use vortex_array::arrays::VarBinViewArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; @@ -149,13 +148,15 @@ mod tests { } fn make_data_chunked() -> (ChunkedArray, Vec>>) { + let mut ctx = SESSION.create_execution_ctx(); #[expect(clippy::type_complexity)] let (arr_vec, data_vec): (Vec, Vec>>>) = (0..10) .map(|_| { let (array, data) = make_data(); let compressor = fsst_train_compressor(&array); ( - fsst_compress(&array, array.len(), array.dtype(), &compressor).into_array(), + fsst_compress(&array, array.len(), array.dtype(), &compressor, &mut ctx) + .into_array(), data, ) }) @@ -169,6 +170,7 @@ mod tests { #[test] fn test_to_canonical() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let (chunked_arr, data) = make_data_chunked(); let mut builder = @@ -176,7 +178,7 @@ mod tests { chunked_arr .clone() .into_array() - .append_to_builder(&mut builder, &mut SESSION.create_execution_ctx())?; + .append_to_builder(&mut builder, &mut ctx)?; { let arr = builder.finish_into_canonical().into_varbinview(); @@ -186,8 +188,10 @@ mod tests { }; { - #[expect(deprecated)] - let arr2 = chunked_arr.as_array().to_varbinview(); + let arr2 = chunked_arr + .as_array() + .clone() + .execute::(&mut ctx)?; let res2 = arr2.with_iterator(|iter| iter.map(|b| b.map(|v| v.to_vec())).collect::>()); assert_eq!(data, res2) @@ -205,14 +209,16 @@ mod tests { [Some(b"long enough too".to_vec().into_boxed_slice())], dtype, ); + let mut ctx = SESSION.create_execution_ctx(); let fsst_array = fsst_compress( &varbin, varbin.len(), varbin.dtype(), &fsst_train_compressor(&varbin), + &mut ctx, ) .into_array(); - fsst_array.append_to_builder(&mut builder, &mut SESSION.create_execution_ctx())?; + fsst_array.append_to_builder(&mut builder, &mut ctx)?; let _result = builder.finish_into_varbinview(); Ok(()) diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index 23f7490529a..872dcbc494d 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -5,6 +5,7 @@ use fsst::Compressor; use fsst::Symbol; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::varbin::builder::VarBinBuilder; @@ -21,8 +22,9 @@ pub fn fsst_compress>( len: usize, dtype: &DType, compressor: &Compressor, + ctx: &mut ExecutionCtx, ) -> FSSTArray { - strings.with_iterator(|iter| fsst_compress_iter(iter, len, dtype.clone(), compressor)) + strings.with_iterator(|iter| fsst_compress_iter(iter, len, dtype.clone(), compressor, ctx)) } /// Train a compressor from an array. @@ -61,6 +63,7 @@ pub fn fsst_compress_iter<'a, I>( len: usize, dtype: DType, compressor: &Compressor, + ctx: &mut ExecutionCtx, ) -> FSSTArray where I: Iterator>, @@ -102,8 +105,15 @@ where let uncompressed_lengths = uncompressed_lengths.into_array(); - FSST::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) - .vortex_expect("FSST parts must be valid") + FSST::try_new( + dtype, + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + ctx, + ) + .vortex_expect("FSST parts must be valid") } #[cfg(test)] @@ -128,16 +138,16 @@ mod tests { let compressor = CompressorBuilder::default().build(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressed = fsst_compress_iter( [Some(big_string.as_bytes())].into_iter(), 1, DType::Utf8(Nullability::NonNullable), &compressor, + &mut ctx, ); - let decoded = compressed - .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(); + let decoded = compressed.execute_scalar(0, &mut ctx).unwrap(); let expected = Scalar::utf8(big_string, Nullability::NonNullable); diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 9553cb4d8a9..550181af309 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBin; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; @@ -23,6 +25,8 @@ impl CastReduce for FSST { .into_array() .cast(array.codes_dtype().with_nullability(dtype.nullability()))?; + // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); Ok(Some( FSST::try_new( dtype.clone(), @@ -30,6 +34,7 @@ impl CastReduce for FSST { array.symbol_lengths().clone(), new_codes.as_::().into_owned(), array.uncompressed_lengths().clone(), + &mut ctx, )? .into_array(), )) @@ -43,6 +48,8 @@ impl CastReduce for FSST { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::compute::conformance::cast::test_cast_conformance; @@ -54,6 +61,7 @@ mod tests { #[test] fn test_cast_fsst_nullability() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = VarBinArray::from_iter( vec![Some("hello"), Some("world"), Some("hello world")], DType::Utf8(Nullability::NonNullable), @@ -62,7 +70,7 @@ mod tests { let compressor = fsst_train_compressor(&strings); let len = strings.len(); let dtype = strings.dtype().clone(); - let fsst = fsst_compress(strings, len, &dtype, &compressor); + let fsst = fsst_compress(strings, len, &dtype, &compressor, &mut ctx); // Cast to nullable let casted = fsst @@ -86,8 +94,9 @@ mod tests { DType::Utf8(Nullability::NonNullable) ))] fn test_cast_fsst_conformance(#[case] array: VarBinArray) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressor = fsst_train_compressor(&array); - let fsst = fsst_compress(&array, array.len(), array.dtype(), &compressor); + let fsst = fsst_compress(&array, array.len(), array.dtype(), &compressor, &mut ctx); test_cast_conformance(&fsst.into_array()); } } diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index f641e52d8bf..d1ef02e391f 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -123,8 +123,8 @@ fn compare_fsst_constant( #[cfg(test)] mod tests { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::VarBinArray; @@ -141,6 +141,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_compare_fsst() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let lhs = VarBinArray::from_iter( [ Some("hello"), @@ -154,18 +155,18 @@ mod tests { let compressor = fsst_train_compressor(&lhs); let len = lhs.len(); let dtype = lhs.dtype().clone(); - let lhs = fsst_compress(lhs, len, &dtype, &compressor); + let lhs = fsst_compress(lhs, len, &dtype, &compressor, &mut ctx); let rhs = ConstantArray::new("world", lhs.len()); // Ensure fastpath for Eq exists, and returns correct answer - #[expect(deprecated)] let equals = lhs .clone() .into_array() .binary(rhs.clone().into_array(), Operator::Eq) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(equals.dtype(), &DType::Bool(Nullability::Nullable)); @@ -175,13 +176,13 @@ mod tests { ); // Ensure fastpath for Eq exists, and returns correct answer - #[expect(deprecated)] let not_equals = lhs .clone() .into_array() .binary(rhs.into_array(), Operator::NotEq) .unwrap() - .to_bool(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(not_equals.dtype(), &DType::Bool(Nullability::Nullable)); assert_arrays_eq!( diff --git a/encodings/fsst/src/compute/filter.rs b/encodings/fsst/src/compute/filter.rs index 9f60cb6c596..74e32cfbd02 100644 --- a/encodings/fsst/src/compute/filter.rs +++ b/encodings/fsst/src/compute/filter.rs @@ -37,6 +37,7 @@ impl FilterKernel for FSST { array.symbol_lengths().clone(), filtered_codes, array.uncompressed_lengths().filter(mask.clone())?, + ctx, )? .into_array(), )) diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index ef050435aba..a6b8c40d4c2 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -5,9 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::varbin::VarBinArrayExt; use vortex_array::match_each_integer_ptype; use vortex_array::scalar_fn::fns::like::LikeKernel; @@ -24,7 +23,7 @@ impl LikeKernel for FSST { array: ArrayView<'_, Self>, pattern: &ArrayRef, options: LikeOptions, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let Some(pattern_scalar) = pattern.as_constant() else { return Ok(None); @@ -59,8 +58,7 @@ impl LikeKernel for FSST { let negated = options.negated; let codes = array.codes(); - #[expect(deprecated)] - let offsets = codes.offsets().to_primitive(); + let offsets = codes.offsets().clone().execute::(ctx)?; let all_bytes = codes.bytes(); let all_bytes = all_bytes.as_slice(); let n = codes.len(); @@ -115,7 +113,13 @@ mod tests { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress( + varbin, + len, + &dtype, + &compressor, + &mut SESSION.create_execution_ctx(), + ) } fn run_like(array: FSSTArray, pattern: &str, opts: LikeOptions) -> VortexResult { diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index ac350e6186c..02efdf7febc 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -25,7 +25,7 @@ impl TakeExecute for FSST { fn take( array: ArrayView<'_, Self>, indices: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( FSST::try_new( @@ -38,7 +38,7 @@ impl TakeExecute for FSST { { let codes = array.codes(); let codes = codes.as_view(); - ::take(codes, indices, _ctx)? + ::take(codes, indices, ctx)? .vortex_expect("VarBin take kernel always returns Some") } .try_downcast::() @@ -49,6 +49,7 @@ impl TakeExecute for FSST { .fill_null(Scalar::zero_value( &array.uncompressed_lengths_dtype().clone(), ))?, + ctx, )? .into_array(), )) @@ -58,7 +59,10 @@ impl TakeExecute for FSST { #[cfg(test)] mod tests { use rstest::rstest; + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinArray; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -72,9 +76,10 @@ mod tests { #[test] fn test_take_null() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = VarBinArray::from_iter([Some("h")], DType::Utf8(Nullability::NonNullable)); let compr = fsst_train_compressor(&arr); - let fsst = fsst_compress(&arr, arr.len(), arr.dtype(), &compr); + let fsst = fsst_compress(&arr, arr.len(), arr.dtype(), &compr, &mut ctx); let idx1: PrimitiveArray = (0..1).collect(); @@ -105,23 +110,26 @@ mod tests { DType::Utf8(Nullability::NonNullable), ))] fn test_take_fsst_conformance(#[case] varbin: VarBinArray) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressor = fsst_train_compressor(&varbin); - let array = fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor); + let array = fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor, &mut ctx); test_take_conformance(&array.into_array()); } + type FsstBuilder = fn(&mut ExecutionCtx) -> FSSTArray; + #[rstest] // Basic string arrays - #[case::fsst_simple({ + #[case::fsst_simple(|ctx: &mut ExecutionCtx| { let varbin = VarBinArray::from_iter( ["hello world", "testing fsst", "compression test", "data array", "vortex encoding"].map(Some), DType::Utf8(Nullability::NonNullable), ); let compressor = fsst_train_compressor(&varbin); - fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor) + fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor, ctx) })] // Nullable strings - #[case::fsst_nullable({ + #[case::fsst_nullable(|ctx: &mut ExecutionCtx| { let varbin = VarBinArray::from_iter( [Some("hello"), None, Some("world"), Some("test"), None], DType::Utf8(Nullability::Nullable), @@ -129,27 +137,27 @@ mod tests { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) })] // Repetitive patterns (good for FSST compression) - #[case::fsst_repetitive({ + #[case::fsst_repetitive(|ctx: &mut ExecutionCtx| { let varbin = VarBinArray::from_iter( ["http://example.com", "http://test.com", "http://vortex.dev", "http://data.org"].map(Some), DType::Utf8(Nullability::NonNullable), ); let compressor = fsst_train_compressor(&varbin); - fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor) + fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor, ctx) })] // Edge cases - #[case::fsst_single({ + #[case::fsst_single(|ctx: &mut ExecutionCtx| { let varbin = VarBinArray::from_iter( ["single element"].map(Some), DType::Utf8(Nullability::NonNullable), ); let compressor = fsst_train_compressor(&varbin); - fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor) + fsst_compress(&varbin, varbin.len(), varbin.dtype(), &compressor, ctx) })] - #[case::fsst_empty_strings({ + #[case::fsst_empty_strings(|ctx: &mut ExecutionCtx| { let varbin = VarBinArray::from_iter( ["", "test", "", "hello", ""].map(Some), DType::Utf8(Nullability::NonNullable), @@ -157,10 +165,10 @@ mod tests { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) })] // Large arrays - #[case::fsst_large({ + #[case::fsst_large(|ctx: &mut ExecutionCtx| { let data: Vec> = (0..1500) .map(|i| Some(match i % 10 { 0 => "https://www.example.com/page", @@ -179,10 +187,12 @@ mod tests { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) })] - fn test_fsst_consistency(#[case] array: FSSTArray) { + fn test_fsst_consistency(#[case] build: FsstBuilder) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = build(&mut ctx); test_array_consistency(&array.into_array()); } } diff --git a/encodings/fsst/src/dfa/tests.rs b/encodings/fsst/src/dfa/tests.rs index caf04adb3c6..6ad30ca685d 100644 --- a/encodings/fsst/src/dfa/tests.rs +++ b/encodings/fsst/src/dfa/tests.rs @@ -229,7 +229,13 @@ fn make_fsst_str(strings: &[Option<&str>]) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress( + varbin, + len, + &dtype, + &compressor, + &mut SESSION.create_execution_ctx(), + ) } fn run_like(array: FSSTArray, pattern_arr: ArrayRef) -> VortexResult { diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs index 05b58e3215d..079efcfecb0 100644 --- a/encodings/fsst/src/kernel.rs +++ b/encodings/fsst/src/kernel.rs @@ -58,7 +58,8 @@ mod tests { let compressor = fsst_train_compressor(&input); let len = input.len(); let dtype = input.dtype().clone(); - fsst_compress(input, len, &dtype, &compressor).into_array() + let mut ctx = SESSION.create_execution_ctx(); + fsst_compress(input, len, &dtype, &compressor, &mut ctx).into_array() } #[test] @@ -131,8 +132,15 @@ mod tests { let input = builder.finish(DType::Utf8(Nullability::Nullable)); let compressor = fsst_train_compressor(&input); - let fsst_array: ArrayRef = - fsst_compress(input.clone(), input.len(), input.dtype(), &compressor).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let fsst_array: ArrayRef = fsst_compress( + input.clone(), + input.len(), + input.dtype(), + &compressor, + &mut ctx, + ) + .into_array(); // Filter: only select the last element (index 22) let mut mask = vec![false; 22]; @@ -140,7 +148,6 @@ mod tests { let mask = Mask::from_iter(mask); let filter_array = FilterArray::new(fsst_array, mask.clone()).into_array(); - let mut ctx = SESSION.create_execution_ctx(); let result = filter_array.execute::(&mut ctx)?; let expected = input.filter(mask)?; @@ -160,13 +167,19 @@ mod tests { let input = builder.finish(DType::Utf8(Nullability::Nullable)); let compressor = fsst_train_compressor(&input); - let fsst_array: ArrayRef = - fsst_compress(input.clone(), input.len(), input.dtype(), &compressor).into_array(); + let mut ctx = SESSION.create_execution_ctx(); + let fsst_array: ArrayRef = fsst_compress( + input.clone(), + input.len(), + input.dtype(), + &compressor, + &mut ctx, + ) + .into_array(); let mask = Mask::from_iter([true, false, true]); let filter_array = FilterArray::new(fsst_array, mask.clone()).into_array(); - let mut ctx = SESSION.create_execution_ctx(); let result = filter_array.execute::(&mut ctx)?; let expected = input.filter(mask)?; diff --git a/encodings/fsst/src/test_utils.rs b/encodings/fsst/src/test_utils.rs index d0de924ad68..eeca412e377 100644 --- a/encodings/fsst/src/test_utils.rs +++ b/encodings/fsst/src/test_utils.rs @@ -7,6 +7,7 @@ use rand::RngExt; use rand::SeedableRng; use rand::prelude::StdRng; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; @@ -20,7 +21,12 @@ use crate::FSSTArray; use crate::fsst_compress; use crate::fsst_train_compressor; -pub fn gen_fsst_test_data(len: usize, avg_str_len: usize, unique_chars: u8) -> ArrayRef { +pub fn gen_fsst_test_data( + len: usize, + avg_str_len: usize, + unique_chars: u8, + ctx: &mut ExecutionCtx, +) -> ArrayRef { let mut rng = StdRng::seed_from_u64(0); let mut strings = Vec::with_capacity(len); @@ -45,7 +51,7 @@ pub fn gen_fsst_test_data(len: usize, avg_str_len: usize, unique_chars: u8) -> A let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor).into_array() + fsst_compress(varbin, len, &dtype, &compressor, ctx).into_array() } pub fn gen_dict_fsst_test_data( @@ -53,8 +59,9 @@ pub fn gen_dict_fsst_test_data( unique_values: usize, str_len: usize, unique_char_count: u8, + ctx: &mut ExecutionCtx, ) -> DictArray { - let values = gen_fsst_test_data(len, str_len, unique_char_count); + let values = gen_fsst_test_data(len, str_len, unique_char_count, ctx); let mut rng = StdRng::seed_from_u64(0); let codes = (0..len) .map(|_| T::from(rng.random_range(0..unique_values)).unwrap()) @@ -136,12 +143,12 @@ pub fn generate_url_data_n(n: usize) -> VarBinArray { VarBinArray::from_iter(urls, DType::Utf8(Nullability::NonNullable)) } -pub fn make_fsst_urls(n: usize) -> FSSTArray { +pub fn make_fsst_urls(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let varbin = generate_url_data_n(n); let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -228,7 +235,7 @@ pub fn generate_clickbench_urls(n: usize) -> Vec { .collect() } -pub fn make_fsst_clickbench_urls(n: usize) -> FSSTArray { +pub fn make_fsst_clickbench_urls(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let urls = generate_clickbench_urls(n); let varbin = VarBinArray::from_iter( urls.iter().map(|s| Some(s.as_str())), @@ -237,7 +244,7 @@ pub fn make_fsst_clickbench_urls(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -296,7 +303,7 @@ pub fn generate_short_urls(n: usize) -> Vec { .collect() } -pub fn make_fsst_short_urls(n: usize) -> FSSTArray { +pub fn make_fsst_short_urls(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let urls = generate_short_urls(n); let varbin = VarBinArray::from_iter( urls.iter().map(|s| Some(s.as_str())), @@ -305,7 +312,7 @@ pub fn make_fsst_short_urls(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -368,7 +375,7 @@ pub fn generate_log_lines(n: usize) -> Vec { .collect() } -pub fn make_fsst_log_lines(n: usize) -> FSSTArray { +pub fn make_fsst_log_lines(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let lines = generate_log_lines(n); let varbin = VarBinArray::from_iter( lines.iter().map(|s| Some(s.as_str())), @@ -377,7 +384,7 @@ pub fn make_fsst_log_lines(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -427,7 +434,7 @@ pub fn generate_json_strings(n: usize) -> Vec { .collect() } -pub fn make_fsst_json_strings(n: usize) -> FSSTArray { +pub fn make_fsst_json_strings(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let jsons = generate_json_strings(n); let varbin = VarBinArray::from_iter( jsons.iter().map(|s| Some(s.as_str())), @@ -436,7 +443,7 @@ pub fn make_fsst_json_strings(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -499,7 +506,7 @@ pub fn generate_file_paths(n: usize) -> Vec { .collect() } -pub fn make_fsst_file_paths(n: usize) -> FSSTArray { +pub fn make_fsst_file_paths(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let paths = generate_file_paths(n); let varbin = VarBinArray::from_iter( paths.iter().map(|s| Some(s.as_str())), @@ -508,7 +515,7 @@ pub fn make_fsst_file_paths(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -552,7 +559,7 @@ pub fn generate_emails(n: usize) -> Vec { .collect() } -pub fn make_fsst_emails(n: usize) -> FSSTArray { +pub fn make_fsst_emails(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let emails = generate_emails(n); let varbin = VarBinArray::from_iter( emails.iter().map(|s| Some(s.as_str())), @@ -561,7 +568,7 @@ pub fn make_fsst_emails(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } // --------------------------------------------------------------------------- @@ -591,7 +598,7 @@ pub fn generate_rare_match_strings(n: usize, match_rate: f64) -> Vec { .collect() } -pub fn make_fsst_rare_match(n: usize) -> FSSTArray { +pub fn make_fsst_rare_match(n: usize, ctx: &mut ExecutionCtx) -> FSSTArray { let strings = generate_rare_match_strings(n, 0.00001); let varbin = VarBinArray::from_iter( strings.iter().map(|s| Some(s.as_str())), @@ -600,5 +607,5 @@ pub fn make_fsst_rare_match(n: usize) -> FSSTArray { let compressor = fsst_train_compressor(&varbin); let len = varbin.len(); let dtype = varbin.dtype().clone(); - fsst_compress(varbin, len, &dtype, &compressor) + fsst_compress(varbin, len, &dtype, &compressor, ctx) } diff --git a/encodings/fsst/src/tests.rs b/encodings/fsst/src/tests.rs index 441e217a883..fb4cb8bdc5c 100644 --- a/encodings/fsst/src/tests.rs +++ b/encodings/fsst/src/tests.rs @@ -3,8 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::varbin::builder::VarBinBuilder; use vortex_array::assert_arrays_eq; use vortex_array::assert_nth_scalar; @@ -30,11 +31,13 @@ pub(crate) fn build_fsst_array() -> ArrayRef { let compressor = fsst_train_compressor(&input_array); let len = input_array.len(); let dtype = input_array.dtype().clone(); - fsst_compress(input_array, len, &dtype, &compressor).into_array() + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + fsst_compress(input_array, len, &dtype, &compressor, &mut ctx).into_array() } #[test] fn test_fsst_array_ops() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // first test the scalar_at values let fsst_array = build_fsst_array(); assert_nth_scalar!( @@ -96,8 +99,11 @@ fn test_fsst_array_ops() { ); // test to_canonical - #[expect(deprecated)] - let canonical_array = fsst_array.to_varbinview().into_array(); + let canonical_array = fsst_array + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); assert_arrays_eq!(fsst_array, canonical_array); } diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 27fe03ed57b..a36c852503c 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -4,7 +4,7 @@ pub struct vortex_pco::Pco impl vortex_pco::Pco -pub fn vortex_pco::Pco::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize) -> vortex_error::VortexResult +pub fn vortex_pco::Pco::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_pco::Pco @@ -86,9 +86,9 @@ impl vortex_pco::PcoData pub fn vortex_pco::PcoData::decompress(&self, unsliced_validity: &vortex_array::validity::Validity, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult +pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_pco::PcoData::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize) -> vortex_error::VortexResult +pub fn vortex_pco::PcoData::from_primitive(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, level: usize, values_per_page: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::PcoData::is_empty(&self) -> bool diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 8ea7c72c44e..edeb59b4a26 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -28,11 +28,7 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -258,13 +254,18 @@ pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { } } -fn collect_valid(parray: ArrayView<'_, Primitive>) -> VortexResult { - let mask = parray.array().validity()?.to_mask( - parray.array().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; - #[expect(deprecated)] - let result = parray.array().filter(mask)?.to_primitive(); +fn collect_valid( + parray: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = parray + .array() + .validity()? + .to_mask(parray.array().len(), ctx)?; + let result = parray + .array() + .filter(mask)? + .execute::(ctx)?; Ok(result) } @@ -299,10 +300,11 @@ impl Pco { parray: ArrayView<'_, Primitive>, level: usize, values_per_page: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let dtype = parray.dtype().clone(); let validity = parray.validity()?; - let data = PcoData::from_primitive(parray, level, values_per_page)?; + let data = PcoData::from_primitive(parray, level, values_per_page, ctx)?; Self::try_new(dtype, data, validity) } } @@ -408,8 +410,15 @@ impl PcoData { parray: ArrayView<'_, Primitive>, level: usize, values_per_page: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - Self::from_primitive_with_values_per_chunk(parray, level, VALUES_PER_CHUNK, values_per_page) + Self::from_primitive_with_values_per_chunk( + parray, + level, + VALUES_PER_CHUNK, + values_per_page, + ctx, + ) } pub(crate) fn from_primitive_with_values_per_chunk( @@ -417,6 +426,7 @@ impl PcoData { level: usize, values_per_chunk: usize, values_per_page: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let number_type = number_type_from_dtype(parray.dtype()); let values_per_page = if values_per_page == 0 { @@ -430,7 +440,7 @@ impl PcoData { .with_compression_level(level) .with_paging_spec(PagingSpec::EqualPagesUpTo(values_per_page)); - let values = collect_valid(parray)?; + let values = collect_valid(parray, ctx)?; let n_values = values.len(); let fc = FileCompressor::default(); @@ -484,14 +494,19 @@ impl PcoData { )) } - pub fn from_array(array: ArrayRef, level: usize, nums_per_page: usize) -> VortexResult { + pub fn from_array( + array: ArrayRef, + level: usize, + nums_per_page: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { let parray = array.try_downcast::().map_err(|a| { vortex_err!( "Pco can only encode primitive arrays, got {}", a.encoding_id() ) })?; - Self::from_primitive(parray.as_view(), level, nums_per_page) + Self::from_primitive(parray.as_view(), level, nums_per_page, ctx) } pub fn decompress( @@ -649,6 +664,8 @@ impl OperationsVTable for Pco { #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; @@ -658,12 +675,13 @@ mod tests { #[test] fn test_slice_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a nullable array with some nulls let values = PrimitiveArray::new( buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([false, true, true, true, true, false]), ); - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); assert_arrays_eq!( pco, PrimitiveArray::from_option_iter([ diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 1d5d61861a6..831a454c3b6 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -59,6 +59,8 @@ impl CastReduce for Pco { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -73,8 +75,9 @@ mod tests { #[test] fn test_cast_pco_f32_to_f64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([1.0f32, 2.0, 3.0, 4.0, 5.0]); - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); let casted = pco .into_array() @@ -93,9 +96,10 @@ mod tests { #[test] fn test_cast_pco_nullability_change() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test casting from NonNullable to Nullable let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); let casted = pco .into_array() @@ -109,11 +113,12 @@ mod tests { #[test] fn test_cast_sliced_pco_nullable_to_nonnullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::new( buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([true, true, true, true, true, true]), ); - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); let sliced = pco.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -128,6 +133,7 @@ mod tests { #[test] fn test_cast_sliced_pco_part_valid_to_nonnullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_option_iter([ None, Some(20u32), @@ -136,7 +142,7 @@ mod tests { Some(50), Some(60), ]); - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); let sliced = pco.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -170,7 +176,8 @@ mod tests { Validity::NonNullable, ))] fn test_cast_pco_conformance(#[case] values: PrimitiveArray) { - let pco = Pco::from_primitive(values.as_view(), 0, 128).unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let pco = Pco::from_primitive(values.as_view(), 0, 128, &mut ctx).unwrap(); test_cast_conformance(&pco.into_array()); } } diff --git a/encodings/pco/src/compute/mod.rs b/encodings/pco/src/compute/mod.rs index 62f1ba1fd23..036a639ae31 100644 --- a/encodings/pco/src/compute/mod.rs +++ b/encodings/pco/src/compute/mod.rs @@ -7,6 +7,8 @@ mod cast; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -15,42 +17,90 @@ mod tests { fn pco_f32() -> PcoArray { let values = PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_f64() -> PcoArray { let values = PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_i32() -> PcoArray { let values = PrimitiveArray::from_iter([100i32, 200, 300, 400, 500]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_u64() -> PcoArray { let values = PrimitiveArray::from_iter([1000u64, 2000, 3000, 4000]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_i16() -> PcoArray { let values = PrimitiveArray::from_iter([10i16, 20, 30, 40, 50]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_i32_alt() -> PcoArray { let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_single() -> PcoArray { let values = PrimitiveArray::from_iter([42.42f64]); - Pco::from_primitive(values.as_view(), 0, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 0, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } fn pco_large() -> PcoArray { let values = PrimitiveArray::from_iter(0u32..1000); - Pco::from_primitive(values.as_view(), 3, 128).unwrap() + Pco::from_primitive( + values.as_view(), + 3, + 128, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } #[rstest] diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 1f318207aec..294ab45e8e2 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -7,8 +7,6 @@ use std::sync::LazyLock; use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -43,14 +41,14 @@ static SESSION: LazyLock = LazyLock::new(|| { use crate::Pco; #[test] fn test_compress_decompress() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..200).collect(); let array = PrimitiveArray::from_iter(data.clone()); - let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0, &mut ctx).unwrap(); // this data should be compressible assert!(compressed.pages.len() < array.into_array().nbytes() as usize); // check full decompression works - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unsliced_validity = child_to_validity( &compressed.as_ref().slots()[0], compressed.dtype().nullability(), @@ -71,13 +69,13 @@ fn test_compress_decompress() { #[test] fn test_compress_decompress_small() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter([None, Some(1)]); - let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0, &mut ctx).unwrap(); let expected = array.into_array(); assert_arrays_eq!(compressed, expected); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unsliced_validity = child_to_validity( &compressed.as_ref().slots()[0], compressed.dtype().nullability(), @@ -88,10 +86,10 @@ fn test_compress_decompress_small() { #[test] fn test_empty() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = vec![]; let array = PrimitiveArray::from_iter(data.clone()); - let compressed = Pco::from_primitive(array.as_view(), 3, 100).unwrap(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let compressed = Pco::from_primitive(array.as_view(), 3, 100, &mut ctx).unwrap(); let unsliced_validity = child_to_validity( &compressed.as_ref().slots()[0], compressed.dtype().nullability(), @@ -102,6 +100,7 @@ fn test_empty() { #[test] fn test_validity_and_multiple_chunks_and_pages() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..200).collect(); let mut validity: Vec = vec![true; 200]; validity[7..15].fill(false); @@ -121,6 +120,7 @@ fn test_validity_and_multiple_chunks_and_pages() { compression_level, values_per_chunk, values_per_page, + &mut ctx, ) .unwrap(), validity, @@ -141,10 +141,8 @@ fn test_validity_and_multiple_chunks_and_pages() { let slice = compressed.slice(100..103).unwrap(); assert_nth_scalar!(slice, 0, 100); assert_nth_scalar!(slice, 2, 102); - #[expect(deprecated)] - let primitive = slice.to_primitive(); + let primitive = slice.execute::(&mut ctx).unwrap(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert!( primitive .validity() @@ -159,18 +157,19 @@ fn test_validity_and_multiple_chunks_and_pages() { #[test] fn test_validity_vtable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..5).collect(); let mask_bools = vec![false, true, true, false, true]; let array = PrimitiveArray::new( Buffer::from(data), Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); - let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); + let compressed = Pco::from_primitive(array.as_view(), 3, 0, &mut ctx).unwrap(); let arr = compressed.as_array(); assert_eq!( arr.validity() .unwrap() - .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(arr.len(), &mut ctx) .unwrap(), Mask::from_iter(mask_bools) ); @@ -179,7 +178,7 @@ fn test_validity_vtable() { sliced .validity() .unwrap() - .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(sliced.len(), &mut ctx) .unwrap(), Mask::from_iter(vec![true, true, false]) ); @@ -187,8 +186,9 @@ fn test_validity_vtable() { #[test] fn test_serde() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let data: PrimitiveArray = (0i32..1_000_000).collect(); - let pco = Pco::from_primitive(data.as_view(), 3, 100)?.into_array(); + let pco = Pco::from_primitive(data.as_view(), 3, 100, &mut ctx)?.into_array(); let context = ArrayContext::empty(); @@ -213,7 +213,6 @@ fn test_serde() -> VortexResult<()> { &ReadContext::new(context.to_ids()), &SESSION, )?; - let mut ctx = SESSION.create_execution_ctx(); let data_type = data.dtype().to_arrow_dtype()?; let pco_arrow = pco.execute_arrow(Some(&data_type), &mut ctx)?; let decoded_arrow = decoded.execute_arrow(Some(&data_type), &mut ctx)?; diff --git a/encodings/runend/benches/run_end_compress.rs b/encodings/runend/benches/run_end_compress.rs index a177aa25021..1c183f3e5fd 100644 --- a/encodings/runend/benches/run_end_compress.rs +++ b/encodings/runend/benches/run_end_compress.rs @@ -47,8 +47,8 @@ fn compress(bencher: Bencher, (length, run_step): (usize, usize)) { ); bencher - .with_inputs(|| &values) - .bench_refs(|values| runend_encode(values.as_view())); + .with_inputs(|| (&values, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(values, ctx)| runend_encode(values.as_view(), ctx)); } #[divan::bench(types = [u8, u16, u32, u64], args = BENCH_ARGS)] @@ -64,7 +64,7 @@ fn decompress(bencher: Bencher, (length, run_step): (usize, usi .collect::>() .into_array(); - let run_end_array = RunEnd::new(ends, values); + let run_end_array = RunEnd::new(ends, values, &mut LEGACY_SESSION.create_execution_ctx()); let array = run_end_array.into_array(); bencher @@ -88,8 +88,9 @@ fn take_indices(bencher: Bencher, (length, run_step): (usize, usize)) { ); let source_array = PrimitiveArray::from_iter(0..(length as i32)).into_array(); - let (ends, values) = runend_encode(values.as_view()); - let runend_array = RunEnd::try_new(ends.into_array(), values) + let mut encode_ctx = LEGACY_SESSION.create_execution_ctx(); + let (ends, values) = runend_encode(values.as_view(), &mut encode_ctx); + let runend_array = RunEnd::try_new(ends.into_array(), values, &mut encode_ctx) .unwrap() .into_array(); @@ -121,7 +122,7 @@ fn decompress_utf8(bencher: Bencher, (length, run_step): (usize, usize)) { let values = VarBinViewArray::from_iter_str((0..num_runs).map(|i| format!("run_value_{i}"))) .into_array(); - let run_end_array = RunEnd::new(ends, values); + let run_end_array = RunEnd::new(ends, values, &mut LEGACY_SESSION.create_execution_ctx()); let array = run_end_array.into_array(); bencher diff --git a/encodings/runend/benches/run_end_decode.rs b/encodings/runend/benches/run_end_decode.rs index d44611e1e88..818c8eb4ed6 100644 --- a/encodings/runend/benches/run_end_decode.rs +++ b/encodings/runend/benches/run_end_decode.rs @@ -6,6 +6,8 @@ use std::fmt; use divan::Bencher; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; @@ -208,9 +210,15 @@ fn decode_bool(bencher: Bencher, args: BoolBenchArgs) { } = args; let (ends, values) = create_bool_test_data(total_length, avg_run_length, distribution); bencher - .with_inputs(|| (ends.clone(), values.clone())) - .bench_refs(|(ends, values)| { - runend_decode_bools(ends.clone(), values.clone(), 0, total_length) + .with_inputs(|| { + ( + ends.clone(), + values.clone(), + LEGACY_SESSION.create_execution_ctx(), + ) + }) + .bench_refs(|(ends, values, ctx)| { + runend_decode_bools(ends.clone(), values.clone(), 0, total_length, ctx) }); } @@ -371,8 +379,14 @@ fn decode_bool_nullable(bencher: Bencher, args: NullableBoolBenchArgs) { let (ends, values) = create_nullable_bool_test_data(total_length, avg_run_length, distribution, validity); bencher - .with_inputs(|| (ends.clone(), values.clone())) - .bench_refs(|(ends, values)| { - runend_decode_bools(ends.clone(), values.clone(), 0, total_length) + .with_inputs(|| { + ( + ends.clone(), + values.clone(), + LEGACY_SESSION.create_execution_ctx(), + ) + }) + .bench_refs(|(ends, values, ctx)| { + runend_decode_bools(ends.clone(), values.clone(), 0, total_length, ctx) }); } diff --git a/encodings/runend/benches/run_end_null_count.rs b/encodings/runend/benches/run_end_null_count.rs index 6fa318c7192..c60c72c5e0e 100644 --- a/encodings/runend/benches/run_end_null_count.rs +++ b/encodings/runend/benches/run_end_null_count.rs @@ -72,5 +72,5 @@ fn fixture(n: usize, run_step: usize, valid_density: f64) -> RunEndArray { ) .into_array(); - RunEnd::new(ends, values) + RunEnd::new(ends, values, &mut LEGACY_SESSION.create_execution_ctx()) } diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 80d22cb75dc..c768bb61f62 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -2,17 +2,17 @@ pub mod vortex_runend pub mod vortex_runend::compress -pub fn vortex_runend::compress::runend_decode_primitive(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::primitive::vtable::PrimitiveArray, offset: usize, length: usize) -> vortex_error::VortexResult +pub fn vortex_runend::compress::runend_decode_primitive(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::primitive::vtable::PrimitiveArray, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::compress::runend_decode_typed_primitive(run_ends: impl core::iter::traits::iterator::Iterator, values: &[T], values_validity: vortex_mask::Mask, values_nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_array::arrays::primitive::vtable::PrimitiveArray -pub fn vortex_runend::compress::runend_decode_varbinview(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::varbinview::vtable::VarBinViewArray, offset: usize, length: usize) -> vortex_error::VortexResult +pub fn vortex_runend::compress::runend_decode_varbinview(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::varbinview::vtable::VarBinViewArray, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_runend::compress::runend_encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> (vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::array::erased::ArrayRef) +pub fn vortex_runend::compress::runend_encode(array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> (vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::array::erased::ArrayRef) pub mod vortex_runend::decompress_bool -pub fn vortex_runend::decompress_bool::runend_decode_bools(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::bool::vtable::BoolArray, offset: usize, length: usize) -> vortex_error::VortexResult +pub fn vortex_runend::decompress_bool::runend_decode_bools(ends: vortex_array::arrays::primitive::vtable::PrimitiveArray, values: vortex_array::arrays::bool::vtable::BoolArray, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::decompress_bool::runend_decode_typed_bool(run_ends: impl core::iter::traits::iterator::Iterator, values: &vortex_buffer::bit::buf::BitBuffer, values_validity: vortex_mask::Mask, values_nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_array::array::erased::ArrayRef @@ -20,13 +20,13 @@ pub struct vortex_runend::RunEnd impl vortex_runend::RunEnd -pub fn vortex_runend::RunEnd::encode(array: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::encode(array: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_runend::RunEnd::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_runend::RunEndArray +pub fn vortex_runend::RunEnd::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray -pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_runend::RunEndArray +pub unsafe fn vortex_runend::RunEnd::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_runend::RunEndArray -pub fn vortex_runend::RunEnd::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::RunEnd::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult @@ -100,7 +100,7 @@ pub struct vortex_runend::RunEndData impl vortex_runend::RunEndData -pub fn vortex_runend::RunEndData::encode(array: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_runend::RunEndData::encode(array: vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_runend::RunEndData::into_parts(self, ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_runend::RunEndDataParts diff --git a/encodings/runend/src/arbitrary.rs b/encodings/runend/src/arbitrary.rs index 5347935c578..f32a32dbf4a 100644 --- a/encodings/runend/src/arbitrary.rs +++ b/encodings/runend/src/arbitrary.rs @@ -5,6 +5,8 @@ use arbitrary::Arbitrary; use arbitrary::Result; use arbitrary::Unstructured; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::arbitrary::ArbitraryArray; use vortex_array::dtype::DType; @@ -39,11 +41,13 @@ impl ArbitraryRunEndArray { // Number of runs (values/ends pairs) let num_runs = u.int_in_range(0..=20)?; + // TODO(ctx): trait fixes - Arbitrary::arbitrary has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); if num_runs == 0 { // Empty RunEndArray let ends = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values = ArbitraryArray::arbitrary_with(u, Some(0), dtype)?.0; - let runend_array = RunEnd::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values, &mut ctx) .vortex_expect("Empty RunEndArray creation should succeed"); return Ok(ArbitraryRunEndArray(runend_array)); } @@ -55,7 +59,7 @@ impl ArbitraryRunEndArray { // Each end must be > previous end, and first end must be >= 1 let ends = random_strictly_sorted_ends(u, num_runs, len)?; - let runend_array = RunEnd::try_new(ends, values) + let runend_array = RunEnd::try_new(ends, values, &mut ctx) .vortex_expect("RunEndArray creation should succeed in arbitrary impl"); Ok(ArbitraryRunEndArray(runend_array)) diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index be353bf6a2d..a8f99a42e94 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -99,7 +99,9 @@ impl VTable for RunEnd { let values = slots[VALUES_SLOT] .as_ref() .vortex_expect("RunEndArray values slot"); - RunEndData::validate_parts(ends, values, data.offset, len)?; + // TODO(ctx): trait fixes - VTable::validate has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + RunEndData::validate_parts(ends, values, data.offset, len, &mut ctx)?; vortex_ensure!( values.dtype() == dtype, "expected dtype {}, got {}", @@ -254,10 +256,11 @@ impl RunEnd { values: ArrayRef, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> RunEndArray { let dtype = values.dtype().clone(); let slots = vec![Some(ends.clone()), Some(values.clone())]; - RunEndData::validate_parts(&ends, &values, offset, length) + RunEndData::validate_parts(&ends, &values, offset, length, ctx) .vortex_expect("RunEndArray validation failed"); let data = unsafe { RunEndData::new_unchecked(offset) }; unsafe { @@ -268,8 +271,12 @@ impl RunEnd { } /// Build a new [`RunEndArray`] from ends and values. - pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let len = RunEndData::logical_len_from_ends(&ends)?; + pub fn try_new( + ends: ArrayRef, + values: ArrayRef, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let len = RunEndData::logical_len_from_ends(&ends, ctx)?; let dtype = values.dtype().clone(); let slots = vec![Some(ends), Some(values)]; let data = RunEndData::new(0); @@ -290,14 +297,14 @@ impl RunEnd { } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). - pub fn new(ends: ArrayRef, values: ArrayRef) -> RunEndArray { - Self::try_new(ends, values).vortex_expect("RunEndData is always valid") + pub fn new(ends: ArrayRef, values: ArrayRef, ctx: &mut ExecutionCtx) -> RunEndArray { + Self::try_new(ends, values, ctx).vortex_expect("RunEndData is always valid") } /// Run the array through run-end encoding. - pub fn encode(array: ArrayRef) -> VortexResult { + pub fn encode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(parray) = array.as_opt::() { - let (ends, values) = runend_encode(parray); + let (ends, values) = runend_encode(parray, ctx); let ends = ends.into_array(); let len = array.len(); let dtype = values.dtype().clone(); @@ -311,13 +318,11 @@ impl RunEnd { } impl RunEndData { - fn logical_len_from_ends(ends: &ArrayRef) -> VortexResult { + fn logical_len_from_ends(ends: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { if ends.is_empty() { Ok(0) } else { - usize::try_from( - &ends.execute_scalar(ends.len() - 1, &mut LEGACY_SESSION.create_execution_ctx())?, - ) + usize::try_from(&ends.execute_scalar(ends.len() - 1, ctx)?) } } @@ -326,6 +331,7 @@ impl RunEndData { values: &ArrayRef, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { // DType validation vortex_ensure!( @@ -359,10 +365,9 @@ impl RunEndData { // Run ends must be strictly sorted for binary search to work correctly. let pre_validation = ends.statistics().to_owned(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let is_sorted = ends .statistics() - .compute_is_strict_sorted(&mut ctx) + .compute_is_strict_sorted(ctx) .unwrap_or(false); // Preserve the original statistics since compute_is_strict_sorted may have mutated them. @@ -378,17 +383,13 @@ impl RunEndData { // Validate the offset and length are valid for the given ends and values if offset != 0 && length != 0 { - let first_run_end = usize::try_from( - &ends.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, - )?; + let first_run_end = usize::try_from(&ends.execute_scalar(0, ctx)?)?; if first_run_end < offset { vortex_bail!("First run end {first_run_end} must be >= offset {offset}"); } } - let last_run_end = usize::try_from( - &ends.execute_scalar(ends.len() - 1, &mut LEGACY_SESSION.create_execution_ctx())?, - )?; + let last_run_end = usize::try_from(&ends.execute_scalar(ends.len() - 1, ctx)?)?; let min_required_end = offset + length; if last_run_end < min_required_end { vortex_bail!("Last run end {last_run_end} must be >= offset+length {min_required_end}"); @@ -414,12 +415,12 @@ impl RunEndData { /// # use vortex_error::VortexResult; /// # use vortex_runend::RunEnd; /// # fn main() -> VortexResult<()> { + /// let mut ctx = LEGACY_SESSION.create_execution_ctx(); /// let ends = buffer![2u8, 3u8].into_array(); /// let values = BoolArray::from_iter([false, true]).into_array(); - /// let run_end = RunEnd::new(ends, values); + /// let run_end = RunEnd::new(ends, values, &mut ctx); /// /// // Array encodes - /// let mut ctx = LEGACY_SESSION.create_execution_ctx(); /// assert_eq!(run_end.execute_scalar(0, &mut ctx)?, false.into()); /// assert_eq!(run_end.execute_scalar(1, &mut ctx)?, false.into()); /// assert_eq!(run_end.execute_scalar(2, &mut ctx)?, true.into()); @@ -444,9 +445,9 @@ impl RunEndData { } /// Run the array through run-end encoding. - pub fn encode(array: ArrayRef) -> VortexResult { + pub fn encode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { if let Some(parray) = array.as_opt::() { - let (_ends, _values) = runend_encode(parray); + let (_ends, _values) = runend_encode(parray, ctx); // SAFETY: runend_encode handles this unsafe { Ok(Self::new_unchecked(0)) } } else { @@ -468,15 +469,20 @@ impl ValidityVTable for RunEnd { Ok(match array.values().validity()? { Validity::NonNullable | Validity::AllValid => Validity::AllValid, Validity::AllInvalid => Validity::AllInvalid, - Validity::Array(values_validity) => Validity::Array(unsafe { - RunEnd::new_unchecked( - array.ends().clone(), - values_validity, - array.offset(), - array.len(), - ) - .into_array() - }), + Validity::Array(values_validity) => { + // TODO(ctx): trait fixes - ValidityVTable::validity has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + Validity::Array(unsafe { + RunEnd::new_unchecked( + array.ends().clone(), + values_validity, + array.offset(), + array.len(), + &mut ctx, + ) + .into_array() + }) + } }) } } @@ -490,18 +496,18 @@ pub(super) fn run_end_canonicalize( Ok(match array.dtype() { DType::Bool(_) => { let bools = array.values().clone().execute_as("values", ctx)?; - runend_decode_bools(pends, bools, array.offset(), array.len())? + runend_decode_bools(pends, bools, array.offset(), array.len(), ctx)? } DType::Primitive(..) => { let pvalues = array.values().clone().execute_as("values", ctx)?; - runend_decode_primitive(pends, pvalues, array.offset(), array.len())?.into_array() + runend_decode_primitive(pends, pvalues, array.offset(), array.len(), ctx)?.into_array() } DType::Utf8(_) | DType::Binary(_) => { let values = array .values() .clone() .execute_as::("values", ctx)?; - runend_decode_varbinview(pends, values, array.offset(), array.len())?.into_array() + runend_decode_varbinview(pends, values, array.offset(), array.len(), ctx)?.into_array() } _ => vortex_bail!("Unsupported RunEnd value type: {}", array.dtype()), }) @@ -510,6 +516,8 @@ pub(super) fn run_end_canonicalize( #[cfg(test)] mod tests { use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::assert_arrays_eq; @@ -522,9 +530,11 @@ mod tests { #[test] fn test_runend_constructor() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = RunEnd::new( buffer![2u32, 5, 10].into_array(), buffer![1i32, 2, 3].into_array(), + &mut ctx, ); assert_eq!(arr.len(), 10); assert_eq!( @@ -541,8 +551,9 @@ mod tests { #[test] fn test_runend_utf8() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = VarBinViewArray::from_iter_str(["a", "b", "c"]).into_array(); - let arr = RunEnd::new(buffer![2u32, 5, 10].into_array(), values); + let arr = RunEnd::new(buffer![2u32, 5, 10].into_array(), values, &mut ctx); assert_eq!(arr.len(), 10); assert_eq!(arr.dtype(), &DType::Utf8(Nullability::NonNullable)); @@ -554,11 +565,17 @@ mod tests { #[test] fn test_runend_dict() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let dict_values = VarBinViewArray::from_iter_str(["x", "y", "z"]).into_array(); let dict_codes = buffer![0u32, 1, 2].into_array(); let dict = DictArray::try_new(dict_codes, dict_values).unwrap(); - let arr = RunEnd::try_new(buffer![2u32, 5, 10].into_array(), dict.into_array()).unwrap(); + let arr = RunEnd::try_new( + buffer![2u32, 5, 10].into_array(), + dict.into_array(), + &mut ctx, + ) + .unwrap(); assert_eq!(arr.len(), 10); let expected = diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 206a014fa7f..34f8387ff19 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -5,6 +5,8 @@ use arrow_array::RunArray; use arrow_array::types::RunEndIndexType; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrow::FromArrowArray; @@ -53,7 +55,9 @@ where }; // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees. - RunEndData::validate_parts(&ends_slice, &values_slice, offset, len)?; + // TODO(ctx): trait fixes - FromArrowArray::from_arrow has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + RunEndData::validate_parts(&ends_slice, &values_slice, offset, len, &mut ctx)?; Ok(unsafe { RunEndData::new_unchecked(offset) }) } } @@ -76,6 +80,7 @@ mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray as _; + use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute as _; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -331,7 +336,11 @@ mod tests { #[case] expected_ends: &[i32], #[case] expected_values: &[i32], ) -> VortexResult<()> { - let array = RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = RunEnd::encode( + PrimitiveArray::from_iter(input.iter().copied()).into_array(), + &mut ctx, + )?; let sliced = array.into_array().slice(slice_range.clone())?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(sliced, &target)?; diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index be0c30128a2..60c272b6372 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -4,11 +4,8 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; @@ -37,7 +34,10 @@ use vortex_mask::Mask; use crate::iter::trimmed_ends_iter; /// Run-end encode a `PrimitiveArray`, returning a tuple of `(ends, values)`. -pub fn runend_encode(array: ArrayView) -> (PrimitiveArray, ArrayRef) { +pub fn runend_encode( + array: ArrayView, + ctx: &mut ExecutionCtx, +) -> (PrimitiveArray, ArrayRef) { let validity = match array .validity() .vortex_expect("run-end validity should be derivable") @@ -55,8 +55,9 @@ pub fn runend_encode(array: ArrayView) -> (PrimitiveArray, ArrayRef) ); } Validity::Array(a) => { - #[expect(deprecated)] - let bool_array = a.to_bool(); + let bool_array = a + .execute::(ctx) + .vortex_expect("validity array must be convertible to bool"); Some(bool_array.to_bit_buffer()) } }; @@ -183,11 +184,12 @@ pub fn runend_decode_primitive( values: PrimitiveArray, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let validity_mask = values.as_ref().validity()?.to_mask( - values.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity_mask = values + .as_ref() + .validity()? + .to_mask(values.as_ref().len(), ctx)?; Ok(match_each_native_ptype!(values.ptype(), |P| { match_each_unsigned_integer_ptype!(ends.ptype(), |E| { runend_decode_typed_primitive( @@ -286,11 +288,12 @@ pub fn runend_decode_varbinview( values: VarBinViewArray, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let validity_mask = values.as_ref().validity()?.to_mask( - values.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity_mask = values + .as_ref() + .validity()? + .to_mask(values.as_ref().len(), ctx)?; let views = values.views(); let (decoded_views, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| { @@ -314,9 +317,9 @@ pub fn runend_decode_varbinview( } #[cfg(test)] -mod test { - #[expect(deprecated)] - use vortex_array::ToCanonical; +mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; @@ -328,58 +331,62 @@ mod test { use crate::compress::runend_encode; #[test] - fn encode() { + fn encode() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = PrimitiveArray::from_iter([1i32, 1, 2, 2, 2, 3, 3, 3, 3, 3]); - let (ends, values) = runend_encode(arr.as_view()); - #[expect(deprecated)] - let values = values.to_primitive(); + let (ends, values) = runend_encode(arr.as_view(), &mut ctx); + let values = values.execute::(&mut ctx)?; let expected_ends = PrimitiveArray::from_iter(vec![2u8, 5, 10]); assert_arrays_eq!(ends, expected_ends); let expected_values = PrimitiveArray::from_iter(vec![1i32, 2, 3]); assert_arrays_eq!(values, expected_values); + Ok(()) } #[test] - fn encode_nullable() { + fn encode_nullable() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = PrimitiveArray::new( buffer![1i32, 1, 2, 2, 2, 3, 3, 3, 3, 3], Validity::from(BitBuffer::from(vec![ true, true, false, false, true, true, true, true, false, false, ])), ); - let (ends, values) = runend_encode(arr.as_view()); - #[expect(deprecated)] - let values = values.to_primitive(); + let (ends, values) = runend_encode(arr.as_view(), &mut ctx); + let values = values.execute::(&mut ctx)?; let expected_ends = PrimitiveArray::from_iter(vec![2u8, 4, 5, 8, 10]); assert_arrays_eq!(ends, expected_ends); let expected_values = PrimitiveArray::from_option_iter(vec![Some(1i32), None, Some(2), Some(3), None]); assert_arrays_eq!(values, expected_values); + Ok(()) } #[test] - fn encode_all_null() { + fn encode_all_null() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = PrimitiveArray::new( buffer![0, 0, 0, 0, 0], Validity::from(BitBuffer::new_unset(5)), ); - let (ends, values) = runend_encode(arr.as_view()); - #[expect(deprecated)] - let values = values.to_primitive(); + let (ends, values) = runend_encode(arr.as_view(), &mut ctx); + let values = values.execute::(&mut ctx)?; let expected_ends = PrimitiveArray::from_iter(vec![5u64]); assert_arrays_eq!(ends, expected_ends); let expected_values = PrimitiveArray::from_option_iter(vec![Option::::None]); assert_arrays_eq!(values, expected_values); + Ok(()) } #[test] fn decode() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ends = PrimitiveArray::from_iter([2u32, 5, 10]); let values = PrimitiveArray::from_iter([1i32, 2, 3]); - let decoded = runend_decode_primitive(ends, values, 0, 10)?; + let decoded = runend_decode_primitive(ends, values, 0, 10, &mut ctx)?; let expected = PrimitiveArray::from_iter(vec![1i32, 1, 2, 2, 2, 3, 3, 3, 3, 3]); assert_arrays_eq!(decoded, expected); diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index 4e134195d0e..8b5ada03ff3 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; @@ -16,6 +18,8 @@ impl CastReduce for RunEnd { // Cast the values array to the target type let casted_values = array.values().cast(dtype.clone())?; + // TODO(ctx): trait fixes - CastReduce::cast has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // SAFETY: casting does not affect the ends being valid unsafe { Ok(Some( @@ -24,6 +28,7 @@ impl CastReduce for RunEnd { casted_values, array.offset(), array.len(), + &mut ctx, ) .into_array(), )) @@ -36,8 +41,6 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -54,9 +57,11 @@ mod tests { #[test] fn test_cast_runend_i32_to_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let runend = RunEnd::try_new( buffer![3u64, 5, 8, 10].into_array(), buffer![100i32, 200, 100, 300].into_array(), + &mut ctx, ) .unwrap(); @@ -70,53 +75,34 @@ mod tests { ); // Verify by decoding to canonical form - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); // RunEnd encoding should expand to [100, 100, 100, 200, 200, 100, 100, 100, 300, 300] assert_eq!(decoded.len(), 10); assert_eq!( - TryInto::::try_into( - &decoded - .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap(), + TryInto::::try_into(&decoded.execute_scalar(0, &mut ctx).unwrap()).unwrap(), 100i64 ); assert_eq!( - TryInto::::try_into( - &decoded - .execute_scalar(3, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap(), + TryInto::::try_into(&decoded.execute_scalar(3, &mut ctx).unwrap()).unwrap(), 200i64 ); assert_eq!( - TryInto::::try_into( - &decoded - .execute_scalar(5, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap(), + TryInto::::try_into(&decoded.execute_scalar(5, &mut ctx).unwrap()).unwrap(), 100i64 ); assert_eq!( - TryInto::::try_into( - &decoded - .execute_scalar(8, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap(), + TryInto::::try_into(&decoded.execute_scalar(8, &mut ctx).unwrap()).unwrap(), 300i64 ); } #[test] fn test_cast_runend_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let runend = RunEnd::try_new( buffer![2u64, 4, 7].into_array(), PrimitiveArray::from_option_iter([Some(10i32), None, Some(20)]).into_array(), + &mut ctx, ) .unwrap(); @@ -132,10 +118,12 @@ mod tests { #[test] fn test_cast_runend_with_offset() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a RunEndArray: [100, 100, 100, 200, 200, 300, 300, 300, 300, 300] let runend = RunEnd::try_new( buffer![3u64, 5, 10].into_array(), buffer![100i32, 200, 300].into_array(), + &mut ctx, ) .unwrap(); @@ -157,28 +145,37 @@ mod tests { ); } + type RunEndBuilder = fn(&mut vortex_array::ExecutionCtx) -> RunEndArray; + #[rstest] - #[case(RunEnd::try_new( + #[case(|ctx: &mut vortex_array::ExecutionCtx| RunEnd::try_new( buffer![3u64, 5, 8].into_array(), - buffer![100i32, 200, 300].into_array() + buffer![100i32, 200, 300].into_array(), + ctx, ).unwrap())] - #[case(RunEnd::try_new( + #[case(|ctx: &mut vortex_array::ExecutionCtx| RunEnd::try_new( buffer![1u64, 4, 10].into_array(), - buffer![1.5f32, 2.5, 3.5].into_array() + buffer![1.5f32, 2.5, 3.5].into_array(), + ctx, ).unwrap())] - #[case(RunEnd::try_new( + #[case(|ctx: &mut vortex_array::ExecutionCtx| RunEnd::try_new( buffer![2u64, 3, 5].into_array(), - PrimitiveArray::from_option_iter([Some(42i32), None, Some(84)]).into_array() + PrimitiveArray::from_option_iter([Some(42i32), None, Some(84)]).into_array(), + ctx, ).unwrap())] - #[case(RunEnd::try_new( + #[case(|ctx: &mut vortex_array::ExecutionCtx| RunEnd::try_new( buffer![10u64].into_array(), - buffer![255u8].into_array() + buffer![255u8].into_array(), + ctx, ).unwrap())] - #[case(RunEnd::try_new( + #[case(|ctx: &mut vortex_array::ExecutionCtx| RunEnd::try_new( buffer![2u64, 4, 6, 8, 10].into_array(), - BoolArray::from_iter(vec![true, false, true, false, true]).into_array() + BoolArray::from_iter(vec![true, false, true, false, true]).into_array(), + ctx, ).unwrap())] - fn test_cast_runend_conformance(#[case] array: RunEndArray) { + fn test_cast_runend_conformance(#[case] build: RunEndBuilder) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = build(&mut ctx); test_cast_conformance(&array.into_array()); } } diff --git a/encodings/runend/src/compute/compare.rs b/encodings/runend/src/compute/compare.rs index 8dd253e453b..4da7a453000 100644 --- a/encodings/runend/src/compute/compare.rs +++ b/encodings/runend/src/compute/compare.rs @@ -36,6 +36,7 @@ impl CompareKernel for RunEnd { values.execute::(ctx)?, lhs.offset(), lhs.len(), + ctx, ) .map(Some); } @@ -47,7 +48,10 @@ impl CompareKernel for RunEnd { #[cfg(test)] mod test { + use vortex_array::ExecutionCtx; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -58,14 +62,18 @@ mod test { use crate::RunEnd; use crate::RunEndArray; - fn ree_array() -> RunEndArray { - RunEnd::encode(PrimitiveArray::from_iter([1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5]).into_array()) - .unwrap() + fn ree_array(ctx: &mut ExecutionCtx) -> RunEndArray { + RunEnd::encode( + PrimitiveArray::from_iter([1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5]).into_array(), + ctx, + ) + .unwrap() } #[test] fn compare_run_end() { - let arr = ree_array(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let arr = ree_array(&mut ctx); let res = arr .into_array() .binary(ConstantArray::new(5, 12).into_array(), Operator::Eq) diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index ddce31184eb..a4597950b95 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -4,6 +4,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; @@ -18,6 +20,8 @@ impl FillNullReduce for RunEnd { fill_value: &Scalar, ) -> VortexResult> { let new_values = array.values().fill_null(fill_value.clone())?; + // TODO(ctx): trait fixes - FillNullReduce::fill_null has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // SAFETY: modifying values only, does not affect ends Ok(Some( unsafe { @@ -26,6 +30,7 @@ impl FillNullReduce for RunEnd { new_values, array.offset(), array.len(), + &mut ctx, ) } .into_array(), diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index 23c72e4b6d2..7e0a2af29b0 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -42,6 +42,7 @@ impl FilterKernel for RunEnd { array, mask_values.indices(), &Validity::NonNullable, + ctx, )?)) } else { let primitive_run_ends = array.ends().clone().execute::(ctx)?; @@ -64,6 +65,7 @@ impl FilterKernel for RunEnd { values, 0, mask_values.true_count(), + ctx, ) .into_array(), )) @@ -126,8 +128,11 @@ mod tests { use crate::RunEndArray; fn ree_array() -> RunEndArray { - RunEnd::encode(PrimitiveArray::from_iter([1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5]).into_array()) - .unwrap() + RunEnd::encode( + PrimitiveArray::from_iter([1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5]).into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } #[test] @@ -135,11 +140,13 @@ mod tests { let arr = ree_array().slice(2..7).unwrap(); let filtered = arr.filter(Mask::from_iter([true, false, false, true, true]))?; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_arrays_eq!( filtered, RunEnd::new( PrimitiveArray::from_iter([1u8, 2, 3]).into_array(), - PrimitiveArray::from_iter([1i32, 4, 2]).into_array() + PrimitiveArray::from_iter([1i32, 4, 2]).into_array(), + &mut ctx, ) ); Ok(()) @@ -157,7 +164,8 @@ mod tests { .iter() .flat_map(|&v| std::iter::repeat_n(v, 32)) .collect(); - let arr = RunEnd::encode(PrimitiveArray::from_iter(values).into_array())?; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let arr = RunEnd::encode(PrimitiveArray::from_iter(values).into_array(), &mut ctx)?; // Slice off the first 16 rows. Slice(RunEnd), 112 rows, 4 runs. let sliced = arr.into_array().slice(16..128)?; @@ -166,7 +174,6 @@ mod tests { let mask = Mask::from_iter((0..sliced.len()).map(|i| i % 2 == 0)); let filtered = sliced.filter(mask)?; - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let executed = filtered.execute_until::(&mut ctx)?; assert_eq!( executed.encoding_id().as_ref(), diff --git a/encodings/runend/src/compute/mod.rs b/encodings/runend/src/compute/mod.rs index f1f6dd97593..9bdb6d67c00 100644 --- a/encodings/runend/src/compute/mod.rs +++ b/encodings/runend/src/compute/mod.rs @@ -15,6 +15,8 @@ pub(crate) mod take_from; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::consistency::test_array_consistency; use vortex_buffer::buffer; @@ -25,24 +27,30 @@ mod tests { #[rstest] // Simple run-end arrays #[case::runend_i32(RunEnd::encode( - buffer![1i32, 1, 1, 2, 2, 3, 3, 3, 3].into_array() + buffer![1i32, 1, 1, 2, 2, 3, 3, 3, 3].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] #[case::runend_single_run(RunEnd::encode( - buffer![5i32, 5, 5, 5, 5].into_array() + buffer![5i32, 5, 5, 5, 5].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] #[case::runend_alternating(RunEnd::encode( - buffer![1i32, 2, 1, 2, 1, 2].into_array() + buffer![1i32, 2, 1, 2, 1, 2].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] // Different types #[case::runend_u64(RunEnd::encode( - buffer![100u64, 100, 200, 200, 200].into_array() + buffer![100u64, 100, 200, 200, 200].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] // Edge cases #[case::runend_single(RunEnd::encode( - buffer![42i32].into_array() + buffer![42i32].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] #[case::runend_large(RunEnd::encode( - PrimitiveArray::from_iter((0..1000).map(|i| i / 10)).into_array() + PrimitiveArray::from_iter((0..1000).map(|i| i / 10)).into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] fn test_runend_consistency(#[case] array: RunEndArray) { diff --git a/encodings/runend/src/compute/take.rs b/encodings/runend/src/compute/take.rs index cf346ff81b3..7100faf9eac 100644 --- a/encodings/runend/src/compute/take.rs +++ b/encodings/runend/src/compute/take.rs @@ -7,8 +7,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::TakeExecute; use vortex_array::match_each_integer_ptype; @@ -51,7 +49,7 @@ impl TakeExecute for RunEnd { }); let indices_validity = primitive_indices.validity()?; - take_indices_unchecked(array, &checked_indices, &indices_validity).map(Some) + take_indices_unchecked(array, &checked_indices, &indices_validity, ctx).map(Some) } } @@ -60,9 +58,9 @@ pub fn take_indices_unchecked>( array: ArrayView<'_, RunEnd>, indices: &[T], validity: &Validity, + ctx: &mut ExecutionCtx, ) -> VortexResult { - #[expect(deprecated)] - let ends = array.ends().to_primitive(); + let ends = array.ends().clone().execute::(ctx)?; let ends_len = ends.len(); // TODO(joe): use the validity mask to skip search sorted. @@ -91,7 +89,7 @@ pub fn take_indices_unchecked>( } #[cfg(test)] -mod test { +mod tests { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::Canonical; @@ -107,7 +105,11 @@ mod test { use crate::RunEndArray; fn ree_array() -> RunEndArray { - RunEnd::encode(buffer![1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5].into_array()).unwrap() + RunEnd::encode( + buffer![1, 1, 1, 4, 4, 4, 2, 2, 5, 5, 5, 5].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() } #[test] @@ -157,6 +159,7 @@ mod test { #[case(ree_array())] #[case(RunEnd::encode( buffer![1u8, 1, 2, 2, 2, 3, 3, 3, 3, 4].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] #[case(RunEnd::encode( PrimitiveArray::from_option_iter([ @@ -169,11 +172,14 @@ mod test { Some(20), ]) .into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] - #[case(RunEnd::encode(buffer![42i32, 42, 42, 42, 42].into_array()) + #[case(RunEnd::encode(buffer![42i32, 42, 42, 42, 42].into_array(), + &mut LEGACY_SESSION.create_execution_ctx()) .unwrap())] #[case(RunEnd::encode( buffer![1i32, 2, 3, 4, 5, 6, 7, 8, 9, 10].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ).unwrap())] #[case({ let mut values = Vec::new(); @@ -182,7 +188,11 @@ mod test { values.push(i); } } - RunEnd::encode(PrimitiveArray::from_iter(values).into_array()).unwrap() + RunEnd::encode( + PrimitiveArray::from_iter(values).into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() })] fn test_take_runend_conformance(#[case] array: RunEndArray) { test_take_conformance(&array.into_array()); @@ -193,6 +203,7 @@ mod test { #[case({ let array = RunEnd::encode( buffer![1i32, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3].into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(); array.slice(2..8).unwrap() diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs index 52fbe7ddcc0..57bc207a385 100644 --- a/encodings/runend/src/compute/take_from.rs +++ b/encodings/runend/src/compute/take_from.rs @@ -25,7 +25,7 @@ impl ExecuteParentKernel for RunEndTakeFrom { array: ArrayView<'_, RunEnd>, dict: ArrayView<'_, Dict>, child_idx: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { if child_idx != 0 { return Ok(None); @@ -44,6 +44,7 @@ impl ExecuteParentKernel for RunEndTakeFrom { dict.values().take(array.values().clone())?, array.offset(), array.len(), + ctx, ) }; // @@ -53,6 +54,7 @@ impl ExecuteParentKernel for RunEndTakeFrom { #[cfg(test)] mod tests { + use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::DictArray; @@ -74,8 +76,8 @@ mod tests { /// Dict values: `[2, 3]` /// Codes: `[0, 0, 0, 1, 1, 0, 0]` /// RunEnd encoded codes: ends=`[3, 5, 7]`, values=`[0, 1, 0]` - fn make_dict_with_runend_codes() -> (RunEndArray, DictArray) { - let codes = RunEnd::encode(buffer![0u32, 0, 0, 1, 1, 0, 0].into_array()).unwrap(); + fn make_dict_with_runend_codes(ctx: &mut ExecutionCtx) -> (RunEndArray, DictArray) { + let codes = RunEnd::encode(buffer![0u32, 0, 0, 1, 1, 0, 0].into_array(), ctx).unwrap(); let values = buffer![2i32, 3].into_array(); let dict = DictArray::try_new(codes.clone().into_array(), values).unwrap(); (codes, dict) @@ -83,23 +85,23 @@ mod tests { #[test] fn test_execute_parent_no_offset() -> VortexResult<()> { - let (codes, dict) = make_dict_with_runend_codes(); let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let (codes, dict) = make_dict_with_runend_codes(&mut ctx); let result = RunEndTakeFrom .execute_parent(codes.as_view(), dict.as_view(), 0, &mut ctx)? .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([2i32, 2, 2, 3, 3, 2, 2]); - #[expect(deprecated)] - let canonical = result.to_canonical()?.into_array(); + let canonical = result.execute::(&mut ctx)?.into_array(); assert_arrays_eq!(canonical, expected); Ok(()) } #[test] fn test_execute_parent_with_offset() -> VortexResult<()> { - let (codes, dict) = make_dict_with_runend_codes(); + let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice codes to positions 2..5 → logical codes [0, 1, 1] → values [2, 3, 3] let sliced_codes = unsafe { RunEnd::new_unchecked( @@ -107,24 +109,24 @@ mod tests { codes.values().clone(), 2, // offset 3, // len + &mut ctx, ) }; - let mut ctx = ExecutionCtx::new(VortexSession::empty()); let result = RunEndTakeFrom .execute_parent(sliced_codes.as_view(), dict.as_view(), 0, &mut ctx)? .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([2i32, 3, 3]); - #[expect(deprecated)] - let canonical = result.to_canonical()?.into_array(); + let canonical = result.execute::(&mut ctx)?.into_array(); assert_arrays_eq!(canonical, expected); Ok(()) } #[test] fn test_execute_parent_offset_at_run_boundary() -> VortexResult<()> { - let (codes, dict) = make_dict_with_runend_codes(); + let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice codes to positions 3..7 → logical codes [1, 1, 0, 0] → values [3, 3, 2, 2] let sliced_codes = unsafe { RunEnd::new_unchecked( @@ -132,24 +134,24 @@ mod tests { codes.values().clone(), 3, // offset at exact run boundary 4, // len + &mut ctx, ) }; - let mut ctx = ExecutionCtx::new(VortexSession::empty()); let result = RunEndTakeFrom .execute_parent(sliced_codes.as_view(), dict.as_view(), 0, &mut ctx)? .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([3i32, 3, 2, 2]); - #[expect(deprecated)] - let canonical = result.to_canonical()?.into_array(); + let canonical = result.execute::(&mut ctx)?.into_array(); assert_arrays_eq!(canonical, expected); Ok(()) } #[test] fn test_execute_parent_single_element_offset() -> VortexResult<()> { - let (codes, dict) = make_dict_with_runend_codes(); + let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice to single element at position 4 → code=1 → value=3 let sliced_codes = unsafe { RunEnd::new_unchecked( @@ -157,25 +159,24 @@ mod tests { codes.values().slice(1..3)?, 4, // offset 1, // len + &mut ctx, ) }; - let mut ctx = ExecutionCtx::new(VortexSession::empty()); let result = RunEndTakeFrom .execute_parent(sliced_codes.as_view(), dict.as_view(), 0, &mut ctx)? .expect("kernel should return Some"); let expected = PrimitiveArray::from_iter([3i32]); - #[expect(deprecated)] - let canonical = result.to_canonical()?.into_array(); + let canonical = result.execute::(&mut ctx)?.into_array(); assert_arrays_eq!(canonical, expected); Ok(()) } #[test] fn test_execute_parent_returns_none_for_non_codes_child() -> VortexResult<()> { - let (codes, dict) = make_dict_with_runend_codes(); let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let (codes, dict) = make_dict_with_runend_codes(&mut ctx); let result = RunEndTakeFrom.execute_parent(codes.as_view(), dict.as_view(), 1, &mut ctx)?; assert!(result.is_none()); diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index e10c314dc30..c0ccd8c4820 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -8,9 +8,8 @@ use itertools::Itertools; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -37,11 +36,12 @@ pub fn runend_decode_bools( values: BoolArray, offset: usize, length: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - let validity = values.as_ref().validity()?.to_mask( - values.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity = values + .as_ref() + .validity()? + .to_mask(values.as_ref().len(), ctx)?; let values_buf = values.to_bit_buffer(); let nullability = values.dtype().nullability(); @@ -247,8 +247,6 @@ fn decode_nullable_sequential( #[cfg(test)] mod tests { use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -262,10 +260,11 @@ mod tests { #[test] fn decode_bools_alternating() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Alternating true/false: [T, T, F, F, F, T, T, T, T, T] let ends = PrimitiveArray::from_iter([2u32, 5, 10]); let values = BoolArray::from(BitBuffer::from(vec![true, false, true])); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![ true, true, false, false, false, true, true, true, true, true, @@ -276,10 +275,11 @@ mod tests { #[test] fn decode_bools_mostly_true() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Mostly true: [T, T, T, T, T, F, T, T, T, T] let ends = PrimitiveArray::from_iter([5u32, 6, 10]); let values = BoolArray::from(BitBuffer::from(vec![true, false, true])); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![ true, true, true, true, true, false, true, true, true, true, @@ -290,10 +290,11 @@ mod tests { #[test] fn decode_bools_mostly_false() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Mostly false: [F, F, F, F, F, T, F, F, F, F] let ends = PrimitiveArray::from_iter([5u32, 6, 10]); let values = BoolArray::from(BitBuffer::from(vec![false, true, false])); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![ false, false, false, false, false, true, false, false, false, false, @@ -304,9 +305,10 @@ mod tests { #[test] fn decode_bools_all_true_single_run() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ends = PrimitiveArray::from_iter([10u32]); let values = BoolArray::from(BitBuffer::from(vec![true])); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![ true, true, true, true, true, true, true, true, true, true, @@ -317,9 +319,10 @@ mod tests { #[test] fn decode_bools_all_false_single_run() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ends = PrimitiveArray::from_iter([10u32]); let values = BoolArray::from(BitBuffer::from(vec![false])); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![ false, false, false, false, false, false, false, false, false, false, @@ -330,10 +333,11 @@ mod tests { #[test] fn decode_bools_with_offset() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test with offset: [T, T, F, F, F, T, T, T, T, T] -> slice [2..8] = [F, F, F, T, T, T] let ends = PrimitiveArray::from_iter([2u32, 5, 10]); let values = BoolArray::from(BitBuffer::from(vec![true, false, true])); - let decoded = runend_decode_bools(ends, values, 2, 6)?; + let decoded = runend_decode_bools(ends, values, 2, 6, &mut ctx)?; let expected = BoolArray::from(BitBuffer::from(vec![false, false, false, true, true, true])); @@ -345,13 +349,14 @@ mod tests { fn decode_bools_nullable() -> VortexResult<()> { use vortex_array::validity::Validity; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // 3 runs: T (valid), F (null), T (valid) -> [T, T, null, null, null, T, T, T, T, T] let ends = PrimitiveArray::from_iter([2u32, 5, 10]); let values = BoolArray::new( BitBuffer::from(vec![true, false, true]), Validity::from(BitBuffer::from(vec![true, false, true])), ); - let decoded = runend_decode_bools(ends, values, 0, 10)?; + let decoded = runend_decode_bools(ends, values, 0, 10, &mut ctx)?; // Expected: values=[T, T, F, F, F, T, T, T, T, T], validity=[1, 1, 0, 0, 0, 1, 1, 1, 1, 1] let expected = BoolArray::new( @@ -368,14 +373,15 @@ mod tests { #[test] fn decode_bools_nullable_few_runs() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test few runs (uses fast path): 5 runs of length 2000 each let ends = PrimitiveArray::from_iter([2000u32, 4000, 6000, 8000, 10000]); let values = BoolArray::new( BitBuffer::from(vec![true, false, true, false, true]), Validity::from(BitBuffer::from(vec![true, false, true, false, true])), ); - #[expect(deprecated)] - let decoded = runend_decode_bools(ends, values, 0, 10000)?.to_bool(); + let decoded = runend_decode_bools(ends, values, 0, 10000, &mut ctx)? + .execute::(&mut ctx)?; // Check length and a few values assert_eq!(decoded.len(), 10000); @@ -384,10 +390,7 @@ mod tests { decoded .as_ref() .validity()? - .to_mask( - decoded.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(0) ); @@ -397,10 +400,7 @@ mod tests { !decoded .as_ref() .validity()? - .to_mask( - decoded.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(2000) ); @@ -409,10 +409,7 @@ mod tests { decoded .as_ref() .validity()? - .to_mask( - decoded.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(4000) ); diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 9480e9e6f04..f619dae3d67 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -7,8 +7,6 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Slice; use vortex_array::arrays::dict::TakeExecuteAdaptor; @@ -45,13 +43,17 @@ impl ExecuteParentKernel for RunEndSliceKernel { array: ArrayView<'_, RunEnd>, parent: ArrayView<'_, Slice>, _child_idx: usize, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { - slice(array, parent.slice_range().clone()).map(Some) + slice(array, parent.slice_range().clone(), ctx).map(Some) } } -fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult { +fn slice( + array: ArrayView<'_, RunEnd>, + range: Range, + ctx: &mut ExecutionCtx, +) -> VortexResult { let new_length = range.len(); let slice_begin = array.find_physical_index(range.start)?; @@ -59,9 +61,7 @@ fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult, range: Range) -> VortexResult for RunEndScalarFnRule { ScalarFnArray::try_new(parent.scalar_fn().clone(), new_children, values_len)? .into_array(); + // TODO(ctx): trait fixes - ArrayParentReduceRule::reduce_parent has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); Ok(Some( unsafe { RunEnd::new_unchecked( @@ -87,6 +91,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { new_values, run_end.offset(), run_end.len(), + &mut ctx, ) } .into_array(), diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index 455a2fd9809..0a9f3b117bd 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -122,6 +122,6 @@ pub vortex_sequence::SequenceDataParts::ptype: vortex_array::dtype::ptype::PType pub fn vortex_sequence::initialize(session: &vortex_session::VortexSession) -pub fn vortex_sequence::sequence_encode(primitive_array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>) -> vortex_error::VortexResult> +pub fn vortex_sequence::sequence_encode(primitive_array: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> pub type vortex_sequence::SequenceArray = vortex_array::array::typed::Array diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index 2baaba5df5b..9f5e62a5217 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -7,9 +7,8 @@ use num_traits::CheckedAdd; use num_traits::CheckedSub; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; @@ -90,16 +89,14 @@ pub fn sequence_decompress(array: &SequenceArray) -> VortexResult { /// we might want a different array for that since sequence provide fast access. pub fn sequence_encode( primitive_array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, ) -> VortexResult> { if primitive_array.is_empty() { // we cannot encode an empty array return Ok(None); } - if !primitive_array - .array() - .all_valid(&mut LEGACY_SESSION.create_execution_ctx())? - { + if !primitive_array.array().all_valid(ctx)? { return Ok(None); } @@ -153,8 +150,8 @@ fn encode_primitive_array + CheckedAdd + CheckedSu mod tests { #[expect(unused_imports)] use itertools::Itertools; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -162,47 +159,58 @@ mod tests { #[test] fn test_encode_array_success() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive_array = PrimitiveArray::from_iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let encoded = sequence_encode(primitive_array.as_view()).unwrap(); + let encoded = sequence_encode(primitive_array.as_view(), &mut ctx).unwrap(); assert!(encoded.is_some()); - #[expect(deprecated)] - let decoded = encoded.unwrap().to_primitive(); + let decoded = encoded + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(decoded, primitive_array); } #[test] fn test_encode_array_1_success() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive_array = PrimitiveArray::from_iter([0]); - let encoded = sequence_encode(primitive_array.as_view()).unwrap(); + let encoded = sequence_encode(primitive_array.as_view(), &mut ctx).unwrap(); assert!(encoded.is_some()); - #[expect(deprecated)] - let decoded = encoded.unwrap().to_primitive(); + let decoded = encoded + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(decoded, primitive_array); } #[test] fn test_encode_array_fail() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive_array = PrimitiveArray::from_iter([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); - let encoded = sequence_encode(primitive_array.as_view()).unwrap(); + let encoded = sequence_encode(primitive_array.as_view(), &mut ctx).unwrap(); assert!(encoded.is_none()); } #[test] fn test_encode_array_fail_oob() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive_array = PrimitiveArray::from_iter(vec![100i8; 1000]); - let encoded = sequence_encode(primitive_array.as_view()).unwrap(); + let encoded = sequence_encode(primitive_array.as_view(), &mut ctx).unwrap(); assert!(encoded.is_none()); } #[test] fn test_encode_all_u8_values() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive_array = PrimitiveArray::from_iter(0u8..=255); - let encoded = sequence_encode(primitive_array.as_view()).unwrap(); + let encoded = sequence_encode(primitive_array.as_view(), &mut ctx).unwrap(); assert!(encoded.is_some()); - #[expect(deprecated)] - let decoded = encoded.unwrap().to_primitive(); + let decoded = encoded + .unwrap() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(decoded, primitive_array); } } diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index b53e7188c52..c532689a275 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -89,8 +89,8 @@ impl CastReduce for Sequence { mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -119,6 +119,7 @@ mod tests { #[test] fn test_cast_sequence_u32_to_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let sequence = Sequence::try_new_typed(100u32, 10u32, Nullability::NonNullable, 4).unwrap(); let casted = sequence @@ -131,13 +132,13 @@ mod tests { ); // Verify the values - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([100i64, 110, 120, 130])); } #[test] fn test_cast_sequence_i16_to_i32_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test ptype change AND nullability change in one cast let sequence = Sequence::try_new_typed(5i16, 3i16, Nullability::NonNullable, 3).unwrap(); @@ -151,8 +152,7 @@ mod tests { ); // Verify the values - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!( decoded, PrimitiveArray::from_option_iter([Some(5i32), Some(8), Some(11)]) @@ -161,6 +161,7 @@ mod tests { #[test] fn test_cast_sequence_to_float_delegates_to_canonical() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let sequence = Sequence::try_new_typed(0i32, 1i32, Nullability::NonNullable, 5).unwrap(); // Cast to float should delegate to canonical (SequenceArray doesn't support float) @@ -175,8 +176,7 @@ mod tests { ); // Verify the values were correctly converted - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!( decoded, PrimitiveArray::from_iter([0.0f32, 1.0, 2.0, 3.0, 4.0]) diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index 9bcfb086753..f93e853b8b9 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -4,7 +4,7 @@ pub struct vortex_sparse::Sparse impl vortex_sparse::Sparse -pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayRef, fill_value: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayRef, fill_value: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sparse::Sparse::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult @@ -82,7 +82,7 @@ impl vortex_sparse::SparseData pub fn vortex_sparse::SparseData::dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_sparse::SparseData::encode(array: &vortex_array::array::erased::ArrayRef, fill_value: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_sparse::SparseData::encode(array: &vortex_array::array::erased::ArrayRef, fill_value: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_sparse::SparseData::fill_scalar(&self) -> &vortex_array::scalar::Scalar diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 024975a4e76..5786d718a77 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -8,8 +8,6 @@ use num_traits::NumCast; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; @@ -165,11 +163,13 @@ fn execute_sparse_lists( array.len(), total_canonical_values, validity, + ctx, ) }) })) } +#[expect(clippy::too_many_arguments)] fn execute_sparse_lists_inner( patch_indices: &[I], patch_values: ListViewArray, @@ -178,6 +178,7 @@ fn execute_sparse_lists_inner( len: usize, total_canonical_values: usize, validity: Validity, + ctx: &mut ExecutionCtx, ) -> ArrayRef { // Create the builder with appropriate types. It is easy to just use the same type for both // `offsets` and `sizes` since we have no other constraints. @@ -204,7 +205,7 @@ fn execute_sparse_lists_inner( builder .append_value( patch_values - .execute_scalar(patch_idx, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(patch_idx, ctx) .vortex_expect("scalar_at") .as_list(), ) @@ -250,6 +251,7 @@ fn execute_sparse_fixed_size_list( fill_value, array.len(), validity, + ctx, ) .into_array() })) @@ -267,6 +269,7 @@ fn execute_sparse_fixed_size_list_inner( fill_value: ListScalar, array_len: usize, validity: Validity, + ctx: &mut ExecutionCtx, ) -> FixedSizeListArray { let list_size = values.list_size(); let element_dtype = values.elements().dtype(); @@ -300,11 +303,7 @@ fn execute_sparse_fixed_size_list_inner( .vortex_expect("fixed_size_list_elements_at"); for i in 0..list_size as usize { builder - .append_scalar( - &patch_list - .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("scalar_at"), - ) + .append_scalar(&patch_list.execute_scalar(i, ctx).vortex_expect("scalar_at")) .vortex_expect("element dtype must match"); } } else { @@ -566,8 +565,6 @@ mod test { use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -605,11 +602,15 @@ mod test { #[case(Some(false))] #[case(None)] fn test_sparse_bool(#[case] fill_value: Option) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let indices = buffer![0u64, 1, 7].into_array(); let values = BoolArray::from_iter([Some(true), None, Some(false)]).into_array(); let sparse_bools = Sparse::try_new(indices, values, 10, Scalar::from(fill_value)).unwrap(); - #[expect(deprecated)] - let actual = sparse_bools.as_array().to_bool(); + let actual = sparse_bools + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = BoolArray::from_iter([ Some(true), @@ -632,13 +633,17 @@ mod test { #[case(Some(-1i32))] #[case(None)] fn test_sparse_primitive(#[case] fill_value: Option) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let indices = buffer![0u64, 1, 7].into_array(); let values = PrimitiveArray::from_option_iter([Some(0i32), None, Some(1)]).into_array(); let sparse_ints = Sparse::try_new(indices, values, 10, Scalar::from(fill_value)).unwrap(); assert_eq!(*sparse_ints.dtype(), DType::Primitive(PType::I32, Nullable)); - #[expect(deprecated)] - let flat_ints = sparse_ints.as_array().to_primitive(); + let flat_ints = sparse_ints + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = PrimitiveArray::from_option_iter([ Some(0i32), None, @@ -657,6 +662,7 @@ mod test { #[test] fn test_sparse_struct_valid_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let field_names = FieldNames::from_iter(["a", "b"]); let field_types = vec![ DType::Primitive(PType::I32, Nullable), @@ -721,13 +727,17 @@ mod test { .unwrap() .into_array(); - #[expect(deprecated)] - let actual = sparse_struct.as_array().to_struct(); + let actual = sparse_struct + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(actual, expected); } #[test] fn test_sparse_struct_invalid_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let field_names = FieldNames::from_iter(["a", "b"]); let field_types = vec![ DType::Primitive(PType::I32, Nullable), @@ -789,13 +799,17 @@ mod test { .unwrap() .into_array(); - #[expect(deprecated)] - let actual = sparse_struct.as_array().to_struct(); + let actual = sparse_struct + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(actual, expected); } #[test] fn test_sparse_decimal() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let indices = buffer![0u32, 1u32, 7u32, 8u32].into_array(); let decimal_dtype = DecimalDType::new(3, 2); let patch_values = DecimalArray::new( @@ -818,10 +832,11 @@ mod test { .into_arrow_preferred() .unwrap(); - #[expect(deprecated)] let actual = sparse_struct .as_array() - .to_decimal() + .clone() + .execute::(&mut ctx) + .unwrap() .into_array() .into_arrow_preferred() .unwrap(); @@ -832,6 +847,7 @@ mod test { #[test] fn test_sparse_utf8_varbinview_non_null_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = >::from_iter([ Some("hello"), Some("goodbye"), @@ -851,8 +867,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = >::from_iter([ Some("hello"), Some("123"), @@ -874,6 +894,7 @@ mod test { #[test] fn test_sparse_utf8_varbinview_null_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = >::from_iter([ Some("hello"), Some("goodbye"), @@ -893,8 +914,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = >::from_iter([ Some("hello"), None, @@ -916,6 +941,7 @@ mod test { #[test] fn test_sparse_utf8_varbinview_non_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = VarBinViewArray::from_iter_str(["hello", "goodbye", "hello", "bonjour", "你好"]) .into_array(); @@ -928,8 +954,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = VarBinViewArray::from_iter_str([ "hello", "123", "123", "goodbye", "hello", "bonjour", "123", "123", "你好", ]) @@ -940,6 +970,7 @@ mod test { #[test] fn test_sparse_utf8_varbin_null_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = >::from_iter([ Some("hello"), Some("goodbye"), @@ -959,8 +990,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = >::from_iter([ Some("hello"), None, @@ -982,6 +1017,7 @@ mod test { #[test] fn test_sparse_binary_varbinview_non_null_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let binaries = VarBinViewArray::from_iter_nullable_bin([ Some(b"hello" as &[u8]), Some(b"goodbye"), @@ -1001,8 +1037,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = VarBinViewArray::from_iter_nullable_bin([ Some(b"hello" as &[u8]), Some(b"123"), @@ -1024,6 +1064,7 @@ mod test { #[test] fn test_sparse_list_null_fill() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Use ListViewArray consistently let elements = buffer![1i32, 2, 1, 2].into_array(); // Create ListView with offsets and sizes @@ -1045,11 +1086,8 @@ mod test { .unwrap() .into_array(); - let actual = sparse - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? - .into_array(); - #[expect(deprecated)] - let result_listview = actual.to_listview(); + let actual = sparse.execute::(&mut ctx)?.into_array(); + let result_listview = actual.execute::(&mut ctx)?; // Check the structure assert_eq!(result_listview.len(), 6); @@ -1063,8 +1101,10 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] // Verify actual values - #[expect(deprecated)] - let elements_array = result_listview.elements().to_primitive(); + let elements_array = result_listview + .elements() + .clone() + .execute::(&mut ctx)?; let elements_slice = elements_array.as_slice::(); let list0_offset = result_listview.offset_at(0); @@ -1084,6 +1124,7 @@ mod test { #[test] fn test_sparse_list_null_fill_sliced_sparse_values() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create ListViewArray with 8 elements forming 8 single-element lists let elements = buffer![1i32, 2, 1, 2, 1, 2, 1, 2].into_array(); let offsets = buffer![0u32, 1, 2, 3, 4, 5, 6, 7].into_array(); @@ -1104,11 +1145,12 @@ mod test { .into_array(); let actual = sparse - .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .execute::(&mut ctx) .vortex_expect("no fail") .into_array(); - #[expect(deprecated)] - let result_listview = actual.to_listview(); + let result_listview = actual + .execute::(&mut ctx) + .vortex_expect("no fail"); // Check the structure assert_eq!(result_listview.len(), 6); @@ -1122,8 +1164,11 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] - extra element beyond original slice // Verify actual values - #[expect(deprecated)] - let elements_array = result_listview.elements().to_primitive(); + let elements_array = result_listview + .elements() + .clone() + .execute::(&mut ctx) + .unwrap(); let elements_slice = elements_array.as_slice::(); let list0_offset = result_listview.offset_at(0); @@ -1135,6 +1180,7 @@ mod test { #[test] fn test_sparse_list_non_null_fill() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create ListViewArray with 4 single-element lists let elements = buffer![1i32, 2, 1, 2].into_array(); let offsets = buffer![0u32, 1, 2, 3].into_array(); @@ -1151,11 +1197,8 @@ mod test { .unwrap() .into_array(); - let actual = sparse - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? - .into_array(); - #[expect(deprecated)] - let result_listview = actual.to_listview(); + let actual = sparse.execute::(&mut ctx)?.into_array(); + let result_listview = actual.execute::(&mut ctx)?; // Check the structure assert_eq!(result_listview.len(), 6); @@ -1169,8 +1212,10 @@ mod test { assert_eq!(result_listview.size_at(5), 1); // [2] from sparse // Verify actual values - #[expect(deprecated)] - let elements_array = result_listview.elements().to_primitive(); + let elements_array = result_listview + .elements() + .clone() + .execute::(&mut ctx)?; let elements_slice = elements_array.as_slice::(); // List 0: [1] @@ -1209,6 +1254,7 @@ mod test { #[test] fn test_sparse_binary_varbin_null_fill() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let strings = >::from_iter([ Some(b"hello" as &[u8]), Some(b"goodbye"), @@ -1228,8 +1274,12 @@ mod test { ) .unwrap(); - #[expect(deprecated)] - let actual = array.as_array().to_varbinview().into_array(); + let actual = array + .as_array() + .clone() + .execute::(&mut ctx) + .unwrap() + .into_array(); let expected = VarBinViewArray::from_iter_nullable_bin([ Some(b"hello" as &[u8]), None, @@ -1472,6 +1522,7 @@ mod test { #[test] fn test_sparse_list_grows_offset_type() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let elements = buffer![1i32, 2, 1, 2].into_array(); let offsets = buffer![0u8, 1, 2, 3, 4].into_array(); let lists = ListArray::try_new(elements, offsets, Validity::AllValid) @@ -1484,9 +1535,7 @@ mod test { .unwrap() .into_array(); - let actual = sparse - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? - .into_array(); + let actual = sparse.execute::(&mut ctx)?.into_array(); let mut expected_elements = buffer_mut![1, 2, 1, 2]; expected_elements.extend(buffer![42i32; 252]); let expected = ListArray::try_new( @@ -1497,8 +1546,7 @@ mod test { .unwrap() .into_array(); - #[expect(deprecated)] - let actual_listview = actual.to_listview(); + let actual_listview = actual.clone().execute::(&mut ctx)?; assert_eq!( actual_listview.offsets().dtype(), &DType::Primitive(PType::U16, NonNullable) @@ -1516,6 +1564,7 @@ mod test { #[test] fn test_sparse_listview_null_fill_with_gaps() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // This test specifically catches the bug where the old implementation // incorrectly tracked `last_valid_offset` as the START of the last list // instead of properly handling ListView's offset/size pairs. @@ -1557,10 +1606,13 @@ mod test { // Convert to canonical form - this triggers the function we're testing let canonical = sparse .into_array() - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .execute::(&mut ctx)? .into_array(); - #[expect(deprecated)] - let result_listview = canonical.to_listview(); + let result_listview = canonical.execute::(&mut ctx)?; + let elements_primitive = result_listview + .elements() + .clone() + .execute::(&mut ctx)?; // Verify the structure assert_eq!(result_listview.len(), 10); @@ -1572,9 +1624,7 @@ mod test { if size == 0 { vec![] // null/empty list } else { - #[expect(deprecated)] - let elements = result_listview.elements().to_primitive(); - let slice = elements.as_slice::(); + let slice = elements_primitive.as_slice::(); slice[offset..offset + size].to_vec() } }; @@ -1597,6 +1647,7 @@ mod test { #[test] fn test_sparse_listview_sliced_values_null_fill() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // This test uses sliced ListView values to ensure proper handling // of non-zero starting offsets in the source data. @@ -1641,10 +1692,13 @@ mod test { let canonical = sparse .into_array() - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .execute::(&mut ctx)? .into_array(); - #[expect(deprecated)] - let result_listview = canonical.to_listview(); + let result_listview = canonical.execute::(&mut ctx)?; + let elements_primitive = result_listview + .elements() + .clone() + .execute::(&mut ctx)?; assert_eq!(result_listview.len(), 5); @@ -1655,9 +1709,7 @@ mod test { if size == 0 { vec![] // null/empty list } else { - #[expect(deprecated)] - let elements = result_listview.elements().to_primitive(); - let slice = elements.as_slice::(); + let slice = elements_primitive.as_slice::(); slice[offset..offset + size].to_vec() } }; diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 4eccd273090..8ef09360e41 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -36,8 +36,8 @@ impl CastReduce for Sparse { mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -53,6 +53,7 @@ mod tests { #[test] fn test_cast_sparse_i32_to_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let sparse = Sparse::try_new( buffer![2u64, 5, 8].into_array(), buffer![100i32, 200, 300].into_array(), @@ -71,8 +72,7 @@ mod tests { ); let expected = PrimitiveArray::from_iter([0i64, 0, 100, 0, 0, 200, 0, 0, 300, 0]); - #[expect(deprecated)] - let casted_primitive = casted.to_primitive(); + let casted_primitive = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!(casted_primitive, expected); } @@ -127,6 +127,7 @@ mod tests { #[test] fn test_cast_sparse_null_fill_all_patched_to_non_nullable() -> vortex_error::VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Regression test for https://github.com/vortex-data/vortex/issues/6932 // // When all positions are patched the null fill is unused, so a cast to @@ -149,8 +150,7 @@ mod tests { ); let expected = PrimitiveArray::from_iter([10u64, 20, 30, 40, 50]); - #[expect(deprecated)] - let casted_primitive = casted.to_primitive(); + let casted_primitive = casted.execute::(&mut ctx)?; assert_arrays_eq!(casted_primitive, expected); Ok(()) } diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 61436f80dea..f903f29621c 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -20,12 +20,10 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::buffer::BufferHandle; use vortex_array::builtins::ArrayBuiltins; @@ -270,8 +268,12 @@ impl Sparse { } /// Encode the given array as a [`SparseArray`]. - pub fn encode(array: &ArrayRef, fill_value: Option) -> VortexResult { - SparseData::encode(array, fill_value) + pub fn encode( + array: &ArrayRef, + fill_value: Option, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + SparseData::encode(array, fill_value, ctx) } } @@ -396,7 +398,11 @@ impl SparseData { /// Encode given array as a SparseArray. /// /// Optionally provided fill value will be respected if the array is less than 90% null. - pub fn encode(array: &ArrayRef, fill_value: Option) -> VortexResult { + pub fn encode( + array: &ArrayRef, + fill_value: Option, + ctx: &mut ExecutionCtx, + ) -> VortexResult { if let Some(fill_value) = fill_value.as_ref() && !array.dtype().eq_ignore_nullability(fill_value.dtype()) { @@ -406,9 +412,7 @@ impl SparseData { fill_value.dtype() ) } - let mask = array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?; + let mask = array.validity()?.to_mask(array.len(), ctx)?; if mask.all_false() { // Array is constant NULL @@ -417,10 +421,9 @@ impl SparseData { ); } else if mask.false_count() as f64 > (0.9 * mask.len() as f64) { // Array is dominated by NULL but has non-NULL values - // TODO(joe): use exe ctx? let non_null_values = array .filter(mask.clone())? - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .execute::(ctx)? .into_array(); let non_null_indices = match mask.indices() { AllOr::All => { @@ -454,8 +457,7 @@ impl SparseData { fill.cast(array.dtype())? } else { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array - #[expect(deprecated)] - let primitive = array.to_primitive(); + let primitive = array.clone().execute::(ctx)?; let (top_pvalue, _) = primitive .top_value()? .vortex_expect("Non empty or all null array"); @@ -464,16 +466,15 @@ impl SparseData { }; let fill_array = ConstantArray::new(fill.clone(), array.len()).into_array(); - #[expect(deprecated)] let non_top_bool = array .binary(fill_array.clone(), Operator::NotEq)? .fill_null(Scalar::bool(true, Nullability::NonNullable))? - .to_bool(); + .execute::(ctx)?; let non_top_mask = Mask::from_buffer(non_top_bool.to_bit_buffer()); let non_top_values = array .filter(non_top_mask.clone())? - .execute::(&mut LEGACY_SESSION.create_execution_ctx())? + .execute::(ctx)? .into_array(); let indices: Buffer = match non_top_mask { @@ -746,26 +747,26 @@ mod test { #[test] fn encode_with_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let original = PrimitiveArray::new( buffer![0i32, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4], Validity::from_iter(vec![ true, true, false, true, false, true, false, true, true, false, true, false, ]), ); - let sparse = Sparse::encode(&original.clone().into_array(), None) + let sparse = Sparse::encode(&original.clone().into_array(), None, &mut ctx) .vortex_expect("Sparse::encode should succeed for test data"); assert_eq!( sparse .validity() .unwrap() - .to_mask(sparse.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(sparse.len(), &mut ctx) .unwrap(), Mask::from_iter(vec![ true, true, false, true, false, true, false, true, true, false, true, false, ]) ); - #[expect(deprecated)] - let sparse_primitive = sparse.to_primitive(); + let sparse_primitive = sparse.execute::(&mut ctx).unwrap(); assert_arrays_eq!(sparse_primitive, original); } diff --git a/encodings/sparse/src/ops.rs b/encodings/sparse/src/ops.rs index 28bfd6d85bd..cfa89c0a733 100644 --- a/encodings/sparse/src/ops.rs +++ b/encodings/sparse/src/ops.rs @@ -25,8 +25,8 @@ impl OperationsVTable for Sparse { #[cfg(test)] mod tests { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_buffer::buffer; @@ -35,6 +35,7 @@ mod tests { #[test] fn slice_partially_invalid() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = buffer![0u64].into_array(); let indices = buffer![0u8].into_array(); @@ -43,8 +44,7 @@ mod tests { let mut expected = vec![999u64; 1000]; expected[0] = 0; - #[expect(deprecated)] - let values = sliced.to_primitive(); + let values = sliced.execute::(&mut ctx).unwrap(); assert_arrays_eq!(values, PrimitiveArray::from_iter(expected)); } } diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index d91bf1f2974..27bf37a7bde 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -271,9 +271,8 @@ impl ValidityChild for ZigZag { mod test { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; use vortex_buffer::buffer; @@ -282,12 +281,11 @@ mod test { #[test] fn test_compute_statistics() -> VortexResult<()> { - #[expect(deprecated)] + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = buffer![1i32, -5i32, 2, 3, 4, 5, 6, 7, 8, 9, 10] .into_array() - .to_primitive(); + .execute::(&mut ctx)?; let zigzag = zigzag_encode(array.as_view())?; - let mut ctx = LEGACY_SESSION.create_execution_ctx(); assert_eq!( zigzag.statistics().compute_max::(&mut ctx), diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index 8668ae6495f..2f52bd89d24 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -78,8 +78,8 @@ where #[cfg(test)] mod test { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; use super::*; @@ -87,42 +87,42 @@ mod test { #[test] fn test_compress_i8() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i8..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); - #[expect(deprecated)] - let decompressed = compressed.to_primitive(); + let decompressed = compressed.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i8..100)); } #[test] fn test_compress_i16() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i16..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); - #[expect(deprecated)] - let decompressed = compressed.to_primitive(); + let decompressed = compressed.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i16..100)); } #[test] fn test_compress_i32() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i32..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); - #[expect(deprecated)] - let decompressed = compressed.to_primitive(); + let decompressed = compressed.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i32..100)); } #[test] fn test_compress_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let compressed = zigzag_encode(PrimitiveArray::from_iter(-100_i64..100).as_view()) .unwrap() .into_array(); assert!(compressed.is::()); - #[expect(deprecated)] - let decompressed = compressed.to_primitive(); + let decompressed = compressed.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(-100_i64..100)); } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 4552506077c..4e105c4aa61 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -74,8 +74,6 @@ mod tests { use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -189,8 +187,8 @@ mod tests { fn test_take_zigzag_conformance(#[case] array: ArrayRef) -> VortexResult<()> { use vortex_array::compute::conformance::take::test_take_conformance; - #[expect(deprecated)] - let array_primitive = array.to_primitive(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array_primitive = array.execute::(&mut ctx)?; let zigzag = zigzag_encode(array_primitive.as_view())?; test_take_conformance(&zigzag.into_array()); Ok(()) diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index e8696d2d0d4..4b8ca95eafc 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -5,6 +5,8 @@ use divan::Bencher; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewRebuildMode; @@ -21,7 +23,7 @@ fn rebuild_naive(bencher: Bencher) { let validity = dudes.validity().unwrap(); let dudes = Zstd::try_new( dtype, - ZstdData::from_array(dudes, 9, 1024).unwrap(), + ZstdData::from_array(dudes, 9, 1024, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(), validity, ) .unwrap() diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index 466229e9776..95b3bff3bf7 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -6,11 +6,11 @@ impl vortex_zstd::Zstd pub fn vortex_zstd::Zstd::decompress(array: &vortex_zstd::ZstdArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::try_new(dtype: vortex_array::dtype::DType, data: vortex_zstd::ZstdData, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult @@ -98,17 +98,17 @@ pub struct vortex_zstd::ZstdData impl vortex_zstd::ZstdData -pub fn vortex_zstd::ZstdData::from_array(array: vortex_array::array::erased::ArrayRef, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::from_array(array: vortex_array::array::erased::ArrayRef, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::from_canonical(canonical: &vortex_array::canonical::Canonical, level: i32, values_per_frame: usize) -> vortex_error::VortexResult> +pub fn vortex_zstd::ZstdData::from_canonical(canonical: &vortex_array::canonical::Canonical, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_zstd::ZstdData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::from_primitive_without_dict(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::from_primitive_without_dict(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult +pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdData::into_parts(self, validity: vortex_array::validity::Validity) -> vortex_zstd::ZstdDataParts diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 5e3d592acca..d5f8d626996 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -21,11 +21,7 @@ use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; use vortex_array::Precision; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -270,11 +266,12 @@ impl Zstd { vbv: &VarBinViewArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = vbv.validity()?; Self::try_new( vbv.dtype().clone(), - ZstdData::from_var_bin_view_without_dict(vbv, level, values_per_frame)?, + ZstdData::from_var_bin_view_without_dict(vbv, level, values_per_frame, ctx)?, validity, ) } @@ -284,11 +281,12 @@ impl Zstd { parray: &PrimitiveArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = parray.validity()?; Self::try_new( parray.dtype().clone(), - ZstdData::from_primitive(parray, level, values_per_frame)?, + ZstdData::from_primitive(parray, level, values_per_frame, ctx)?, validity, ) } @@ -298,11 +296,12 @@ impl Zstd { vbv: &VarBinViewArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = vbv.validity()?; Self::try_new( vbv.dtype().clone(), - ZstdData::from_var_bin_view(vbv, level, values_per_frame)?, + ZstdData::from_var_bin_view(vbv, level, values_per_frame, ctx)?, validity, ) } @@ -366,21 +365,23 @@ fn choose_max_dict_size(uncompressed_size: usize) -> usize { (uncompressed_size / 100).clamp(256, 100 * 1024) } -fn collect_valid_primitive(parray: &PrimitiveArray) -> VortexResult { - let mask = parray.as_ref().validity()?.to_mask( - parray.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; - #[expect(deprecated)] - let result = parray.filter(mask)?.to_primitive(); +fn collect_valid_primitive( + parray: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = parray + .as_ref() + .validity()? + .to_mask(parray.as_ref().len(), ctx)?; + let result = parray.filter(mask)?.execute::(ctx)?; Ok(result) } -fn collect_valid_vbv(vbv: &VarBinViewArray) -> VortexResult<(ByteBuffer, Vec)> { - let mask = vbv.as_ref().validity()?.to_mask( - vbv.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; +fn collect_valid_vbv( + vbv: &VarBinViewArray, + ctx: &mut ExecutionCtx, +) -> VortexResult<(ByteBuffer, Vec)> { + let mask = vbv.as_ref().validity()?.to_mask(vbv.as_ref().len(), ctx)?; let buffer_and_value_byte_indices = match mask.bit_buffer() { AllOr::None => (Buffer::empty(), Vec::new()), _ => { @@ -625,8 +626,9 @@ impl ZstdData { parray: &PrimitiveArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - Self::from_primitive_impl(parray, level, values_per_frame, true) + Self::from_primitive_impl(parray, level, values_per_frame, true, ctx) } /// Creates a ZstdArray from a primitive array without using a dictionary. @@ -646,8 +648,9 @@ impl ZstdData { parray: &PrimitiveArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - Self::from_primitive_impl(parray, level, values_per_frame, false) + Self::from_primitive_impl(parray, level, values_per_frame, false, ctx) } fn from_primitive_impl( @@ -655,11 +658,12 @@ impl ZstdData { level: i32, values_per_frame: usize, use_dictionary: bool, + ctx: &mut ExecutionCtx, ) -> VortexResult { let byte_width = parray.ptype().byte_width(); // We compress only the valid elements. - let values = collect_valid_primitive(parray)?; + let values = collect_valid_primitive(parray, ctx)?; let n_values = values.len(); let values_per_frame = if values_per_frame > 0 { values_per_frame @@ -709,8 +713,9 @@ impl ZstdData { vbv: &VarBinViewArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - Self::from_var_bin_view_impl(vbv, level, values_per_frame, true) + Self::from_var_bin_view_impl(vbv, level, values_per_frame, true, ctx) } /// Creates a ZstdArray from a VarBinView array without using a dictionary. @@ -730,8 +735,9 @@ impl ZstdData { vbv: &VarBinViewArray, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { - Self::from_var_bin_view_impl(vbv, level, values_per_frame, false) + Self::from_var_bin_view_impl(vbv, level, values_per_frame, false, ctx) } fn from_var_bin_view_impl( @@ -739,6 +745,7 @@ impl ZstdData { level: i32, values_per_frame: usize, use_dictionary: bool, + ctx: &mut ExecutionCtx, ) -> VortexResult { // Approach for strings: we prefix each string with its length as a u32. // This is the same as what Parquet does. In some cases it may be better @@ -746,7 +753,7 @@ impl ZstdData { // this approach is simpler and can be best in cases when there is // mutual information between strings and their lengths. // We compress only the valid elements. - let (value_bytes, value_byte_indices) = collect_valid_vbv(vbv)?; + let (value_bytes, value_byte_indices) = collect_valid_vbv(vbv, ctx)?; let n_values = value_byte_indices.len(); let values_per_frame = if values_per_frame > 0 { values_per_frame @@ -785,26 +792,33 @@ impl ZstdData { canonical: &Canonical, level: i32, values_per_frame: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult> { match canonical { Canonical::Primitive(parray) => Ok(Some(ZstdData::from_primitive( parray, level, values_per_frame, + ctx, )?)), Canonical::VarBinView(vbv) => Ok(Some(ZstdData::from_var_bin_view( vbv, level, values_per_frame, + ctx, )?)), _ => Ok(None), } } - pub fn from_array(array: ArrayRef, level: i32, values_per_frame: usize) -> VortexResult { - #[expect(deprecated)] - let canonical = array.to_canonical()?; - Self::from_canonical(&canonical, level, values_per_frame)? + pub fn from_array( + array: ArrayRef, + level: i32, + values_per_frame: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let canonical = array.execute::(ctx)?; + Self::from_canonical(&canonical, level, values_per_frame, ctx)? .ok_or_else(|| vortex_err!("Zstd can only encode Primitive and VarBinView arrays")) } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index 508fcd6231d..05eb1d16c03 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -88,8 +88,8 @@ impl CastReduce for Zstd { mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -104,8 +104,9 @@ mod tests { #[test] fn test_cast_zstd_i32_to_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); - let zstd = Zstd::from_primitive(&values, 0, 0).unwrap(); + let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap(); let casted = zstd .into_array() @@ -116,15 +117,15 @@ mod tests { &DType::Primitive(PType::I64, Nullability::NonNullable) ); - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([1i64, 2, 3, 4, 5])); } #[test] fn test_cast_zstd_nullability_change() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]); - let zstd = Zstd::from_primitive(&values, 0, 0).unwrap(); + let zstd = Zstd::from_primitive(&values, 0, 0, &mut ctx).unwrap(); let casted = zstd .into_array() @@ -138,11 +139,12 @@ mod tests { #[test] fn test_cast_sliced_zstd_nullable_to_nonnullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::new( buffer![10u32, 20, 30, 40, 50, 60], Validity::from_iter([true, true, true, true, true, true]), ); - let zstd = Zstd::from_primitive(&values, 0, 128).unwrap(); + let zstd = Zstd::from_primitive(&values, 0, 128, &mut ctx).unwrap(); let sliced = zstd.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -152,13 +154,13 @@ mod tests { &DType::Primitive(PType::U32, Nullability::NonNullable) ); // Verify the values are correct - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); assert_arrays_eq!(decoded, PrimitiveArray::from_iter([20u32, 30, 40, 50])); } #[test] fn test_cast_sliced_zstd_part_valid_to_nonnullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_option_iter([ None, Some(20u32), @@ -167,7 +169,7 @@ mod tests { Some(50), Some(60), ]); - let zstd = Zstd::from_primitive(&values, 0, 128).unwrap(); + let zstd = Zstd::from_primitive(&values, 0, 128, &mut ctx).unwrap(); let sliced = zstd.slice(1..5).unwrap(); let casted = sliced .cast(DType::Primitive(PType::U32, Nullability::NonNullable)) @@ -176,8 +178,7 @@ mod tests { casted.dtype(), &DType::Primitive(PType::U32, Nullability::NonNullable) ); - #[expect(deprecated)] - let decoded = casted.to_primitive(); + let decoded = casted.execute::(&mut ctx).unwrap(); let expected = PrimitiveArray::from_iter([20u32, 30, 40, 50]); assert_arrays_eq!(decoded, expected); } @@ -200,7 +201,8 @@ mod tests { Validity::NonNullable, ))] fn test_cast_zstd_conformance(#[case] values: PrimitiveArray) { - let zstd = Zstd::from_primitive(&values, 0, 0).unwrap(); + let zstd = Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); test_cast_conformance(&zstd.into_array()); } } diff --git a/encodings/zstd/src/compute/mod.rs b/encodings/zstd/src/compute/mod.rs index adf6937c246..de175eb3154 100644 --- a/encodings/zstd/src/compute/mod.rs +++ b/encodings/zstd/src/compute/mod.rs @@ -7,6 +7,8 @@ mod cast; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::consistency::test_array_consistency; use vortex_buffer::buffer; @@ -16,23 +18,23 @@ mod tests { fn zstd_i32() -> ZstdArray { let values = PrimitiveArray::from_iter([100i32, 200, 300, 400, 500]); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_f64() -> ZstdArray { let values = PrimitiveArray::from_iter([1.1f64, 2.2, 3.3, 4.4, 5.5]); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_u32() -> ZstdArray { let values = PrimitiveArray::from_iter([10u32, 20, 30, 40, 50]); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_nullable_i64() -> ZstdArray { let values = PrimitiveArray::from_option_iter([Some(1000i64), None, Some(3000), Some(4000), None]); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_single() -> ZstdArray { @@ -40,7 +42,7 @@ mod tests { buffer![42i64], vortex_array::validity::Validity::NonNullable, ); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_large() -> ZstdArray { @@ -48,7 +50,7 @@ mod tests { buffer![0u32..1000], vortex_array::validity::Validity::NonNullable, ); - Zstd::from_primitive(&values, 3, 0).unwrap() + Zstd::from_primitive(&values, 3, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_all_same() -> ZstdArray { @@ -56,12 +58,12 @@ mod tests { buffer![42i32; 100], vortex_array::validity::Validity::NonNullable, ); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } fn zstd_negative() -> ZstdArray { let values = PrimitiveArray::from_iter([-100i32, -50, 0, 50, 100]); - Zstd::from_primitive(&values, 0, 0).unwrap() + Zstd::from_primitive(&values, 0, 0, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() } #[rstest] diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index da467d3f2ed..88a49aefe07 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -4,8 +4,6 @@ use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -23,16 +21,16 @@ use crate::Zstd; #[test] fn test_zstd_compress_decompress() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..200).collect(); let array = PrimitiveArray::from_iter(data.clone()); - let compressed = Zstd::from_primitive(&array, 3, 0).unwrap(); + let compressed = Zstd::from_primitive(&array, 3, 0, &mut ctx).unwrap(); // this data should be compressible assert!(compressed.frames.len() < array.into_array().nbytes() as usize); assert!(compressed.dictionary.is_none()); // check full decompression works - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); @@ -49,19 +47,21 @@ fn test_zstd_compress_decompress() { #[test] fn test_zstd_empty() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = vec![]; let array = PrimitiveArray::new( data.iter().cloned().collect::>(), Validity::NonNullable, ); - let compressed = Zstd::from_primitive(&array, 3, 100).unwrap(); + let compressed = Zstd::from_primitive(&array, 3, 100, &mut ctx).unwrap(); assert_arrays_eq!(compressed, PrimitiveArray::from_iter(data)); } #[test] fn test_zstd_with_validity_and_multi_frame() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..200).collect(); let mut validity: Vec = vec![false; 200]; validity[3] = true; @@ -71,18 +71,17 @@ fn test_zstd_with_validity_and_multi_frame() { Validity::Array(BoolArray::from_iter(validity).into_array()), ); - let compressed = Zstd::from_primitive(&array, 0, 30).unwrap(); + let compressed = Zstd::from_primitive(&array, 0, 30, &mut ctx).unwrap(); assert!(compressed.dictionary.is_none()); assert_nth_scalar!(compressed, 0, None::); assert_nth_scalar!(compressed, 3, 3); assert_nth_scalar!(compressed, 10, None::); assert_nth_scalar!(compressed, 177, 177); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); let decompressed_values = decompressed.as_slice::(); assert_eq!(decompressed_values[3], 3); assert_eq!(decompressed_values[177], 177); @@ -96,15 +95,9 @@ fn test_zstd_with_validity_and_multi_frame() { // check slicing works let slice = compressed.slice(176..179).unwrap(); - #[expect(deprecated)] - let primitive = slice.to_primitive(); + let primitive = slice.execute::(&mut ctx).unwrap(); assert_eq!( - i32::try_from( - &primitive - .execute_scalar(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ) - .unwrap(), + i32::try_from(&primitive.execute_scalar(1, &mut ctx).unwrap()).unwrap(), 177 ); assert!( @@ -121,39 +114,39 @@ fn test_zstd_with_validity_and_multi_frame() { #[test] fn test_zstd_with_dict() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec = (0..200).collect(); let array = PrimitiveArray::new( data.iter().cloned().collect::>(), Validity::NonNullable, ); - let compressed = Zstd::from_primitive(&array, 0, 16).unwrap(); + let compressed = Zstd::from_primitive(&array, 0, 16, &mut ctx).unwrap(); assert!(compressed.dictionary.is_some()); assert_nth_scalar!(compressed, 0, 0); assert_nth_scalar!(compressed, 199, 199); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works let slice = compressed.slice(176..179).unwrap(); - #[expect(deprecated)] - let primitive = slice.to_primitive(); + let primitive = slice.execute::(&mut ctx).unwrap(); assert_arrays_eq!(primitive, PrimitiveArray::from_iter([176, 177, 178])); } #[test] fn test_validity_vtable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mask_bools = vec![false, true, true, false, true]; let array = PrimitiveArray::new( (0..5).collect::>(), Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); - let compressed = Zstd::from_primitive(&array, 3, 0).unwrap(); + let compressed = Zstd::from_primitive(&array, 3, 0, &mut ctx).unwrap(); let arr = compressed.as_array(); assert_eq!( arr.validity() @@ -175,6 +168,7 @@ fn test_validity_vtable() { #[test] fn test_zstd_var_bin_view() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: [Option<&'static [u8]>; 5] = [ Some(b"foo"), Some(b"bar"), @@ -184,7 +178,7 @@ fn test_zstd_var_bin_view() { ]; let array = VarBinViewArray::from_iter(data, DType::Utf8(Nullability::Nullable)); - let compressed = Zstd::from_var_bin_view(&array, 0, 3).unwrap(); + let compressed = Zstd::from_var_bin_view(&array, 0, 3, &mut ctx).unwrap(); assert!(compressed.dictionary.is_none()); assert_nth_scalar!(compressed, 0, "foo"); assert_nth_scalar!(compressed, 1, "bar"); @@ -200,6 +194,7 @@ fn test_zstd_var_bin_view() { #[test] fn test_zstd_decompress_var_bin_view() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: [Option<&'static [u8]>; 5] = [ Some(b"foo"), Some(b"bar"), @@ -209,7 +204,7 @@ fn test_zstd_decompress_var_bin_view() { ]; let array = VarBinViewArray::from_iter(data, DType::Utf8(Nullability::Nullable)); - let compressed = Zstd::from_var_bin_view(&array, 0, 3).unwrap(); + let compressed = Zstd::from_var_bin_view(&array, 0, 3, &mut ctx).unwrap(); assert!(compressed.dictionary.is_none()); assert_nth_scalar!(compressed, 0, "foo"); assert_nth_scalar!(compressed, 1, "bar"); @@ -217,11 +212,10 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 3, "Lorem ipsum dolor sit amet"); assert_nth_scalar!(compressed, 4, "baz"); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - #[expect(deprecated)] let decompressed = Zstd::decompress(&compressed, &mut ctx) .unwrap() - .to_varbinview(); + .execute::(&mut ctx) + .unwrap(); assert_nth_scalar!(decompressed, 0, "foo"); assert_nth_scalar!(decompressed, 1, "bar"); assert_nth_scalar!(decompressed, 2, None::); @@ -231,8 +225,10 @@ fn test_zstd_decompress_var_bin_view() { #[test] fn test_sliced_array_children() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data: Vec> = (0..10).map(|v| (v != 5).then_some(v)).collect(); - let compressed = Zstd::from_primitive(&PrimitiveArray::from_option_iter(data), 0, 100).unwrap(); + let compressed = + Zstd::from_primitive(&PrimitiveArray::from_option_iter(data), 0, 100, &mut ctx).unwrap(); let sliced = compressed.slice(0..4).unwrap(); sliced.children(); } @@ -241,11 +237,12 @@ fn test_sliced_array_children() { /// the buffer alignment when compressing primitive arrays. #[test] fn test_zstd_frame_start_buffer_alignment() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data = vec![0u8; 2]; let aligned_buffer = Buffer::copy_from_aligned(&data, Alignment::new(8)); // u8 array now has a 8-byte alignment. let array = PrimitiveArray::new(aligned_buffer, Validity::NonNullable); - let compressed = Zstd::from_primitive(&array, 0, 1); + let compressed = Zstd::from_primitive(&array, 0, 1, &mut ctx); assert!(compressed.is_ok()); } diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index d4d3865507d..f62693ccc49 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -8,9 +8,8 @@ use libfuzzer_sys::Corpus; use libfuzzer_sys::fuzz_target; use vortex_array::Canonical; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::builtins::ArrayBuiltins; @@ -46,14 +45,16 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { return Corpus::Reject; } + let mut ctx = SESSION.create_execution_ctx(); let expected_array = { let bool_mask = array_data .clone() .apply(&filter_expr.clone().unwrap_or_else(|| lit(true))) .vortex_expect("filter expression evaluation should succeed in fuzz test"); - #[expect(deprecated)] - let bool_mask_bool = bool_mask.to_bool(); - let mask = bool_mask_bool.to_mask_fill_null_false(&mut SESSION.create_execution_ctx()); + let bool_mask_bool = bool_mask + .execute::(&mut ctx) + .vortex_expect("execute bool"); + let mask = bool_mask_bool.to_mask_fill_null_false(&mut ctx); let filtered = array_data .filter(mask) .vortex_expect("filter operation should succeed in fuzz test"); @@ -111,13 +112,12 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { output_array.dtype() ); - #[expect(deprecated)] let bool_result = expected_array .binary(output_array.clone(), Operator::Eq) .vortex_expect("compare operation should succeed in fuzz test") - .to_bool(); + .execute::(&mut ctx) + .vortex_expect("execute bool"); let true_count = bool_result.to_bit_buffer().true_count(); - let mut ctx = SESSION.create_execution_ctx(); if true_count != expected_array.len() && (bool_result .into_array() diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index fdb399bac43..26cc9409884 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -2,11 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::Nullable; @@ -16,7 +13,11 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; -pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult> { +pub fn cast_canonical_array( + array: &ArrayRef, + target: &DType, + ctx: &mut ExecutionCtx, +) -> VortexResult> { // TODO(joe): support more casting options let is_int_to_int = target.is_int() && array.dtype().is_int(); let is_float_to_float = target.is_float() && array.dtype().is_float(); @@ -36,18 +37,14 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult(ctx)?; PrimitiveArray::new( prim.as_slice::() .iter() .map(|v| *v as Out) .collect::>(), Validity::from_mask( - array.validity()?.to_mask( - array.len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?, + array.validity()?.to_mask(array.len(), ctx)?, target.nullability(), ), ) @@ -69,8 +66,7 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult { - #[expect(deprecated)] - let prim = array.to_primitive(); + let prim = array.clone().execute::(ctx)?; Ok(Some( PrimitiveArray::new( prim.as_slice::() @@ -78,9 +74,7 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), Validity::from_mask( - array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + array.validity()?.to_mask(array.len(), ctx)?, target.nullability(), ), ) @@ -88,8 +82,7 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult { - #[expect(deprecated)] - let prim = array.to_primitive(); + let prim = array.clone().execute::(ctx)?; #[expect(clippy::cast_possible_truncation)] Ok(Some( PrimitiveArray::new( @@ -98,9 +91,7 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), Validity::from_mask( - array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + array.validity()?.to_mask(array.len(), ctx)?, target.nullability(), ), ) diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index c45c25d4125..60a702f2839 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -2,13 +2,13 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::DType; @@ -27,6 +27,7 @@ pub fn compare_canonical_array( array: &ArrayRef, value: &Scalar, operator: CompareOperator, + ctx: &mut ExecutionCtx, ) -> ArrayRef { if value.is_null() { return BoolArray::new(BitBuffer::new_unset(array.len()), Validity::AllInvalid) @@ -41,8 +42,10 @@ pub fn compare_canonical_array( .as_bool() .value() .vortex_expect("nulls handled before"); - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array + .clone() + .execute::(ctx) + .vortex_expect("to bool"); compare_to( bool_array .to_bit_buffer() @@ -51,7 +54,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), @@ -64,8 +67,10 @@ pub fn compare_canonical_array( } DType::Primitive(p, _) => { let primitive = value.as_primitive(); - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array + .clone() + .execute::(ctx) + .vortex_expect("to primitive"); match_each_native_ptype!(p, |P| { let pval = primitive .typed_value::

().slice(start..stop), @@ -59,8 +52,7 @@ pub fn slice_canonical_array( }) } DType::Utf8(_) | DType::Binary(_) => { - #[expect(deprecated)] - let utf8 = array.to_varbinview(); + let utf8 = array.clone().execute::(ctx)?; let values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); Ok(VarBinViewArray::from_iter( @@ -70,11 +62,10 @@ pub fn slice_canonical_array( .into_array()) } DType::Struct(..) => { - #[expect(deprecated)] - let struct_array = array.to_struct(); + let struct_array = array.clone().execute::(ctx)?; let sliced_children = struct_array .iter_unmasked_fields() - .map(|c| slice_canonical_array(c, start, stop)) + .map(|c| slice_canonical_array(c, start, stop, ctx)) .collect::>>()?; StructArray::try_new_with_dtype( sliced_children, @@ -85,11 +76,10 @@ pub fn slice_canonical_array( .map(|a| a.into_array()) } DType::List(..) => { - #[expect(deprecated)] - let list_array = array.to_listview(); + let list_array = array.clone().execute::(ctx)?; - let offsets = slice_canonical_array(list_array.offsets(), start, stop)?; - let sizes = slice_canonical_array(list_array.sizes(), start, stop)?; + let offsets = slice_canonical_array(list_array.offsets(), start, stop, ctx)?; + let sizes = slice_canonical_array(list_array.sizes(), start, stop, ctx)?; // Since the list view elements can be stored out of order, we cannot slice it. let elements = list_array.elements().clone(); @@ -104,19 +94,21 @@ pub fn slice_canonical_array( .into_array()) } DType::FixedSizeList(..) => { - #[expect(deprecated)] - let fsl_array = array.to_fixed_size_list(); + let fsl_array = array.clone().execute::(ctx)?; let list_size = fsl_array.list_size() as usize; - let elements = - slice_canonical_array(fsl_array.elements(), start * list_size, stop * list_size)?; + let elements = slice_canonical_array( + fsl_array.elements(), + start * list_size, + stop * list_size, + ctx, + )?; let new_len = stop - start; FixedSizeListArray::try_new(elements, fsl_array.list_size(), validity, new_len) .map(|a| a.into_array()) } DType::Decimal(decimal_dtype, _) => { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array.clone().execute::(ctx)?; Ok( match_each_decimal_value_type!(decimal_array.values_type(), |D| { DecimalArray::new( diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index dbe9cff9179..8ef59930556 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -4,11 +4,8 @@ use std::cmp::Ordering; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -24,11 +21,10 @@ use vortex_error::VortexResult; use crate::array::take_canonical_array_non_nullable_indices; -pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { +pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { match array.dtype() { DType::Bool(_) => { - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array.clone().execute::(ctx)?; let mut opt_values = bool_array .to_bit_buffer() .iter() @@ -36,10 +32,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { bool_array .as_ref() .validity()? - .to_mask( - bool_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(bool_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) @@ -49,8 +42,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { Ok(BoolArray::from_iter(opt_values).into_array()) } DType::Primitive(p, _) => { - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array.clone().execute::(ctx)?; match_each_native_ptype!(p, |P| { let mut opt_values = primitive_array .as_slice::

() @@ -60,10 +52,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { primitive_array .as_ref() .validity()? - .to_mask( - primitive_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(primitive_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) @@ -74,8 +63,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { }) } DType::Decimal(d, _) => { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array.clone().execute::(ctx)?; match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); let mut opt_values = buf @@ -86,10 +74,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { decimal_array .as_ref() .validity()? - .to_mask( - decimal_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(decimal_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) @@ -100,8 +85,7 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { }) } DType::Utf8(_) | DType::Binary(_) => { - #[expect(deprecated)] - let utf8 = array.to_varbinview(); + let utf8 = array.clone().execute::(ctx)?; let mut opt_values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); opt_values.sort(); @@ -109,18 +93,13 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { let mut sort_indices = (0..array.len()).collect::>(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); sort_indices.sort_by(|a, b| { - let lhs = array - .execute_scalar(*a, &mut ctx) - .vortex_expect("scalar_at"); - let rhs = array - .execute_scalar(*b, &mut ctx) - .vortex_expect("scalar_at"); + let lhs = array.execute_scalar(*a, ctx).vortex_expect("scalar_at"); + let rhs = array.execute_scalar(*b, ctx).vortex_expect("scalar_at"); lhs.partial_cmp(&rhs) .vortex_expect("must be a valid comparison") }); - take_canonical_array_non_nullable_indices(array, &sort_indices) + take_canonical_array_non_nullable_indices(array, &sort_indices, ctx) } d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => { unreachable!("DType {d} not supported for fuzzing") diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index d33dcdeaf2c..1a6218eccf7 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -2,11 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -31,6 +28,7 @@ use vortex_error::VortexResult; pub fn take_canonical_array_non_nullable_indices( array: &ArrayRef, indices: &[usize], + ctx: &mut ExecutionCtx, ) -> VortexResult { take_canonical_array( array, @@ -39,17 +37,19 @@ pub fn take_canonical_array_non_nullable_indices( .map(|i| Some(*i)) .collect::>() .as_slice(), + ctx, ) } -pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> VortexResult { +pub fn take_canonical_array( + array: &ArrayRef, + indices: &[Option], + ctx: &mut ExecutionCtx, +) -> VortexResult { let nullable: Nullability = indices.contains(&None).into(); let validity = if array.dtype().is_nullable() || nullable == Nullability::Nullable { - let validity_idx = array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? - .to_bit_buffer(); + let validity_idx = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); Validity::from_iter( indices @@ -65,8 +65,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort match array.dtype() { DType::Bool(_) => { - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array.clone().execute::(ctx)?; let vec_values = bool_array.to_bit_buffer().iter().collect::>(); Ok(BoolArray::new( indices_slice_non_opt @@ -78,8 +77,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort .into_array()) } DType::Primitive(p, _) => { - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array.clone().execute::(ctx)?; match_each_native_ptype!(p, |P| { Ok(take_primitive::

( primitive_array, @@ -89,8 +87,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort }) } DType::Decimal(d, _) => { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array.clone().execute::(ctx)?; match_each_decimal_value_type!(decimal_array.values_type(), |D| { Ok(take_decimal::( @@ -102,8 +99,7 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort }) } DType::Utf8(_) | DType::Binary(_) => { - #[expect(deprecated)] - let utf8 = array.to_varbinview(); + let utf8 = array.clone().execute::(ctx)?; let values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); Ok(VarBinViewArray::from_iter( @@ -115,11 +111,10 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort .into_array()) } DType::Struct(..) => { - #[expect(deprecated)] - let struct_array = array.to_struct(); + let struct_array = array.clone().execute::(ctx)?; let taken_children = struct_array .iter_unmasked_fields() - .map(|c| take_canonical_array_non_nullable_indices(c, indices_slice_non_opt)) + .map(|c| take_canonical_array_non_nullable_indices(c, indices_slice_non_opt, ctx)) .collect::>>()?; StructArray::try_new( @@ -135,12 +130,11 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort &array.dtype().union_nullability(nullable), indices_slice_non_opt.len(), ); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); for idx in indices { if let Some(idx) = idx { builder.append_scalar( &array - .execute_scalar(*idx, &mut ctx)? + .execute_scalar(*idx, ctx)? .cast(&array.dtype().union_nullability(nullable)) .vortex_expect("cannot cast scalar nullability"), )?; diff --git a/fuzz/src/compress.rs b/fuzz/src/compress.rs index 59e2b2d7c88..1284a7be45e 100644 --- a/fuzz/src/compress.rs +++ b/fuzz/src/compress.rs @@ -9,11 +9,15 @@ use arbitrary::Arbitrary; use arbitrary::Unstructured; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::constant::ArbitraryConstantArray; use vortex_array::arrays::dict::ArbitraryDictArray; use vortex_runend::ArbitraryRunEndArray; +use crate::SESSION; + /// Which compressed encoding to generate. #[derive(Debug, Clone, Copy)] pub enum EncodingKind { @@ -70,9 +74,10 @@ pub fn run_compress_roundtrip(fuzz: FuzzCompressRoundtrip) -> crate::error::Vort let original_len = array.len(); let original_dtype = array.dtype().clone(); + let mut ctx = SESSION.create_execution_ctx(); + // Try to canonicalize - this is the main thing we're testing - #[expect(deprecated)] - let canonical = match array.to_canonical() { + let canonical = match array.clone().execute::(&mut ctx) { Ok(c) => c, Err(e) => { // Canonicalization failed - this is a bug diff --git a/fuzz/src/fsst_like.rs b/fuzz/src/fsst_like.rs index a36e6b91974..5ca10af310a 100644 --- a/fuzz/src/fsst_like.rs +++ b/fuzz/src/fsst_like.rs @@ -113,8 +113,14 @@ pub fn run_fsst_like_fuzz(fuzz: FuzzFsstLike) -> VortexFuzzResult { // Train FSST compressor and compress. let compressor = fsst_train_compressor(&varbin); - let fsst_array: FSSTArray = - fsst_compress(varbin.clone(), varbin.len(), varbin.dtype(), &compressor); + let mut ctx = SESSION.create_execution_ctx(); + let fsst_array: FSSTArray = fsst_compress( + varbin.clone(), + varbin.len(), + varbin.dtype(), + &compressor, + &mut ctx, + ); let opts = LikeOptions { negated, diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs index e7f53156500..9429e06fd8f 100644 --- a/vortex-bench/src/datasets/mod.rs +++ b/vortex-bench/src/datasets/mod.rs @@ -8,6 +8,7 @@ use async_trait::async_trait; use serde::Deserialize; use serde::Serialize; use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use crate::clickbench::Flavor; @@ -25,7 +26,7 @@ use std::path::PathBuf; pub trait Dataset { fn name(&self) -> &str; - async fn to_vortex_array(&self) -> Result; + async fn to_vortex_array(&self, ctx: &mut ExecutionCtx) -> Result; /// Get the path to the parquet file for this dataset. /// diff --git a/vortex-bench/src/datasets/struct_list_of_ints.rs b/vortex-bench/src/datasets/struct_list_of_ints.rs index 920d5cf7a52..6bed5151c5e 100644 --- a/vortex-bench/src/datasets/struct_list_of_ints.rs +++ b/vortex-bench/src/datasets/struct_list_of_ints.rs @@ -11,6 +11,7 @@ use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; @@ -63,7 +64,7 @@ impl Dataset for StructListOfInts { &self.name } - async fn to_vortex_array(&self) -> Result { + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> Result { let names: FieldNames = (0..self.num_columns) .map(|col_idx| col_idx.to_string()) .collect(); @@ -115,7 +116,8 @@ impl Dataset for StructListOfInts { idempotent_async(&parquet_path, |temp_path| async move { // Generate the data - let array = self.to_vortex_array().await?; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = self.to_vortex_array(&mut ctx).await?; // Convert to Arrow RecordBatches and write to parquet let chunked = array.as_::(); diff --git a/vortex-bench/src/datasets/taxi_data.rs b/vortex-bench/src/datasets/taxi_data.rs index cc3c6f61b81..919a0fa4dff 100644 --- a/vortex-bench/src/datasets/taxi_data.rs +++ b/vortex-bench/src/datasets/taxi_data.rs @@ -8,6 +8,7 @@ use async_trait::async_trait; use tokio::fs::File as TokioFile; use tokio::io::AsyncWriteExt; use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::stream::ArrayStreamExt; use vortex::file::OpenOptionsSessionExt; @@ -37,7 +38,7 @@ impl Dataset for TaxiData { "taxi" } - async fn to_vortex_array(&self) -> Result { + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> Result { fetch_taxi_data().await } diff --git a/vortex-bench/src/datasets/tpch_l_comment.rs b/vortex-bench/src/datasets/tpch_l_comment.rs index 3488ef92979..ebe6ff2e96a 100644 --- a/vortex-bench/src/datasets/tpch_l_comment.rs +++ b/vortex-bench/src/datasets/tpch_l_comment.rs @@ -8,10 +8,11 @@ use async_trait::async_trait; use futures::TryStreamExt; use glob::glob; use vortex::array::ArrayRef; +use vortex::array::Canonical; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::arrays::ChunkedArray; +use vortex::array::arrays::StructArray; use vortex::dtype::Nullability::NonNullable; use vortex::expr::col; use vortex::expr::pack; @@ -52,7 +53,7 @@ impl Dataset for TPCHLCommentChunked { "TPC-H l_comment chunked" } - async fn to_vortex_array(&self) -> Result { + async fn to_vortex_array(&self, ctx: &mut ExecutionCtx) -> Result { let base_path = "tpch".to_data_path(); let scale_factor_dir = base_path.join("1.0"); let data_dir = scale_factor_dir.join(Format::OnDiskVortex.name()); @@ -77,10 +78,13 @@ impl Dataset for TPCHLCommentChunked { let file_chunks: Vec<_> = file .scan()? .with_projection(pack(vec![("l_comment", col("l_comment"))], NonNullable)) - .map(|a| { - #[expect(deprecated)] - let canonical = a.to_canonical()?; - Ok(canonical.into_array()) + .map({ + let ctx = ctx.clone(); + move |a| { + let mut ctx = ctx.clone(); + let canonical = a.execute::(&mut ctx)?; + Ok(canonical.into_array()) + } }) .into_array_stream()? .try_collect() @@ -104,10 +108,9 @@ impl Dataset for TPCHLCommentCanonical { "TPC-H l_comment canonical" } - async fn to_vortex_array(&self) -> Result { - let comments_chunked = TPCHLCommentChunked.to_vortex_array().await?; - #[expect(deprecated)] - let comments_canonical = comments_chunked.to_struct().into_array(); + async fn to_vortex_array(&self, ctx: &mut ExecutionCtx) -> Result { + let comments_chunked = TPCHLCommentChunked.to_vortex_array(ctx).await?; + let comments_canonical = comments_chunked.execute::(ctx)?.into_array(); Ok(ChunkedArray::from_iter([comments_canonical]).into_array()) } diff --git a/vortex-bench/src/downloadable_dataset.rs b/vortex-bench/src/downloadable_dataset.rs index 78ab162d688..cd73ba5af84 100644 --- a/vortex-bench/src/downloadable_dataset.rs +++ b/vortex-bench/src/downloadable_dataset.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; use tokio::fs::File; use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::stream::ArrayStreamExt; use vortex::file::OpenOptionsSessionExt; @@ -57,7 +58,7 @@ impl Dataset for DownloadableDataset { } } - async fn to_vortex_array(&self) -> anyhow::Result { + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> anyhow::Result { let parquet = self.to_parquet_path().await?; let dir = format!("{}/", self.name()).to_data_path(); let vortex = dir.join(format!("{}.vortex", self.name())); diff --git a/vortex-bench/src/public_bi.rs b/vortex-bench/src/public_bi.rs index fe937876275..7406347d370 100644 --- a/vortex-bench/src/public_bi.rs +++ b/vortex-bench/src/public_bi.rs @@ -26,6 +26,7 @@ use tracing::info; use tracing::trace; use url::Url; use vortex::array::ArrayRef; +use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::stream::ArrayStreamExt; use vortex::error::VortexResult; @@ -452,7 +453,7 @@ impl Dataset for PBIBenchmark { &self.name } - async fn to_vortex_array(&self) -> anyhow::Result { + async fn to_vortex_array(&self, _ctx: &mut ExecutionCtx) -> anyhow::Result { let dataset = self.dataset()?; dataset.write_as_vortex().await?; // reading only the first table, each table in a PBI benchmark diff --git a/vortex-btrblocks/benches/compress.rs b/vortex-btrblocks/benches/compress.rs index a82dafacf3b..a2daa33b432 100644 --- a/vortex-btrblocks/benches/compress.rs +++ b/vortex-btrblocks/benches/compress.rs @@ -13,8 +13,9 @@ mod benchmarks { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::PrimitiveArray; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; use vortex_utils::aliases::hash_set::HashSet; @@ -40,8 +41,10 @@ mod benchmarks { #[divan::bench] fn btrblocks(bencher: Bencher) { - #[expect(deprecated)] - let array = make_clickbench_window_name().to_primitive(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = make_clickbench_window_name() + .execute::(&mut ctx) + .unwrap(); let compressor = BtrBlocksCompressor::default(); bencher .with_inputs(|| &array) diff --git a/vortex-btrblocks/public-api.lock b/vortex-btrblocks/public-api.lock index cfb2f49b773..179cd45cf5c 100644 --- a/vortex-btrblocks/public-api.lock +++ b/vortex-btrblocks/public-api.lock @@ -98,7 +98,7 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::ALP impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::ALPRDScheme -pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::ALPRDScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -230,7 +230,7 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::Pco impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::PcoScheme -pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::PcoScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -270,7 +270,7 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::B impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::BitPackingScheme -pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -368,7 +368,7 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::P impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::PcoScheme -pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::PcoScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -436,7 +436,7 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::S pub fn vortex_btrblocks::schemes::integer::SequenceScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::SequenceScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -614,7 +614,7 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::Zs impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::ZstdScheme -pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::string::ZstdScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate @@ -706,4 +706,4 @@ pub fn vortex_btrblocks::BtrBlocksCompressorBuilder::fmt(&self, f: &mut core::fm pub const vortex_btrblocks::ALL_SCHEMES: &[&dyn vortex_compressor::scheme::Scheme] -pub fn vortex_btrblocks::compress_patches(patches: vortex_array::patches::Patches) -> vortex_error::VortexResult +pub fn vortex_btrblocks::compress_patches(patches: vortex_array::patches::Patches, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index a801b9bfd9b..e68b4b8191f 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -6,8 +6,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::decimal::narrowed_decimal; use vortex_array::dtype::DecimalType; @@ -60,8 +59,10 @@ impl Scheme for DecimalScheme { ) -> VortexResult { // TODO(joe): add support splitting i128/256 buffers into chunks of primitive values // for compression. 2 for i128 and 4 for i256. - #[expect(deprecated)] - let decimal = data.array().clone().to_decimal(); + let decimal = data + .array() + .clone() + .execute::(&mut compressor.execution_ctx())?; let decimal = narrowed_decimal(decimal); let validity = decimal.validity()?; let prim = match decimal.values_type() { diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 9e7e56fd7a5..a3526f141e6 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -14,10 +14,9 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::PType; @@ -111,7 +110,7 @@ impl Scheme for ALPScheme { let alp_encoded = alp_encode( data.array_as_primitive(), None, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut compressor.execution_ctx(), )?; // Compress the ALP ints. @@ -132,13 +131,16 @@ impl Scheme for ALPScheme { Some(p) => Ok(Patched::from_array_and_patches( alp_array, &p, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut compressor.execution_ctx(), )? .with_stats_set(alp_stats) .into_array()), } } else { - let patches = alp_encoded.patches().map(compress_patches).transpose()?; + let patches = alp_encoded + .patches() + .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .transpose()?; Ok(ALP::new(compressed_alp_ints, exponents, patches).into_array()) } @@ -169,7 +171,7 @@ impl Scheme for ALPRDScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { @@ -181,11 +183,14 @@ impl Scheme for ALPRDScheme { ptype => vortex_panic!("cannot ALPRD compress ptype {ptype}"), }; - let alp_rd = encoder.encode(primitive_array); + let alp_rd = encoder.encode(primitive_array, &mut compressor.execution_ctx()); let dtype = alp_rd.dtype().clone(); let right_bit_width = alp_rd.right_bit_width(); let mut parts = ALPRDArrayOwnedExt::into_data_parts(alp_rd); - parts.left_parts_patches = parts.left_parts_patches.map(compress_patches).transpose()?; + parts.left_parts_patches = parts + .left_parts_patches + .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .transpose()?; Ok(vortex_alp::ALPRD::try_new( dtype, @@ -194,6 +199,7 @@ impl Scheme for ALPRDScheme { parts.right_parts, right_bit_width, parts.left_parts_patches, + &mut compressor.execution_ctx(), )? .into_array()) } @@ -227,7 +233,10 @@ impl Scheme for NullDominatedSparseScheme { _ctx: CompressorContext, ) -> CompressionEstimate { let len = data.array_len() as f64; - let stats = data.float_stats(); + // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature + // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. + let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.float_stats(&mut exec_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -251,11 +260,15 @@ impl Scheme for NullDominatedSparseScheme { ctx: CompressorContext, ) -> VortexResult { // We pass None as we only run this pathway for NULL-dominated float arrays. - let sparse_encoded = Sparse::encode(data.array(), None)?; + let sparse_encoded = Sparse::encode(data.array(), None, &mut compressor.execution_ctx())?; if let Some(sparse) = sparse_encoded.as_opt::() { - #[expect(deprecated)] - let indices = sparse.patches().indices().to_primitive().narrow()?; + let indices = sparse + .patches() + .indices() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_indices = compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; @@ -292,7 +305,7 @@ impl Scheme for PcoScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { @@ -300,6 +313,7 @@ impl Scheme for PcoScheme { data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, + &mut compressor.execution_ctx(), )? .into_array()) } @@ -337,7 +351,12 @@ impl Scheme for FloatRLEScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - if data.float_stats().average_run_length() < super::integer::RUN_LENGTH_THRESHOLD { + // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature + // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. + let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); + if data.float_stats(&mut exec_ctx).average_run_length() + < super::integer::RUN_LENGTH_THRESHOLD + { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index cde4f7c7556..de428f967ac 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -7,11 +7,10 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Patched; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::scalar::Scalar; @@ -140,7 +139,9 @@ impl Scheme for FoRScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut local_ctx); // Only apply when the min is not already zero. if stats.erased().min_is_zero() { @@ -187,11 +188,15 @@ impl Scheme for FoRScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - #[expect(deprecated)] - let primitive = data.array().to_primitive(); + let primitive = data + .array() + .clone() + .execute::(&mut compressor.execution_ctx())?; let for_array = FoR::encode(primitive)?; - #[expect(deprecated)] - let biased = for_array.encoded().to_primitive(); + let biased = for_array + .encoded() + .clone() + .execute::(&mut compressor.execution_ctx())?; // Immediately bitpack. If any other scheme was preferable, it would be chosen instead // of bitpacking. @@ -275,7 +280,9 @@ impl Scheme for ZigZagScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut local_ctx); // ZigZag is only useful when there are negative values. if !stats.erased().min_is_negative() { @@ -293,8 +300,10 @@ impl Scheme for ZigZagScheme { ) -> VortexResult { // Zigzag encode the values, then recursively compress the inner values. let zag = zigzag_encode(data.array_as_primitive())?; - #[expect(deprecated)] - let encoded = zag.encoded().to_primitive(); + let encoded = zag + .encoded() + .clone() + .execute::(&mut compressor.execution_ctx())?; let compressed = compressor.compress_child(&encoded.into_array(), &ctx, self.id(), 0)?; @@ -318,7 +327,9 @@ impl Scheme for BitPackingScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut local_ctx); // BitPacking only works for non-negative values. if stats.erased().min_is_negative() { @@ -330,13 +341,13 @@ impl Scheme for BitPackingScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { let primitive_array = data.array_as_primitive(); - let histogram = bit_width_histogram(primitive_array)?; + let histogram = bit_width_histogram(primitive_array, &mut compressor.execution_ctx())?; let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; // If best bw is determined to be the current bit-width, return the original array. @@ -346,7 +357,12 @@ impl Scheme for BitPackingScheme { // Otherwise we can bitpack the array. let primitive_array = primitive_array.into_owned(); - let packed = bitpack_encode(&primitive_array, bw, Some(&histogram))?; + let packed = bitpack_encode( + &primitive_array, + bw, + Some(&histogram), + &mut compressor.execution_ctx(), + )?; let packed_stats = packed.statistics().to_owned(); let ptype = packed.dtype().as_ptype(); @@ -368,17 +384,19 @@ impl Scheme for BitPackingScheme { match patches { None => array, - Some(p) => Patched::from_array_and_patches( - array, - &p, - &mut LEGACY_SESSION.create_execution_ctx(), - )? - .with_stats_set(packed_stats) - .into_array(), + Some(p) => { + Patched::from_array_and_patches(array, &p, &mut compressor.execution_ctx())? + .with_stats_set(packed_stats) + .into_array() + } } } else { // Compress patches and place back into BitPackedArray. - let patches = parts.patches.take().map(compress_patches).transpose()?; + let patches = parts + .patches + .take() + .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .transpose()?; parts.patches = patches; BitPacked::try_new( parts.packed, @@ -446,7 +464,9 @@ impl Scheme for SparseScheme { _ctx: CompressorContext, ) -> CompressionEstimate { let len = data.array_len() as f64; - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut local_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -492,7 +512,7 @@ impl Scheme for SparseScheme { ) -> VortexResult { let len = data.array_len(); // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats().clone(); + let stats = data.integer_stats(&mut compressor.execution_ctx()).clone(); let array = data.array(); let (most_frequent_value, most_frequent_count) = stats @@ -522,11 +542,15 @@ impl Scheme for SparseScheme { most_frequent_value.ptype(), array.dtype().nullability(), )), + &mut compressor.execution_ctx(), )?; if let Some(sparse) = sparse_encoded.as_opt::() { - #[expect(deprecated)] - let sparse_values_primitive = sparse.patches().values().to_primitive(); + let sparse_values_primitive = sparse + .patches() + .values() + .clone() + .execute::(&mut compressor.execution_ctx())?; let compressed_values = compressor.compress_child( &sparse_values_primitive.into_array(), &ctx, @@ -534,8 +558,12 @@ impl Scheme for SparseScheme { 0, )?; - #[expect(deprecated)] - let indices = sparse.patches().indices().to_primitive().narrow()?; + let indices = sparse + .patches() + .indices() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_indices = compressor.compress_child(&indices.into_array(), &ctx, self.id(), 1)?; @@ -615,7 +643,9 @@ impl Scheme for RunEndScheme { _ctx: CompressorContext, ) -> CompressionEstimate { // If the run length is below the threshold, drop it. - if data.integer_stats().average_run_length() < RUN_END_THRESHOLD { + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + if data.integer_stats(&mut local_ctx).average_run_length() < RUN_END_THRESHOLD { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } @@ -629,10 +659,10 @@ impl Scheme for RunEndScheme { ctx: CompressorContext, ) -> VortexResult { // Run-end encode the ends. - let (ends, values) = runend_encode(data.array_as_primitive()); + let (ends, values) = + runend_encode(data.array_as_primitive(), &mut compressor.execution_ctx()); - #[expect(deprecated)] - let values_primitive = values.to_primitive(); + let values_primitive = values.execute::(&mut compressor.execution_ctx())?; let compressed_values = compressor.compress_child(&values_primitive.into_array(), &ctx, self.id(), 0)?; @@ -640,8 +670,14 @@ impl Scheme for RunEndScheme { // SAFETY: compression doesn't affect invariants. Ok(unsafe { - RunEnd::new_unchecked(compressed_ends, compressed_values, 0, data.array_len()) - .into_array() + RunEnd::new_unchecked( + compressed_ends, + compressed_values, + 0, + data.array_len(), + &mut compressor.execution_ctx(), + ) + .into_array() }) } } @@ -686,7 +722,9 @@ impl Scheme for SequenceScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut local_ctx); // `SequenceArray` does not support nulls. if stats.null_count() > 0 { @@ -706,8 +744,10 @@ impl Scheme for SequenceScheme { // why do we divide the ratio by 2??? CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, _ctx| { - let Some(encoded) = sequence_encode(data.array_as_primitive())? else { + |compressor, data, _ctx| { + let Some(encoded) = + sequence_encode(data.array_as_primitive(), &mut compressor.execution_ctx())? + else { // If we are unable to sequence encode this array, make sure we skip. return Ok(EstimateVerdict::Skip); }; @@ -722,16 +762,16 @@ impl Scheme for SequenceScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { - let stats = data.integer_stats(); + let stats = data.integer_stats(&mut compressor.execution_ctx()); if stats.null_count() > 0 { vortex_bail!("sequence encoding does not support nulls"); } - sequence_encode(data.array_as_primitive())? + sequence_encode(data.array_as_primitive(), &mut compressor.execution_ctx())? .ok_or_else(|| vortex_err!("cannot sequence encode array")) } } @@ -763,7 +803,7 @@ impl Scheme for PcoScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { @@ -771,6 +811,7 @@ impl Scheme for PcoScheme { data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, + &mut compressor.execution_ctx(), )? .into_array()) } @@ -783,18 +824,23 @@ pub(crate) fn rle_compress( data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let rle_array = RLE::encode(data.array_as_primitive())?; + let rle_array = RLE::encode(data.array_as_primitive(), &mut compressor.execution_ctx())?; - #[expect(deprecated)] - let rle_values_primitive = rle_array.values().to_primitive(); + let rle_values_primitive = rle_array + .values() + .clone() + .execute::(&mut compressor.execution_ctx())?; let compressed_values = compressor.compress_child(&rle_values_primitive.into_array(), &ctx, scheme.id(), 0)?; // Delta is an unstable encoding, once we deem it stable we can switch over to this always. #[cfg(feature = "unstable_encodings")] let compressed_indices = { - #[expect(deprecated)] - let rle_indices_primitive = rle_array.indices().to_primitive().narrow()?; + let rle_indices_primitive = rle_array + .indices() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; try_compress_delta( compressor, &rle_indices_primitive.into_array(), @@ -806,13 +852,19 @@ pub(crate) fn rle_compress( #[cfg(not(feature = "unstable_encodings"))] let compressed_indices = { - #[expect(deprecated)] - let rle_indices_primitive = rle_array.indices().to_primitive().narrow()?; + let rle_indices_primitive = rle_array + .indices() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; compressor.compress_child(&rle_indices_primitive.into_array(), &ctx, scheme.id(), 1)? }; - #[expect(deprecated)] - let rle_offsets_primitive = rle_array.values_idx_offsets().to_primitive().narrow()?; + let rle_offsets_primitive = rle_array + .values_idx_offsets() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_offsets = compressor.compress_child(&rle_offsets_primitive.into_array(), &ctx, scheme.id(), 2)?; @@ -837,8 +889,9 @@ fn try_compress_delta( parent_id: SchemeId, child_index: usize, ) -> VortexResult { - #[expect(deprecated)] - let child_primitive = child.to_primitive(); + let child_primitive = child + .clone() + .execute::(&mut compressor.execution_ctx())?; let (bases, deltas) = vortex_fastlanes::delta_compress(&child_primitive, &mut compressor.execution_ctx())?; @@ -882,7 +935,9 @@ impl Scheme for IntRLEScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - if data.integer_stats().average_run_length() < RUN_LENGTH_THRESHOLD { + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + if data.integer_stats(&mut local_ctx).average_run_length() < RUN_LENGTH_THRESHOLD { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } diff --git a/vortex-btrblocks/src/schemes/patches.rs b/vortex-btrblocks/src/schemes/patches.rs index 3513ea5ad06..0d9854c3b23 100644 --- a/vortex-btrblocks/src/schemes/patches.rs +++ b/vortex-btrblocks/src/schemes/patches.rs @@ -2,32 +2,33 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::patches::Patches; use vortex_error::VortexError; use vortex_error::VortexResult; /// Compresses the given patches by downscaling integers and checking for constant values. -pub fn compress_patches(patches: Patches) -> VortexResult { +pub fn compress_patches(patches: Patches, ctx: &mut ExecutionCtx) -> VortexResult { // Downscale the patch indices. - #[expect(deprecated)] - let indices = patches.indices().to_primitive().narrow()?.into_array(); + let indices = patches + .indices() + .clone() + .execute::(ctx)? + .narrow()? + .into_array(); // Check if the values are constant. let values = patches.values(); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = if values .statistics() - .compute_is_constant(&mut ctx) + .compute_is_constant(ctx) .unwrap_or_default() { - ConstantArray::new(values.execute_scalar(0, &mut ctx)?, values.len()).into_array() + ConstantArray::new(values.execute_scalar(0, ctx)?, values.len()).into_array() } else { values.clone() }; @@ -35,8 +36,11 @@ pub fn compress_patches(patches: Patches) -> VortexResult { .chunk_offsets() .as_ref() .map(|offsets| { - #[expect(deprecated)] - let offsets_primitive = offsets.to_primitive().narrow()?.into_array(); + let offsets_primitive = offsets + .clone() + .execute::(ctx)? + .narrow()? + .into_array(); Ok::(offsets_primitive) }) .transpose()?; diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 60bee2cf099..6aee7e8d634 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -6,8 +6,9 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrays::varbin::VarBinArrayExt; @@ -87,10 +88,19 @@ impl Scheme for FSSTScheme { ) -> VortexResult { let utf8 = data.array_as_utf8().into_owned(); let compressor_fsst = fsst_train_compressor(&utf8); - let fsst = fsst_compress(&utf8, utf8.len(), utf8.dtype(), &compressor_fsst); - - #[expect(deprecated)] - let uncompressed_lengths_primitive = fsst.uncompressed_lengths().to_primitive().narrow()?; + let fsst = fsst_compress( + &utf8, + utf8.len(), + utf8.dtype(), + &compressor_fsst, + &mut compressor.execution_ctx(), + ); + + let uncompressed_lengths_primitive = fsst + .uncompressed_lengths() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_original_lengths = compressor.compress_child( &uncompressed_lengths_primitive.into_array(), &ctx, @@ -98,8 +108,12 @@ impl Scheme for FSSTScheme { 0, )?; - #[expect(deprecated)] - let codes_offsets_primitive = fsst.codes().offsets().to_primitive().narrow()?; + let codes_offsets_primitive = fsst + .codes() + .offsets() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_codes_offsets = compressor.compress_child(&codes_offsets_primitive.into_array(), &ctx, self.id(), 1)?; let compressed_codes = VarBinArray::try_new( @@ -115,6 +129,7 @@ impl Scheme for FSSTScheme { fsst.symbol_lengths().clone(), compressed_codes, compressed_original_lengths, + &mut compressor.execution_ctx(), )?; Ok(fsst.into_array()) @@ -155,7 +170,9 @@ impl Scheme for NullDominatedSparseScheme { _ctx: CompressorContext, ) -> CompressionEstimate { let len = data.array_len() as f64; - let stats = data.string_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.string_stats(&mut local_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -179,12 +196,16 @@ impl Scheme for NullDominatedSparseScheme { ctx: CompressorContext, ) -> VortexResult { // We pass None as we only run this pathway for NULL-dominated string arrays. - let sparse_encoded = Sparse::encode(data.array(), None)?; + let sparse_encoded = Sparse::encode(data.array(), None, &mut compressor.execution_ctx())?; if let Some(sparse) = sparse_encoded.as_opt::() { // Compress the indices only (not the values for strings). - #[expect(deprecated)] - let indices = sparse.patches().indices().to_primitive().narrow()?; + let indices = sparse + .patches() + .indices() + .clone() + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let compressed_indices = compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; @@ -221,12 +242,18 @@ impl Scheme for ZstdScheme { fn compress( &self, - _compressor: &CascadingCompressor, + compressor: &CascadingCompressor, data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { let compacted = data.array_as_utf8().into_owned().compact_buffers()?; - Ok(vortex_zstd::Zstd::from_var_bin_view_without_dict(&compacted, 3, 8192)?.into_array()) + Ok(vortex_zstd::Zstd::from_var_bin_view_without_dict( + &compacted, + 3, + 8192, + &mut compressor.execution_ctx(), + )? + .into_array()) } } @@ -254,10 +281,7 @@ impl Scheme for ZstdBuffersScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> VortexResult { - Ok( - vortex_zstd::ZstdBuffers::compress(data.array(), 3, &vortex_array::LEGACY_SESSION)? - .into_array(), - ) + Ok(vortex_zstd::ZstdBuffers::compress(data.array(), 3, &LEGACY_SESSION)?.into_array()) } } diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index c0ab1f3881c..a2eeeb6774a 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -6,12 +6,10 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::extension::Matcher; @@ -76,8 +74,7 @@ impl Scheme for TemporalScheme { ctx: CompressorContext, ) -> VortexResult { let array = data.array().clone(); - #[expect(deprecated)] - let ext_array = array.to_extension(); + let ext_array = array.execute::(&mut compressor.execution_ctx())?; let temporal_array = TemporalArray::try_from(ext_array.clone().into_array())?; // Check for constant array and return early if so. @@ -88,7 +85,7 @@ impl Scheme for TemporalScheme { if is_constant { return Ok(ConstantArray::new( - ext_array.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?, + ext_array.execute_scalar(0, &mut compressor.execution_ctx())?, ext_array.len(), ) .into_array()); @@ -99,17 +96,20 @@ impl Scheme for TemporalScheme { days, seconds, subseconds, - } = split_temporal(temporal_array)?; + } = split_temporal(temporal_array, &mut compressor.execution_ctx())?; - #[expect(deprecated)] - let days_primitive = days.to_primitive().narrow()?; + let days_primitive = days + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let days = compressor.compress_child(&days_primitive.into_array(), &ctx, self.id(), 0)?; - #[expect(deprecated)] - let seconds_primitive = seconds.to_primitive().narrow()?; + let seconds_primitive = seconds + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let seconds = compressor.compress_child(&seconds_primitive.into_array(), &ctx, self.id(), 1)?; - #[expect(deprecated)] - let subseconds_primitive = subseconds.to_primitive().narrow()?; + let subseconds_primitive = subseconds + .execute::(&mut compressor.execution_ctx())? + .narrow()?; let subseconds = compressor.compress_child(&subseconds_primitive.into_array(), &ctx, self.id(), 2)?; diff --git a/vortex-compressor/benches/dict_encode.rs b/vortex-compressor/benches/dict_encode.rs index fee2e42e1f8..02eaa906cb1 100644 --- a/vortex-compressor/benches/dict_encode.rs +++ b/vortex-compressor/benches/dict_encode.rs @@ -5,6 +5,8 @@ use divan::Bencher; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::dict::dict_encode; @@ -40,7 +42,7 @@ fn encode_generic(bencher: Bencher) { #[divan::bench] fn encode_specialized(bencher: Bencher) { let array = make_array(); - let stats = IntegerStats::generate(&array); + let stats = IntegerStats::generate(&array, &mut LEGACY_SESSION.create_execution_ctx()); bencher .with_inputs(|| &stats) .bench_refs(|stats| integer_dictionary_encode(array.as_view(), stats)); diff --git a/vortex-compressor/benches/stats_calc.rs b/vortex-compressor/benches/stats_calc.rs index e435350851c..c93a8373801 100644 --- a/vortex-compressor/benches/stats_calc.rs +++ b/vortex-compressor/benches/stats_calc.rs @@ -5,6 +5,8 @@ #[divan::bench_group(items_count = 64_000u32, bytes_count = 256_000u32)] mod benchmarks { use divan::Bencher; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::Buffer; @@ -53,9 +55,11 @@ mod benchmarks { Distribution::LongRuns => generate_runs(64), }; - bencher.with_inputs(|| &values).bench_refs(|values| { - IntegerStats::generate_opts(values, GenerateStatsOptions::default()); - }); + bencher + .with_inputs(|| (&values, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(values, ctx)| { + IntegerStats::generate_opts(values, GenerateStatsOptions::default(), ctx); + }); } #[divan::bench(args = [Distribution::LowCardinality, Distribution::ShortRuns, Distribution::LongRuns])] @@ -66,14 +70,17 @@ mod benchmarks { Distribution::LongRuns => generate_runs(64), }; - bencher.with_inputs(|| &values).bench_refs(|values| { - IntegerStats::generate_opts( - values, - GenerateStatsOptions { - count_distinct_values: false, - }, - ); - }); + bencher + .with_inputs(|| (&values, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(values, ctx)| { + IntegerStats::generate_opts( + values, + GenerateStatsOptions { + count_distinct_values: false, + }, + ctx, + ); + }); } } diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 5d9957c2ca7..29dc7b7163d 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -690,25 +690,25 @@ pub fn vortex_compressor::stats::ArrayAndStats::array_as_utf8(&self) -> vortex_a pub fn vortex_compressor::stats::ArrayAndStats::array_len(&self) -> usize -pub fn vortex_compressor::stats::ArrayAndStats::bool_stats(&mut self) -> &vortex_compressor::stats::BoolStats +pub fn vortex_compressor::stats::ArrayAndStats::bool_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::BoolStats -pub fn vortex_compressor::stats::ArrayAndStats::float_stats(&mut self) -> &vortex_compressor::stats::FloatStats +pub fn vortex_compressor::stats::ArrayAndStats::float_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::FloatStats pub fn vortex_compressor::stats::ArrayAndStats::get_or_insert_with(&mut self, f: impl core::ops::function::FnOnce() -> T) -> &T -pub fn vortex_compressor::stats::ArrayAndStats::integer_stats(&mut self) -> &vortex_compressor::stats::IntegerStats +pub fn vortex_compressor::stats::ArrayAndStats::integer_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::IntegerStats pub fn vortex_compressor::stats::ArrayAndStats::into_array(self) -> vortex_array::array::erased::ArrayRef pub fn vortex_compressor::stats::ArrayAndStats::new(array: vortex_array::array::erased::ArrayRef, opts: vortex_compressor::stats::GenerateStatsOptions) -> Self -pub fn vortex_compressor::stats::ArrayAndStats::string_stats(&mut self) -> &vortex_compressor::stats::StringStats +pub fn vortex_compressor::stats::ArrayAndStats::string_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::StringStats pub struct vortex_compressor::stats::BoolStats impl vortex_compressor::stats::BoolStats -pub fn vortex_compressor::stats::BoolStats::generate(input: &vortex_array::arrays::bool::vtable::BoolArray) -> vortex_error::VortexResult +pub fn vortex_compressor::stats::BoolStats::generate(input: &vortex_array::arrays::bool::vtable::BoolArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::stats::BoolStats::is_constant(&self) -> bool @@ -748,9 +748,9 @@ pub fn vortex_compressor::stats::FloatStats::average_run_length(&self) -> u32 pub fn vortex_compressor::stats::FloatStats::erased(&self) -> &vortex_compressor::stats::FloatErasedStats -pub fn vortex_compressor::stats::FloatStats::generate(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> Self +pub fn vortex_compressor::stats::FloatStats::generate(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self -pub fn vortex_compressor::stats::FloatStats::generate_opts(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, opts: vortex_compressor::stats::GenerateStatsOptions) -> Self +pub fn vortex_compressor::stats::FloatStats::generate_opts(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, opts: vortex_compressor::stats::GenerateStatsOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self pub fn vortex_compressor::stats::FloatStats::null_count(&self) -> u32 @@ -838,9 +838,9 @@ pub fn vortex_compressor::stats::IntegerStats::average_run_length(&self) -> u32 pub fn vortex_compressor::stats::IntegerStats::erased(&self) -> &vortex_compressor::stats::IntegerErasedStats -pub fn vortex_compressor::stats::IntegerStats::generate(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> Self +pub fn vortex_compressor::stats::IntegerStats::generate(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self -pub fn vortex_compressor::stats::IntegerStats::generate_opts(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, opts: vortex_compressor::stats::GenerateStatsOptions) -> Self +pub fn vortex_compressor::stats::IntegerStats::generate_opts(input: &vortex_array::arrays::primitive::vtable::PrimitiveArray, opts: vortex_compressor::stats::GenerateStatsOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self pub fn vortex_compressor::stats::IntegerStats::null_count(&self) -> u32 @@ -912,9 +912,9 @@ impl vortex_compressor::stats::StringStats pub fn vortex_compressor::stats::StringStats::estimated_distinct_count(&self) -> core::option::Option -pub fn vortex_compressor::stats::StringStats::generate(input: &vortex_array::arrays::varbinview::vtable::VarBinViewArray) -> Self +pub fn vortex_compressor::stats::StringStats::generate(input: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self -pub fn vortex_compressor::stats::StringStats::generate_opts(input: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, opts: vortex_compressor::stats::GenerateStatsOptions) -> Self +pub fn vortex_compressor::stats::StringStats::generate_opts(input: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, opts: vortex_compressor::stats::GenerateStatsOptions, ctx: &mut vortex_array::executor::ExecutionCtx) -> Self pub fn vortex_compressor::stats::StringStats::null_count(&self) -> u32 diff --git a/vortex-compressor/src/builtins/constant/bool.rs b/vortex-compressor/src/builtins/constant/bool.rs index 335bd29c2ba..9ed23911f2d 100644 --- a/vortex-compressor/src/builtins/constant/bool.rs +++ b/vortex-compressor/src/builtins/constant/bool.rs @@ -5,6 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_error::VortexResult; use crate::CascadingCompressor; @@ -37,7 +39,9 @@ impl Scheme for BoolConstantScheme { } let array_len = data.array().len(); - let stats = data.bool_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.bool_stats(&mut ctx); // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { diff --git a/vortex-compressor/src/builtins/constant/float.rs b/vortex-compressor/src/builtins/constant/float.rs index 6cee2014334..3c9214f3c37 100644 --- a/vortex-compressor/src/builtins/constant/float.rs +++ b/vortex-compressor/src/builtins/constant/float.rs @@ -5,6 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_error::VortexResult; @@ -40,7 +42,10 @@ impl Scheme for FloatConstantScheme { } let array_len = data.array().len(); - let stats = data.float_stats(); + // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature + // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. + let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.float_stats(&mut exec_ctx); // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { diff --git a/vortex-compressor/src/builtins/constant/integer.rs b/vortex-compressor/src/builtins/constant/integer.rs index 1062664ca69..e98fc343731 100644 --- a/vortex-compressor/src/builtins/constant/integer.rs +++ b/vortex-compressor/src/builtins/constant/integer.rs @@ -5,6 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_error::VortexResult; use super::is_integer_primitive; @@ -38,7 +40,9 @@ impl Scheme for IntConstantScheme { } let array_len = data.array().len(); - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut ctx); // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { diff --git a/vortex-compressor/src/builtins/constant/string.rs b/vortex-compressor/src/builtins/constant/string.rs index 694eae16872..77bbb5d49c7 100644 --- a/vortex-compressor/src/builtins/constant/string.rs +++ b/vortex-compressor/src/builtins/constant/string.rs @@ -5,6 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_error::VortexResult; @@ -40,7 +42,9 @@ impl Scheme for StringConstantScheme { } let array_len = data.array().len(); - let stats = data.string_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.string_stats(&mut local_ctx); // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index a18926961e1..e800c2c50e7 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -10,6 +10,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -85,7 +87,10 @@ impl Scheme for FloatDictScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - let stats = data.float_stats(); + // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature + // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. + let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.float_stats(&mut exec_ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); @@ -111,7 +116,10 @@ impl Scheme for FloatDictScheme { ctx: CompressorContext, ) -> VortexResult { // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.float_stats().clone(); + let stats = { + let mut exec_ctx = compressor.execution_ctx(); + data.float_stats(&mut exec_ctx).clone() + }; let dict = dictionary_encode(data.array_as_primitive(), &stats)?; let has_all_values_referenced = dict.has_all_values_referenced(); @@ -242,21 +250,23 @@ impl_encode!(f64, u64); #[cfg(test)] mod tests { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_error::VortexResult; use super::dictionary_encode; use crate::stats::FloatStats; use crate::stats::GenerateStatsOptions; #[test] - fn test_float_dict_encode() { + fn test_float_dict_encode() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); @@ -267,8 +277,9 @@ mod tests { GenerateStatsOptions { count_distinct_values: true, }, + &mut ctx, ); - let dict_array = dictionary_encode(array.as_view(), &stats).unwrap(); + let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); @@ -277,8 +288,12 @@ mod tests { Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); - #[expect(deprecated)] - let undict = dict_array.as_array().to_primitive().into_array(); + let undict = dict_array + .as_array() + .clone() + .execute::(&mut ctx)? + .into_array(); assert_arrays_eq!(undict, expected); + Ok(()) } } diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 135dfd8080c..13e3559e989 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -10,6 +10,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -60,7 +62,9 @@ impl Scheme for IntDictScheme { _ctx: CompressorContext, ) -> CompressionEstimate { let bit_width = data.array_as_primitive().ptype().bit_width(); - let stats = data.integer_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.integer_stats(&mut ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); @@ -105,7 +109,7 @@ impl Scheme for IntDictScheme { ctx: CompressorContext, ) -> VortexResult { // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats().clone(); + let stats = data.integer_stats(&mut compressor.execution_ctx()).clone(); let dict = dictionary_encode(data.array_as_primitive(), &stats)?; // Values = child 0. @@ -249,20 +253,22 @@ impl_encode!(i64); #[cfg(test)] mod tests { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_error::VortexResult; use super::dictionary_encode; use crate::stats::IntegerStats; #[test] - fn test_dict_encode_integer_stats() { + fn test_dict_encode_integer_stats() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data = buffer![100i32, 200, 100, 0, 100]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); @@ -273,8 +279,9 @@ mod tests { crate::stats::GenerateStatsOptions { count_distinct_values: true, }, + &mut ctx, ); - let dict_array = dictionary_encode(array.as_view(), &stats).unwrap(); + let dict_array = dictionary_encode(array.as_view(), &stats)?; assert_eq!(dict_array.values().len(), 2); assert_eq!(dict_array.codes().len(), 5); @@ -283,8 +290,12 @@ mod tests { Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()), ) .into_array(); - #[expect(deprecated)] - let undict = dict_array.as_array().to_primitive().into_array(); + let undict = dict_array + .as_array() + .clone() + .execute::(&mut ctx)? + .into_array(); assert_arrays_eq!(undict, expected); + Ok(()) } } diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index efc5511fc29..82a30b2774b 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -9,6 +9,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; @@ -70,7 +72,9 @@ impl Scheme for StringDictScheme { data: &mut ArrayAndStats, _ctx: CompressorContext, ) -> CompressionEstimate { - let stats = data.string_stats(); + // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. + let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); + let stats = data.string_stats(&mut local_ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 26585a352a9..36ed7129d63 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -13,14 +13,13 @@ use vortex_array::CanonicalValidity; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListArray; use vortex_array::arrays::ListViewArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -311,7 +310,7 @@ impl CascadingCompressor { if array.is_empty() { return Ok(array); } - if array.all_invalid(&mut LEGACY_SESSION.create_execution_ctx())? { + if array.all_invalid(&mut self.execution_ctx())? { return Ok( ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()).into_array(), ); @@ -488,8 +487,11 @@ impl CascadingCompressor { // Record the root scheme with the offsets child index so root exclusion rules apply. let offset_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); - #[expect(deprecated)] - let list_offsets_primitive = list_array.offsets().to_primitive().narrow()?; + let list_offsets_primitive = list_array + .offsets() + .clone() + .execute::(&mut self.execution_ctx())? + .narrow()?; let compressed_offsets = self.compress_canonical(Canonical::Primitive(list_offsets_primitive), offset_ctx)?; @@ -511,16 +513,22 @@ impl CascadingCompressor { let offset_ctx = ctx .clone() .descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); - #[expect(deprecated)] - let list_view_offsets_primitive = list_view.offsets().to_primitive().narrow()?; + let list_view_offsets_primitive = list_view + .offsets() + .clone() + .execute::(&mut self.execution_ctx())? + .narrow()?; let compressed_offsets = self.compress_canonical( Canonical::Primitive(list_view_offsets_primitive), offset_ctx, )?; let sizes_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::SIZES); - #[expect(deprecated)] - let list_view_sizes_primitive = list_view.sizes().to_primitive().narrow()?; + let list_view_sizes_primitive = list_view + .sizes() + .clone() + .execute::(&mut self.execution_ctx())? + .narrow()?; let compressed_sizes = self.compress_canonical(Canonical::Primitive(list_view_sizes_primitive), sizes_ctx)?; diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index 9c65be4d049..283ad91b5fb 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -3,8 +3,7 @@ //! Bool compression statistics. -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_error::VortexResult; @@ -27,7 +26,7 @@ impl BoolStats { /// # Errors /// /// Returns an error if getting validity mask fails or values exceed `u32` bounds. - pub fn generate(input: &BoolArray) -> VortexResult { + pub fn generate(input: &BoolArray, ctx: &mut ExecutionCtx) -> VortexResult { if input.is_empty() { return Ok(Self { null_count: 0, @@ -36,8 +35,7 @@ impl BoolStats { }); } - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - if input.all_invalid(&mut ctx)? { + if input.all_invalid(ctx)? { return Ok(Self { null_count: u32::try_from(input.len())?, value_count: 0, @@ -48,7 +46,7 @@ impl BoolStats { let validity = input .as_ref() .validity()? - .to_mask(input.as_ref().len(), &mut ctx)?; + .to_mask(input.as_ref().len(), ctx)?; let null_count = validity.false_count(); let value_count = validity.true_count(); @@ -94,6 +92,8 @@ impl BoolStats { #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -103,11 +103,12 @@ mod tests { #[test] fn test_all_true() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = BoolArray::new( BitBuffer::from(vec![true, true, true]), Validity::NonNullable, ); - let stats = BoolStats::generate(&array)?; + let stats = BoolStats::generate(&array, &mut ctx)?; assert_eq!(stats.value_count, 3); assert_eq!(stats.null_count, 0); assert_eq!(stats.true_count, 3); @@ -117,11 +118,12 @@ mod tests { #[test] fn test_all_false() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = BoolArray::new( BitBuffer::from(vec![false, false, false]), Validity::NonNullable, ); - let stats = BoolStats::generate(&array)?; + let stats = BoolStats::generate(&array, &mut ctx)?; assert_eq!(stats.value_count, 3); assert_eq!(stats.null_count, 0); assert_eq!(stats.true_count, 0); @@ -131,11 +133,12 @@ mod tests { #[test] fn test_mixed() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = BoolArray::new( BitBuffer::from(vec![true, false, true]), Validity::NonNullable, ); - let stats = BoolStats::generate(&array)?; + let stats = BoolStats::generate(&array, &mut ctx)?; assert_eq!(stats.value_count, 3); assert_eq!(stats.null_count, 0); assert_eq!(stats.true_count, 2); @@ -145,11 +148,12 @@ mod tests { #[test] fn test_with_nulls() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = BoolArray::new( BitBuffer::from(vec![true, false, true]), Validity::from_iter([true, false, true]), ); - let stats = BoolStats::generate(&array)?; + let stats = BoolStats::generate(&array, &mut ctx)?; assert_eq!(stats.value_count, 2); assert_eq!(stats.null_count, 1); assert_eq!(stats.true_count, 2); diff --git a/vortex-compressor/src/stats/cache.rs b/vortex-compressor/src/stats/cache.rs index 28d7d0bc390..5f53cb83d41 100644 --- a/vortex-compressor/src/stats/cache.rs +++ b/vortex-compressor/src/stats/cache.rs @@ -8,8 +8,8 @@ use std::any::TypeId; use vortex_array::ArrayRef; use vortex_array::ArrayView; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::ExecutionCtx; +use vortex_array::arrays::Bool; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinView; use vortex_error::VortexExpect; @@ -138,51 +138,59 @@ impl ArrayAndStats { } /// Returns bool stats, generating them lazily on first access. - pub fn bool_stats(&mut self) -> &BoolStats { + pub fn bool_stats(&mut self, ctx: &mut ExecutionCtx) -> &BoolStats { let array = self.array.clone(); self.cache.get_or_insert_with::(|| { - #[expect(deprecated)] - let bool_array = array.to_bool(); - BoolStats::generate(&bool_array).vortex_expect("BoolStats shouldn't fail") + let bool_array = array + .as_opt::() + .vortex_expect("the array is guaranteed to already be canonical by construction") + .into_owned(); + BoolStats::generate(&bool_array, ctx).vortex_expect("BoolStats shouldn't fail") }) } // TODO(connor): These should all have interior mutability instead!!! /// Returns integer stats, generating them lazily on first access. - pub fn integer_stats(&mut self) -> &IntegerStats { + pub fn integer_stats(&mut self, ctx: &mut ExecutionCtx) -> &IntegerStats { let array = self.array.clone(); let opts = self.opts; self.cache.get_or_insert_with::(|| { - #[expect(deprecated)] - let primitive = array.to_primitive(); - IntegerStats::generate_opts(&primitive, opts) + let primitive = array + .as_opt::() + .vortex_expect("the array is guaranteed to already be canonical by construction") + .into_owned(); + IntegerStats::generate_opts(&primitive, opts, ctx) }) } /// Returns float stats, generating them lazily on first access. - pub fn float_stats(&mut self) -> &FloatStats { + pub fn float_stats(&mut self, ctx: &mut ExecutionCtx) -> &FloatStats { let array = self.array.clone(); let opts = self.opts; self.cache.get_or_insert_with::(|| { - #[expect(deprecated)] - let primitive = array.to_primitive(); - FloatStats::generate_opts(&primitive, opts) + let primitive = array + .as_opt::() + .vortex_expect("the array is guaranteed to already be canonical by construction") + .into_owned(); + FloatStats::generate_opts(&primitive, opts, ctx) }) } /// Returns string stats, generating them lazily on first access. - pub fn string_stats(&mut self) -> &StringStats { + pub fn string_stats(&mut self, ctx: &mut ExecutionCtx) -> &StringStats { let array = self.array.clone(); let opts = self.opts; self.cache.get_or_insert_with::(|| { - #[expect(deprecated)] - let varbinview = array.to_varbinview(); - StringStats::generate_opts(&varbinview, opts) + let varbinview = array + .as_opt::() + .vortex_expect("the array is guaranteed to already be canonical by construction") + .into_owned(); + StringStats::generate_opts(&varbinview, opts, ctx) }) } diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 74b1a9828b3..0400bac6cdf 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -8,8 +8,7 @@ use std::hash::Hash; use itertools::Itertools; use num_traits::Float; use rustc_hash::FxBuildHasher; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::NativePType; @@ -109,11 +108,12 @@ impl FloatStats { fn generate_opts_fallible( input: &PrimitiveArray, opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, ) -> VortexResult { match input.ptype() { - PType::F16 => typed_float_stats::(input, opts.count_distinct_values), - PType::F32 => typed_float_stats::(input, opts.count_distinct_values), - PType::F64 => typed_float_stats::(input, opts.count_distinct_values), + PType::F16 => typed_float_stats::(input, opts.count_distinct_values, ctx), + PType::F32 => typed_float_stats::(input, opts.count_distinct_values, ctx), + PType::F64 => typed_float_stats::(input, opts.count_distinct_values, ctx), _ => vortex_panic!("cannot generate FloatStats from ptype {}", input.ptype()), } } @@ -126,13 +126,17 @@ impl FloatStats { impl FloatStats { /// Generates stats with default options. - pub fn generate(input: &PrimitiveArray) -> Self { - Self::generate_opts(input, GenerateStatsOptions::default()) + pub fn generate(input: &PrimitiveArray, ctx: &mut ExecutionCtx) -> Self { + Self::generate_opts(input, GenerateStatsOptions::default(), ctx) } /// Generates stats with provided options. - pub fn generate_opts(input: &PrimitiveArray, opts: GenerateStatsOptions) -> Self { - Self::generate_opts_fallible(input, opts) + pub fn generate_opts( + input: &PrimitiveArray, + opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, + ) -> Self { + Self::generate_opts_fallible(input, opts, ctx) .vortex_expect("FloatStats::generate_opts should not fail") } @@ -161,6 +165,7 @@ impl FloatStats { fn typed_float_stats( array: &PrimitiveArray, count_distinct_values: bool, + ctx: &mut ExecutionCtx, ) -> VortexResult where NativeValue: Hash + Eq, @@ -176,8 +181,7 @@ where }); } - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - if array.all_invalid(&mut ctx)? { + if array.all_invalid(ctx)? { return Ok(FloatStats { null_count: u32::try_from(array.len())?, value_count: 0, @@ -194,7 +198,7 @@ where let null_count = array .statistics() - .compute_null_count(&mut ctx) + .compute_null_count(ctx) .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = array.len() - null_count; @@ -206,10 +210,10 @@ where HashSet::with_hasher(FxBuildHasher) }; - let validity = array.as_ref().validity()?.to_mask( - array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let mut runs = 1; let head_idx = validity @@ -272,35 +276,39 @@ where #[cfg(test)] mod tests { use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_error::VortexResult; use super::FloatStats; #[test] - fn test_float_stats() { + fn test_float_stats() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let floats = buffer![0.0f32, 1.0f32, 2.0f32].into_array(); - #[expect(deprecated)] - let floats = floats.to_primitive(); + let floats = floats.execute::(&mut ctx)?; let stats = FloatStats::generate_opts( &floats, crate::stats::GenerateStatsOptions { count_distinct_values: true, }, + &mut ctx, ); assert_eq!(stats.value_count, 3); assert_eq!(stats.null_count, 0); assert_eq!(stats.average_run_length, 1); assert_eq!(stats.distinct_count().unwrap(), 3); + Ok(()) } #[test] fn test_float_stats_leading_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let floats = PrimitiveArray::new( buffer![0.0f32, 1.0f32, 2.0f32], Validity::from_iter([false, true, true]), @@ -311,6 +319,7 @@ mod tests { crate::stats::GenerateStatsOptions { count_distinct_values: true, }, + &mut ctx, ); assert_eq!(stats.value_count, 2); diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index b75eee2d7aa..6c45515f39d 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -7,8 +7,7 @@ use std::hash::Hash; use num_traits::PrimInt; use rustc_hash::FxBuildHasher; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::IntegerPType; @@ -257,9 +256,10 @@ impl IntegerStats { fn generate_opts_fallible( input: &PrimitiveArray, opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, ) -> VortexResult { match_each_integer_ptype!(input.ptype(), |T| { - typed_int_stats::(input, opts.count_distinct_values) + typed_int_stats::(input, opts.count_distinct_values, ctx) }) } @@ -276,13 +276,17 @@ impl IntegerStats { impl IntegerStats { /// Generates stats with default options. - pub fn generate(input: &PrimitiveArray) -> Self { - Self::generate_opts(input, GenerateStatsOptions::default()) + pub fn generate(input: &PrimitiveArray, ctx: &mut ExecutionCtx) -> Self { + Self::generate_opts(input, GenerateStatsOptions::default(), ctx) } /// Generates stats with provided options. - pub fn generate_opts(input: &PrimitiveArray, opts: GenerateStatsOptions) -> Self { - Self::generate_opts_fallible(input, opts) + pub fn generate_opts( + input: &PrimitiveArray, + opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, + ) -> Self { + Self::generate_opts_fallible(input, opts, ctx) .vortex_expect("IntegerStats::generate_opts should not fail") } @@ -311,6 +315,7 @@ impl IntegerStats { fn typed_int_stats( array: &PrimitiveArray, count_distinct_values: bool, + ctx: &mut ExecutionCtx, ) -> VortexResult where T: IntegerPType + PrimInt + for<'a> TryFrom<&'a Scalar, Error = VortexError>, @@ -332,8 +337,7 @@ where }); } - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - if array.all_invalid(&mut ctx)? { + if array.all_invalid(ctx)? { return Ok(IntegerStats { null_count: u32::try_from(array.len())?, value_count: 0, @@ -352,10 +356,10 @@ where }); } - let validity = array.as_ref().validity()?.to_mask( - array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let validity = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let null_count = validity.false_count(); let value_count = validity.true_count(); @@ -437,12 +441,12 @@ where let array_ref = array.as_ref(); let min = array_ref .statistics() - .compute_as::(Stat::Min, &mut ctx) + .compute_as::(Stat::Min, ctx) .vortex_expect("min should be computed"); let max = array_ref .statistics() - .compute_as::(Stat::Max, &mut ctx) + .compute_as::(Stat::Max, ctx) .vortex_expect("max should be computed"); let distinct = count_distinct_values.then(|| { @@ -557,6 +561,8 @@ fn inner_loop_naive( mod tests { use std::iter; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -569,46 +575,51 @@ mod tests { #[test] fn test_naive_count_distinct_values() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::new(buffer![217u8, 0], Validity::NonNullable); - let stats = typed_int_stats::(&array, true)?; + let stats = typed_int_stats::(&array, true, &mut ctx)?; assert_eq!(stats.distinct_count().unwrap(), 2); Ok(()) } #[test] fn test_naive_count_distinct_values_nullable() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::new( buffer![217u8, 0], Validity::from(BitBuffer::from(vec![true, false])), ); - let stats = typed_int_stats::(&array, true)?; + let stats = typed_int_stats::(&array, true, &mut ctx)?; assert_eq!(stats.distinct_count().unwrap(), 1); Ok(()) } #[test] fn test_count_distinct_values() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::new((0..128u8).collect::>(), Validity::NonNullable); - let stats = typed_int_stats::(&array, true)?; + let stats = typed_int_stats::(&array, true, &mut ctx)?; assert_eq!(stats.distinct_count().unwrap(), 128); Ok(()) } #[test] fn test_count_distinct_values_nullable() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::new( (0..128u8).collect::>(), Validity::from(BitBuffer::from_iter( iter::repeat_n(vec![true, false], 64).flatten(), )), ); - let stats = typed_int_stats::(&array, true)?; + let stats = typed_int_stats::(&array, true, &mut ctx)?; assert_eq!(stats.distinct_count().unwrap(), 64); Ok(()) } #[test] fn test_integer_stats_leading_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ints = PrimitiveArray::new(buffer![0, 1, 2], Validity::from_iter([false, true, true])); let stats = IntegerStats::generate_opts( @@ -616,6 +627,7 @@ mod tests { crate::stats::GenerateStatsOptions { count_distinct_values: true, }, + &mut ctx, ); assert_eq!(stats.value_count, 2); diff --git a/vortex-compressor/src/stats/string.rs b/vortex-compressor/src/stats/string.rs index 8ed13323adc..8613aa5cc37 100644 --- a/vortex-compressor/src/stats/string.rs +++ b/vortex-compressor/src/stats/string.rs @@ -3,8 +3,7 @@ //! String compression statistics. -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::arrays::VarBinViewArray; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -49,10 +48,11 @@ impl StringStats { fn generate_opts_fallible( input: &VarBinViewArray, opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, ) -> VortexResult { let null_count = input .statistics() - .compute_null_count(&mut LEGACY_SESSION.create_execution_ctx()) + .compute_null_count(ctx) .ok_or_else(|| vortex_err!("Failed to compute null_count"))?; let value_count = input.len() - null_count; let estimated_distinct_count = opts @@ -70,13 +70,17 @@ impl StringStats { impl StringStats { /// Generates stats with default options. - pub fn generate(input: &VarBinViewArray) -> Self { - Self::generate_opts(input, GenerateStatsOptions::default()) + pub fn generate(input: &VarBinViewArray, ctx: &mut ExecutionCtx) -> Self { + Self::generate_opts(input, GenerateStatsOptions::default(), ctx) } /// Generates stats with provided options. - pub fn generate_opts(input: &VarBinViewArray, opts: GenerateStatsOptions) -> Self { - Self::generate_opts_fallible(input, opts) + pub fn generate_opts( + input: &VarBinViewArray, + opts: GenerateStatsOptions, + ctx: &mut ExecutionCtx, + ) -> Self { + Self::generate_opts_fallible(input, opts, ctx) .vortex_expect("StringStats::generate_opts should not fail") } diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index 11c95295714..88015a1cbfb 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -20,6 +20,8 @@ use criterion::Throughput; use cudarc::driver::DeviceRepr; use futures::executor::block_on; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; @@ -56,7 +58,8 @@ where .collect(); let primitive_array = PrimitiveArray::new(Buffer::from(values), NonNullable); - BitPackedData::encode(&primitive_array.into_array(), bit_width) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + BitPackedData::encode(&primitive_array.into_array(), bit_width, &mut ctx) .vortex_expect("failed to create BitPacked array") } @@ -96,7 +99,8 @@ where .collect(); let primitive_array = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - BitPackedData::encode(&primitive_array, bit_width) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + BitPackedData::encode(&primitive_array, bit_width, &mut ctx) .vortex_expect("failed to create BitPacked array with patches") } diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index c814c03f3fe..5b81a2c2120 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -19,8 +19,6 @@ use cudarc::driver::PushKernelArg; use cudarc::driver::sys::CUevent_flags; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; @@ -189,7 +187,9 @@ fn bench_for_bitpacked(c: &mut Criterion) { .map(|i| (i as u64 % (max_val + 1)) as u32) .collect(); let prim = PrimitiveArray::new(Buffer::from(residuals), NonNullable); - let bp = BitPackedData::encode(&prim.into_array(), bit_width).vortex_expect("bitpack"); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let bp = + BitPackedData::encode(&prim.into_array(), bit_width, &mut ctx).vortex_expect("bitpack"); let array = FoR::try_new(bp.into_array(), Scalar::from(reference)) .vortex_expect("for") .into_array(); @@ -232,7 +232,8 @@ fn bench_dict_bp_codes(c: &mut Criterion) { let codes: Vec = (0..*len).map(|i| (i % dict_size) as u32).collect(); let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPackedData::encode(&codes_prim.into_array(), dict_bit_width) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let codes_bp = BitPackedData::encode(&codes_prim.into_array(), dict_bit_width, &mut ctx) .vortex_expect("bitpack codes"); let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); let dict = DictArray::new(codes_bp.into_array(), values_prim.into_array()); @@ -276,9 +277,10 @@ fn bench_runend(c: &mut Criterion) { let ends: Vec = (1..=num_runs).map(|i| (i * run_len) as u32).collect(); let values: Vec = (0..num_runs).map(|i| (i * 7 + 42) as u32).collect(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - let re = RunEnd::new(ends_arr, values_arr); + let re = RunEnd::new(ends_arr, values_arr, &mut ctx); let array = re.into_array(); group.bench_with_input( @@ -318,7 +320,8 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { // Dict values: residuals 0..63 bitpacked, FoR adds 1_000_000 let dict_residuals: Vec = (0..dict_size as u32).collect(); let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); - let dict_bp = BitPackedData::encode(&dict_prim.into_array(), dict_bit_width) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let dict_bp = BitPackedData::encode(&dict_prim.into_array(), dict_bit_width, &mut ctx) .vortex_expect("bitpack dict"); let dict_for = FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference)).vortex_expect("for dict"); @@ -328,7 +331,7 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let codes: Vec = (0..*len).map(|i| (i % dict_size) as u32).collect(); let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPackedData::encode(&codes_prim.into_array(), codes_bit_width) + let codes_bp = BitPackedData::encode(&codes_prim.into_array(), codes_bit_width, &mut ctx) .vortex_expect("bitpack codes"); let dict = DictArray::new(codes_bp.into_array(), dict_for.clone().into_array()); @@ -361,6 +364,7 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { // Benchmark: ALP(FoR(BitPacked)) for f32 // --------------------------------------------------------------------------- fn bench_alp_for_bitpacked(c: &mut Criterion) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut group = c.benchmark_group("alp_for_bp_6bw_f32"); let exponents = Exponents { e: 2, f: 0 }; @@ -376,17 +380,18 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable); // Encode: ALP → FoR → BitPacked - let alp = alp_encode( - float_prim.as_view(), - Some(exponents), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("alp_encode"); + let alp = + alp_encode(float_prim.as_view(), Some(exponents), &mut ctx).vortex_expect("alp_encode"); assert!(alp.patches().is_none()); - #[expect(deprecated)] - let for_arr = FoRData::encode(alp.encoded().to_primitive()).vortex_expect("for encode"); - let bp = - BitPackedData::encode(for_arr.encoded(), bit_width).vortex_expect("bitpack encode"); + let for_arr = FoRData::encode( + alp.encoded() + .clone() + .execute::(&mut ctx) + .vortex_expect("to primitive"), + ) + .vortex_expect("for encode"); + let bp = BitPackedData::encode(for_arr.encoded(), bit_width, &mut ctx) + .vortex_expect("bitpack encode"); let tree = ALP::new( FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone()) diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index 0c28045173c..16f54013176 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -20,6 +20,8 @@ use criterion::Throughput; use cudarc::driver::DeviceRepr; use futures::executor::block_on; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::validity::Validity; use vortex::buffer::Buffer; @@ -56,7 +58,9 @@ where PrimitiveArray::new(Buffer::from(data), Validity::NonNullable).into_array(); if bp && T::PTYPE != PType::U8 { - let child = BitPackedData::encode(&primitive_array, 8).vortex_expect("failed to bitpack"); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let child = + BitPackedData::encode(&primitive_array, 8, &mut ctx).vortex_expect("failed to bitpack"); FoR::try_new(child.into_array(), reference.into()) .vortex_expect("failed to create FoR array") } else { diff --git a/vortex-cuda/benches/runend_cuda.rs b/vortex-cuda/benches/runend_cuda.rs index bb9f6e4f108..1606c6a4dcd 100644 --- a/vortex-cuda/benches/runend_cuda.rs +++ b/vortex-cuda/benches/runend_cuda.rs @@ -34,7 +34,11 @@ use vortex_cuda_macros::cuda_not_available; use crate::common::TimedLaunchStrategy; /// Creates a run-end encoded array with the specified output length and average run length. -fn make_runend_array_typed(output_len: usize, avg_run_len: usize) -> RunEndArray +fn make_runend_array_typed( + output_len: usize, + avg_run_len: usize, + ctx: &mut vortex::array::ExecutionCtx, +) -> RunEndArray where T: NativePType + From, { @@ -55,7 +59,7 @@ where let ends_array = PrimitiveArray::new(Buffer::from(ends), Validity::NonNullable).into_array(); let values_array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable).into_array(); - RunEnd::new(ends_array, values_array) + RunEnd::new(ends_array, values_array, ctx) } /// Benchmark run-end decoding for a specific type with varying run lengths @@ -69,7 +73,8 @@ where group.throughput(Throughput::Bytes((len * size_of::()) as u64)); for run_len in [10, 1000, 100000] { - let runend_array = make_runend_array_typed::(len, run_len); + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let runend_array = make_runend_array_typed::(len, run_len, cuda_ctx.execution_ctx()); group.bench_with_input( BenchmarkId::new("runend", format!("{len_str}_{type_name}_runlen_{run_len}")), diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index 089d32a1fc9..68e411a2253 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -50,14 +50,22 @@ fn generate_string_data(count: usize) -> Vec<&'static str> { } /// Create a ZSTD-compressed array -fn make_zstd_array(num_strings: usize) -> VortexResult<(ZstdArray, usize)> { +fn make_zstd_array( + num_strings: usize, + cuda_ctx: &mut vortex_cuda::CudaExecutionCtx, +) -> VortexResult<(ZstdArray, usize)> { let strings = generate_string_data(num_strings); let var_bin_view = VarBinViewArray::from_iter_str(strings.iter().copied()); let uncompressed_size: usize = strings.iter().map(|s| s.len()).sum(); let zstd_compression_level = -10; // Less compression but faster. let zstd_array = // Disable dictionary as nvCOMP doesn't support ZSTD dictionaries. - Zstd::from_var_bin_view_without_dict(&var_bin_view, zstd_compression_level, 2048)?; + Zstd::from_var_bin_view_without_dict( + &var_bin_view, + zstd_compression_level, + 2048, + cuda_ctx.execution_ctx(), + )?; Ok((zstd_array, uncompressed_size)) } @@ -120,8 +128,10 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut group = c.benchmark_group("ZSTD_cuda"); for (num_strings, label) in BENCH_ARGS { - let (zstd_array, uncompressed_size) = - make_zstd_array(*num_strings).vortex_expect("failed to create ZSTD array"); + let mut setup_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + let (zstd_array, uncompressed_size) = make_zstd_array(*num_strings, &mut setup_ctx) + .vortex_expect("failed to create ZSTD array"); group.throughput(Throughput::Bytes(uncompressed_size as u64)); group.bench_with_input( diff --git a/vortex-cuda/gpu-scan-cli/src/main.rs b/vortex-cuda/gpu-scan-cli/src/main.rs index b58b8248894..9078e30c691 100644 --- a/vortex-cuda/gpu-scan-cli/src/main.rs +++ b/vortex-cuda/gpu-scan-cli/src/main.rs @@ -18,9 +18,9 @@ use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use vortex::VortexSessionDefault; -#[expect(deprecated)] -use vortex::array::ToCanonical as _; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::Dict; +use vortex::array::arrays::StructArray; use vortex::array::arrays::struct_::StructArrayExt; use vortex::buffer::ByteBufferMut; use vortex::compressor::BtrBlocksCompressorBuilder; @@ -179,8 +179,7 @@ async fn cmd_scan(path: PathBuf, gpu_file: bool, json_output: bool) -> VortexRes let mut chunk = 0; while let Some(next) = batches.next().await.transpose()? { - #[expect(deprecated)] - let record = next.to_struct(); + let record = next.execute::(cuda_ctx.execution_ctx())?; for (field, field_name) in record .iter_unmasked_fields() diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 3baf6fce9e3..7b5bfadd5ce 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -5,8 +5,7 @@ use async_trait::async_trait; use futures::future::BoxFuture; use vortex::array::ArrayRef; use vortex::array::Canonical; -#[expect(deprecated)] -use vortex::array::ToCanonical; +use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolDataParts; use vortex::array::arrays::decimal::DecimalDataParts; @@ -117,8 +116,10 @@ fn export_canonical( vortex_bail!("only support temporal extension types currently"); } - #[expect(deprecated)] - let values = extension.storage_array().to_primitive(); + let values = extension + .storage_array() + .clone() + .execute::(ctx.execution_ctx())?; let len = extension.len(); let PrimitiveDataParts { diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index c0dda0c1fd5..847fa39340d 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -500,8 +500,6 @@ mod tests { use cudarc::driver::PushKernelArg; use rstest::rstest; use vortex::array::IntoArray; - #[expect(deprecated)] - use vortex::array::ToCanonical; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::scalar::Scalar; @@ -542,8 +540,12 @@ mod tests { .map(|i| ((i as u64) % (max_val + 1)) as u32) .collect(); let primitive = PrimitiveArray::new(Buffer::from(values), NonNullable); - BitPacked::encode(&primitive.into_array(), bit_width) - .vortex_expect("failed to create BitPacked array") + BitPacked::encode( + &primitive.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("failed to create BitPacked array") } fn dispatch_plan( @@ -823,11 +825,11 @@ mod tests { expected.push(values[run]); } + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - let re = RunEnd::new(ends_arr, values_arr); + let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let plan = dispatch_plan(&re.into_array(), &cuda_ctx)?; let actual = @@ -851,12 +853,20 @@ mod tests { // BitPack+FoR the dict values let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); - let dict_bp = BitPacked::encode(&dict_prim.into_array(), 6)?; + let dict_bp = BitPacked::encode( + &dict_prim.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dict_for = FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference))?; // BitPack the codes let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPacked::encode(&codes_prim.into_array(), 6)?; + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?; @@ -872,6 +882,7 @@ mod tests { #[crate::test] fn test_alp_for_bitpacked() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // ALP(FoR(BitPacked)): encode each layer, then reassemble the tree // bottom-up because encode() methods produce flat outputs. let len = 3000; @@ -881,15 +892,14 @@ mod tests { .collect(); let float_prim = PrimitiveArray::new(Buffer::from(floats.clone()), NonNullable); - let alp = alp_encode( - float_prim.as_view(), - Some(exponents), + let alp = alp_encode(float_prim.as_view(), Some(exponents), &mut ctx)?; + assert!(alp.patches().is_none()); + let for_arr = FoR::encode(alp.encoded().clone().execute::(&mut ctx)?)?; + let bp = BitPacked::encode( + for_arr.encoded(), + 6, &mut LEGACY_SESSION.create_execution_ctx(), )?; - assert!(alp.patches().is_none()); - #[expect(deprecated)] - let for_arr = FoR::encode(alp.encoded().to_primitive())?; - let bp = BitPacked::encode(for_arr.encoded(), 6)?; let tree = ALP::new( FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone())?.into_array(), @@ -923,7 +933,11 @@ mod tests { .collect(); let prim = PrimitiveArray::new(Buffer::from(raw), NonNullable); - let bp = BitPacked::encode(&prim.into_array(), bit_width)?; + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let zz = ZigZag::try_new(bp.into_array())?; let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; @@ -950,12 +964,12 @@ mod tests { expected.push(values[run] + reference); } + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - let re = RunEnd::new(ends_arr, values_arr); + let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); let for_arr = FoR::try_new(re.into_array(), Scalar::from(reference))?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let plan = dispatch_plan(&for_arr.into_array(), &cuda_ctx)?; let actual = @@ -1006,7 +1020,11 @@ mod tests { // BitPack codes, then wrap in FoR (reference=0 so values unchanged) let bit_width: u8 = 3; let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPacked::encode(&codes_prim.into_array(), bit_width)?; + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let codes_for = FoR::try_new(codes_bp.into_array(), Scalar::from(0u32))?; let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); @@ -1032,7 +1050,11 @@ mod tests { let bit_width: u8 = 2; let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPacked::encode(&codes_prim.into_array(), bit_width)?; + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; @@ -1117,9 +1139,10 @@ mod tests { let values: Vec = vec![10, 20, 30]; let len = 3000; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - let re = RunEnd::new(ends_arr, values_arr); + let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); let array = re.into_array(); // Ends (u64) are wider than values (u32), so the kernel would truncate @@ -1131,7 +1154,6 @@ mod tests { ); // Execute through the non-fused dispatch path. - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1226,7 +1248,11 @@ mod tests { .collect(); let prim = PrimitiveArray::new(Buffer::from(raw), NonNullable); - let bp = BitPacked::encode(&prim.into_array(), bit_width)?; + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let zz = ZigZag::try_new(bp.into_array())?; let sliced = zz.into_array().slice(slice_start..slice_end)?; @@ -1322,7 +1348,11 @@ mod tests { let data: Vec = (0..len).map(|i| (i as u32) % max_val).collect(); let prim = PrimitiveArray::new(Buffer::from(data.clone()), NonNullable); - let bp = BitPacked::encode(&prim.into_array(), bit_width)?; + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let sliced = bp.into_array().slice(slice_start..slice_end)?; let expected: Vec = data[slice_start..slice_end].to_vec(); @@ -1368,7 +1398,11 @@ mod tests { let encoded_data: Vec = (0..len).map(|i| (i as u32) % max_val).collect(); let prim = PrimitiveArray::new(Buffer::from(encoded_data.clone()), NonNullable); - let bp = BitPacked::encode(&prim.into_array(), bit_width)?; + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let for_arr = FoR::try_new(bp.into_array(), Scalar::from(reference))?; let all_decoded: Vec = encoded_data.iter().map(|&v| v + reference).collect(); @@ -1421,12 +1455,20 @@ mod tests { // BitPack+FoR the dict values let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); - let dict_bp = BitPacked::encode(&dict_prim.into_array(), 6)?; + let dict_bp = BitPacked::encode( + &dict_prim.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dict_for = FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference))?; // BitPack the codes let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); - let codes_bp = BitPacked::encode(&codes_prim.into_array(), 6)?; + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?; @@ -1526,7 +1568,12 @@ mod tests { .collect(); let primitive = PrimitiveArray::new(Buffer::from(residuals), NonNullable); - let bp = BitPacked::encode(&primitive.into_array(), bit_width).vortex_expect("bitpack u8"); + let bp = BitPacked::encode( + &primitive.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack u8"); let for_arr = FoR::try_new( bp.into_array(), Scalar::primitive(reference, Nullability::NonNullable), @@ -1557,7 +1604,12 @@ mod tests { .collect(); let primitive = PrimitiveArray::new(Buffer::from(residuals), NonNullable); - let bp = BitPacked::encode(&primitive.into_array(), bit_width).vortex_expect("bitpack u16"); + let bp = BitPacked::encode( + &primitive.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack u16"); let for_arr = FoR::try_new( bp.into_array(), Scalar::primitive(reference, Nullability::NonNullable), @@ -1586,7 +1638,12 @@ mod tests { .collect(); let primitive = PrimitiveArray::new(Buffer::from(residuals), NonNullable); - let bp = BitPacked::encode(&primitive.into_array(), bit_width).vortex_expect("bitpack u64"); + let bp = BitPacked::encode( + &primitive.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack u64"); let for_arr = FoR::try_new( bp.into_array(), Scalar::primitive(reference, Nullability::NonNullable), @@ -1617,7 +1674,12 @@ mod tests { async fn test_single_element() -> VortexResult<()> { let values: Vec = vec![42]; let primitive = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); - let bp = BitPacked::encode(&primitive.into_array(), 6).vortex_expect("bitpack"); + let bp = BitPacked::encode( + &primitive.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack"); let for_arr = FoR::try_new( bp.into_array(), Scalar::primitive(0u32, Nullability::NonNullable), @@ -1646,7 +1708,12 @@ mod tests { let expected: Vec = residuals.iter().map(|&r| r + reference).collect(); let primitive = PrimitiveArray::new(Buffer::from(residuals), NonNullable); - let bp = BitPacked::encode(&primitive.into_array(), bit_width).vortex_expect("bitpack"); + let bp = BitPacked::encode( + &primitive.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack"); let for_arr = FoR::try_new( bp.into_array(), Scalar::primitive(reference, Nullability::NonNullable), @@ -1683,9 +1750,10 @@ mod tests { let values: Vec = vec![100, 200, 300, 400]; let len = 2000; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); - let re = RunEnd::new(ends_arr, values_arr); + let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); let array = re.into_array(); // Ends (u32) are wider than values (u16), so the kernel would truncate @@ -1696,7 +1764,6 @@ mod tests { "expected Unfused for RunEnd with wider ends" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1729,8 +1796,12 @@ mod tests { let codes_prim = PrimitiveArray::new(Buffer::from(codes.clone()), NonNullable); // BitPack the u8 codes at 2 bits (4 values need 2 bits) - let codes_bp = - BitPacked::encode(&codes_prim.into_array(), 2).vortex_expect("bitpack codes"); + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + 2, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack codes"); let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; let array = dict.into_array(); @@ -1902,7 +1973,11 @@ mod tests { PrimitiveArray::from_option_iter(values.iter().map(|v| v.map(|x| x - reference))); // BitPacked::encode preserves nullable validity from the input. - let bp = BitPacked::encode(&residuals.into_array(), 6)?; + let bp = BitPacked::encode( + &residuals.into_array(), + 6, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let for_arr = FoR::try_new(bp.into_array(), reference.into())?; // Verify the plan actually fuses (not just a LOAD). diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 41e085d00b3..4714aeabcb0 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -124,6 +124,8 @@ mod tests { use vortex::error::VortexResult; use vortex::mask::Mask; use vortex::session::VortexSession; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use crate::CanonicalCudaExt; use crate::executor::CudaArrayExt; @@ -138,6 +140,7 @@ mod tests { let bp = BitPacked::encode( &PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(), 7, + &mut LEGACY_SESSION.create_execution_ctx(), ) .vortex_expect("bp"); let arr = FoR::try_new(bp.into_array(), 1000u32.into()).vortex_expect("for"); @@ -167,6 +170,7 @@ mod tests { let bp = BitPacked::encode( &PrimitiveArray::new(Buffer::from(encoded), NonNullable).into_array(), 9, + &mut LEGACY_SESSION.create_execution_ctx(), ) .vortex_expect("bp"); let alp = ALP::try_new( @@ -253,7 +257,9 @@ mod tests { ) .into_array(); let vals = FoR::try_new( - BitPacked::encode(&vals, 6).vortex_expect("bp").into_array(), + BitPacked::encode(&vals, 6, &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("bp") + .into_array(), 0u32.into(), ) .vortex_expect("for"); @@ -266,7 +272,7 @@ mod tests { ) .into_array(); let codes = FoR::try_new( - BitPacked::encode(&codes, 6) + BitPacked::encode(&codes, 6, &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("bp") .into_array(), 0u32.into(), @@ -302,6 +308,7 @@ mod tests { let bp = BitPacked::encode( &PrimitiveArray::new(Buffer::from(data.clone()), NonNullable).into_array(), 7, + &mut LEGACY_SESSION.create_execution_ctx(), ) .vortex_expect("bp"); let for_arr = FoR::try_new(bp.into_array(), 100u32.into()).vortex_expect("for"); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index d29f6115889..1853557e809 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -207,6 +207,8 @@ mod tests { use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::error::VortexExpect; use vortex::session::VortexSession; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use super::*; use crate::CanonicalCudaExt; @@ -228,7 +230,11 @@ mod tests { let array = PrimitiveArray::new(iter.collect::>(), NonNullable); // Last two items should be patched - let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; + let bp_with_patches = BitPacked::encode( + &array.into_array(), + bw, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!(bp_with_patches.patches().is_some()); let cpu_result = crate::canonicalize_cpu(bp_with_patches.clone())?.into_array(); @@ -259,7 +265,11 @@ mod tests { ); // Last two items should be patched - let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; + let bp_with_patches = BitPacked::encode( + &array.into_array(), + 9, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!(bp_with_patches.patches().is_some()); let cpu_result = crate::canonicalize_cpu(bp_with_patches.clone())?.into_array(); @@ -301,8 +311,12 @@ mod tests { NonNullable, ); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) - .vortex_expect("operation should succeed in test"); + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("operation should succeed in test"); let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { @@ -350,8 +364,12 @@ mod tests { NonNullable, ); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) - .vortex_expect("operation should succeed in test"); + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("operation should succeed in test"); let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { @@ -415,8 +433,12 @@ mod tests { NonNullable, ); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) - .vortex_expect("operation should succeed in test"); + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("operation should succeed in test"); let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { @@ -512,8 +534,12 @@ mod tests { NonNullable, ); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) - .vortex_expect("operation should succeed in test"); + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("operation should succeed in test"); let cpu_result = crate::canonicalize_cpu(bitpacked_array.clone())?; let gpu_result = block_on(async { BitPackedExecutor @@ -545,8 +571,12 @@ mod tests { NonNullable, ); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), bit_width) - .vortex_expect("operation should succeed in test"); + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("operation should succeed in test"); let sliced_array = bitpacked_array.into_array().slice(67..3969)?; assert!(sliced_array.is::()); let cpu_result = crate::canonicalize_cpu(sliced_array.clone())?; @@ -579,7 +609,11 @@ mod tests { let primitive_array = PrimitiveArray::new(buffer![100u8, 101, 102, 3, 4, 5], NonNullable); // Encode with bit width 4. First 3 elements patched, remainder will pack. - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 4)?; + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + 4, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!( bitpacked_array.patches().is_some(), "Expected patches to be present" @@ -625,7 +659,11 @@ mod tests { let primitive_array = PrimitiveArray::new(Buffer::from_iter(values.iter().copied()), NonNullable); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 9)?; + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + 9, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!( bitpacked_array.patches().is_some(), "Expected patches to be present" @@ -678,7 +716,11 @@ mod tests { let primitive_array = PrimitiveArray::new(Buffer::from_iter(values.iter().copied()), NonNullable); - let bitpacked_array = BitPacked::encode(&primitive_array.into_array(), 9)?; + let bitpacked_array = BitPacked::encode( + &primitive_array.into_array(), + 9, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!( bitpacked_array.patches().is_some(), "Expected patches to be present" diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 0c33ee08ca4..cdca6eeab7d 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -139,6 +139,8 @@ mod tests { use vortex::error::VortexExpect; use vortex::scalar::Scalar; use vortex::session::VortexSession; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use super::*; use crate::CanonicalCudaExt; @@ -187,7 +189,9 @@ mod tests { .take(1024) .collect::>() .into_array(); - let packed = BitPacked::encode(&values, 3).unwrap().into_array(); + let packed = BitPacked::encode(&values, 3, &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); let for_array = FoR::try_new(packed, (-8i8).into()).unwrap(); let cpu_result = crate::canonicalize_cpu(for_array.clone()).unwrap(); diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 1fa09702999..fca435478d8 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -181,7 +181,11 @@ mod tests { use crate::executor::CudaArrayExt; use crate::session::CudaSession; - fn make_runend_array(ends: Vec, values: Vec) -> RunEndArray + fn make_runend_array( + ends: Vec, + values: Vec, + ctx: &mut vortex::array::ExecutionCtx, + ) -> RunEndArray where V: NativePType, E: NativePType, @@ -190,21 +194,24 @@ mod tests { PrimitiveArray::new(Buffer::from(ends), Validity::NonNullable).into_array(); let values_array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable).into_array(); - RunEnd::new(ends_array, values_array) + RunEnd::new(ends_array, values_array, ctx) } + type RunEndBuilder = fn(&mut vortex::array::ExecutionCtx) -> RunEndArray; + #[rstest] - #[case::u32_ends_u8_values(make_runend_array(vec![3u32, 6, 10], vec![10u8, 20, 30]))] - #[case::u32_ends_u32_values(make_runend_array(vec![2u32, 5, 10], vec![1u32, 2, 3]))] - #[case::u32_ends_f64_values(make_runend_array(vec![2u32, 5, 8], vec![1.5f64, 2.5, 3.5]))] - #[case::u8_ends_i32_values(make_runend_array(vec![2u8, 5, 10], vec![1i32, 2, 3]))] - #[case::u32_ends_i32_values(make_runend_array(vec![2u32, 5, 10], vec![1i32, 2, 3]))] - #[case::u64_ends_i32_values(make_runend_array(vec![2u64, 5, 10], vec![1i32, 2, 3]))] + #[case::u32_ends_u8_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![3u32, 6, 10], vec![10u8, 20, 30], ctx))] + #[case::u32_ends_u32_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u32, 5, 10], vec![1u32, 2, 3], ctx))] + #[case::u32_ends_f64_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u32, 5, 8], vec![1.5f64, 2.5, 3.5], ctx))] + #[case::u8_ends_i32_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u8, 5, 10], vec![1i32, 2, 3], ctx))] + #[case::u32_ends_i32_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u32, 5, 10], vec![1i32, 2, 3], ctx))] + #[case::u64_ends_i32_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u64, 5, 10], vec![1i32, 2, 3], ctx))] #[crate::test] - async fn test_cuda_runend_types(#[case] runend_array: RunEndArray) -> VortexResult<()> { + async fn test_cuda_runend_types(#[case] build: RunEndBuilder) -> VortexResult<()> { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); + let runend_array = build(cuda_ctx.execution_ctx()); let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; let gpu_result = RunEndExecutor @@ -232,7 +239,7 @@ mod tests { let ends: Vec = (1..=num_runs).map(|i| (i * run_length) as u64).collect(); let values: Vec = (0..num_runs).map(|i| i32::try_from(i).unwrap()).collect(); - let runend_array = make_runend_array(ends, values); + let runend_array = make_runend_array(ends, values, cuda_ctx.execution_ctx()); assert_eq!(runend_array.len(), total_len); let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; @@ -255,7 +262,7 @@ mod tests { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); - let runend_array = make_runend_array(vec![100u32], vec![42i32]); + let runend_array = make_runend_array(vec![100u32], vec![42i32], cuda_ctx.execution_ctx()); let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; @@ -282,7 +289,7 @@ mod tests { let ends: Vec = (1..=num_elements).collect(); let values: Vec = (0..num_elements as i32).collect(); - let runend_array = make_runend_array(ends, values); + let runend_array = make_runend_array(ends, values, cuda_ctx.execution_ctx()); let cpu_result = crate::canonicalize_cpu(runend_array.clone())?; @@ -312,7 +319,7 @@ mod tests { Validity::Array(BoolArray::from_iter([true, false, true].into_iter()).into_array()); let values_array = PrimitiveArray::new(Buffer::from(vec![10i32, 0, 30]), validity).into_array(); - let runend_array = RunEnd::new(ends_array, values_array); + let runend_array = RunEnd::new(ends_array, values_array, cuda_ctx.execution_ctx()); let cpu_result = crate::canonicalize_cpu(runend_array.clone())?.into_array(); diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index 8b3367af6e0..fe46b308e04 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -377,7 +377,7 @@ mod tests { "baz", ]); - let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; + let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0, cuda_ctx.execution_ctx())?; let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())? .execute::(cuda_ctx.execution_ctx())?; @@ -413,7 +413,7 @@ mod tests { // Compress with ZSTD using values_per_frame=3 to create multiple frames. // 14 strings and 3 values per frame = ceil(14/3) = 5 frames. - let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3)?; + let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3, cuda_ctx.execution_ctx())?; let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())? .execute::(cuda_ctx.execution_ctx())?; @@ -443,7 +443,7 @@ mod tests { "final test string", ]); - let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; + let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0, cuda_ctx.execution_ctx())?; // Slice the array to get a subset (indices 2..7) let sliced_zstd = zstd_array.slice(2..7)?; @@ -474,7 +474,7 @@ mod tests { Some("another string"), ]); - let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; + let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0, cuda_ctx.execution_ctx())?; let cpu_result = crate::canonicalize_cpu(zstd_array.clone())?.into_array(); diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index e8fafa3087f..07cd51c3f01 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -101,10 +101,9 @@ mod tests { use std::sync::Arc; use cudarc::driver::DeviceRepr; + use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::primitive::PrimitiveDataParts; @@ -146,32 +145,30 @@ mod tests { } async fn full_test_case() { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter(0..128); - let values = force_cast::(values); + let values = force_cast::(values, &mut ctx); let patch_idx = PrimitiveArray::new(buffer![0, 8, 16, 32], Validity::NonNullable); - let patch_idx = force_cast::(patch_idx); + let patch_idx = force_cast::(patch_idx, &mut ctx); let patch_val = PrimitiveArray::new(buffer![99, 99, 99, 99], Validity::NonNullable); - let patch_val = force_cast::(patch_val); + let patch_val = force_cast::(patch_val, &mut ctx); // Copy all to GPU let patches = Patches::new(128, 0, patch_idx.into_array(), patch_val.into_array(), None).unwrap(); - let cpu_result = values - .clone() - .patch(&patches, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(); + let cpu_result = values.clone().patch(&patches, &mut ctx).unwrap(); let PrimitiveDataParts { buffer: cuda_buffer, .. } = values.into_data_parts(); - let handle = ctx.ensure_on_device(cuda_buffer).await.unwrap(); + let handle = cuda_ctx.ensure_on_device(cuda_buffer).await.unwrap(); let device_buf = handle .as_device() .as_any() @@ -179,7 +176,7 @@ mod tests { .unwrap() .clone(); - let patched_buf = execute_patches::(patches, device_buf, &mut ctx) + let patched_buf = execute_patches::(patches, device_buf, &mut cuda_ctx) .await .unwrap(); @@ -189,7 +186,7 @@ mod tests { Validity::NonNullable, ) .into_array() - .execute::(&mut LEGACY_SESSION.create_execution_ctx()) + .execute::(&mut ctx) .unwrap() .into_host() .await @@ -199,13 +196,12 @@ mod tests { assert_arrays_eq!(cpu_result, gpu_result); } - fn force_cast(array: PrimitiveArray) -> PrimitiveArray { - #[expect(deprecated)] - let result = array + fn force_cast(array: PrimitiveArray, ctx: &mut ExecutionCtx) -> PrimitiveArray { + array .into_array() .cast(DType::Primitive(T::PTYPE, Nullability::NonNullable)) .unwrap() - .to_primitive(); - result + .execute::(ctx) + .unwrap() } } diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index 70c6b60128d..7f1b59753b9 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -376,8 +376,6 @@ mod tests { use std::ffi::CString; use vortex::array::LEGACY_SESSION; - #[expect(deprecated)] - use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -415,6 +413,7 @@ mod tests { #[test] fn test_timestamp_vector_conversion() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1_703_980_800_000_000_i64, 0i64, -86_400_000_000_i64]; // microseconds let len = values.len(); @@ -430,8 +429,11 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -439,6 +441,7 @@ mod tests { #[test] fn test_timestamp_seconds_vector_conversion() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1_703_980_800_i64, 0i64, -86_400_i64]; // seconds let len = values.len(); @@ -454,8 +457,11 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -463,6 +469,7 @@ mod tests { #[test] fn test_timestamp_milliseconds_vector_conversion() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1_703_980_800_000_i64, 0i64, -86_400_000_i64]; // milliseconds let len = values.len(); @@ -478,8 +485,11 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -487,6 +497,7 @@ mod tests { #[test] fn test_timestamp_with_nulls_conversion() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1_703_980_800_000_000_i64, 0i64, -86_400_000_000_i64]; let len = values.len(); @@ -507,8 +518,11 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -517,10 +531,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( - vortex_values.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(vortex_values.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); @@ -550,8 +561,12 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -559,6 +574,7 @@ mod tests { #[test] fn test_timestamp_single_value() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1_703_980_800_000_000_i64]; // Single microsecond timestamp let len = values.len(); @@ -574,8 +590,11 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); let vortex_array = TemporalArray::try_from(result).unwrap(); - #[expect(deprecated)] - let vortex_values = vortex_array.temporal_values().to_primitive(); + let vortex_values = vortex_array + .temporal_values() + .clone() + .execute::(&mut ctx) + .unwrap(); let values_slice = vortex_values.as_slice::(); assert_eq!(values_slice, values); @@ -583,6 +602,7 @@ mod tests { #[test] fn test_boolean_vector_conversion() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![true, false, true, false]; let len = values.len(); @@ -597,14 +617,14 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_bool(); + let vortex_array = result.execute::(&mut ctx).unwrap(); let expected = BoolArray::new(BitBuffer::from(values), Validity::AllValid); assert_arrays_eq!(vortex_array, expected); } #[test] fn test_vector_with_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1i32, 2, 3]; let len = values.len(); @@ -624,8 +644,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_primitive(); + let vortex_array = result.execute::(&mut ctx).unwrap(); let vortex_slice = vortex_array.as_slice::(); assert_eq!(vortex_slice, values); @@ -634,10 +653,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( - vortex_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); @@ -667,8 +683,8 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_listview(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_arrays_eq!( @@ -679,6 +695,7 @@ mod tests { #[test] fn test_fixed_sized_list() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = vec![1i32, 2, 3, 4]; let len = 1; @@ -696,8 +713,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_fixed_size_list(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_arrays_eq!( @@ -708,6 +724,7 @@ mod tests { #[test] fn test_empty_struct() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let len = 4; let logical_type = LogicalType::struct_type([], []) .vortex_expect("LogicalTypeRef creation should succeed for test data"); @@ -715,8 +732,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_struct(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_eq!(vortex_array.struct_fields().nfields(), 0); @@ -724,6 +740,7 @@ mod tests { #[test] fn test_struct() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values1 = vec![1i32, 2, 3, 4]; let values2 = vec![5i32, 6, 7, 8]; let len = values1.len(); @@ -751,8 +768,7 @@ mod tests { // Test conversion let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_struct(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_eq!(vortex_array.struct_fields().nfields(), 2); @@ -768,6 +784,7 @@ mod tests { #[test] fn test_list_with_trailing_null() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Regression test: when the last list entry is null, its offset/length may be 0/0, // so we can't use the last entry to compute child vector length. let child_values = vec![1i32, 2, 3, 4]; @@ -801,8 +818,7 @@ mod tests { // Test conversion - the old bug would compute child length as 0+0=0 instead of // max(4,0)=4. let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_listview(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_arrays_eq!( @@ -814,10 +830,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( - vortex_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(2, vec![0]) ); @@ -825,6 +838,7 @@ mod tests { #[test] fn test_list_out_of_order() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Regression test: list views can be out of order in DuckDB. The child vector length // must be computed as the maximum end offset, not just the last entry's end offset. let child_values = vec![1i32, 2, 3, 4]; @@ -855,8 +869,7 @@ mod tests { // Test conversion - the old bug would compute child length as 0+2=2 instead of // max(4,2)=4. let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_listview(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); assert_arrays_eq!( @@ -871,6 +884,7 @@ mod tests { #[test] fn test_list_null_garbage_data() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Test that null list entries with garbage offset/size values don't cause issues. // DuckDB doesn't guarantee valid offset/size for null list views, so we must check // validity before reading the offset/size values. @@ -911,8 +925,7 @@ mod tests { // Test conversion. The old code would compute child_min_length as 9999+9999=19998, which // would panic when trying to read that much data from the child vector. let result = flat_vector_to_vortex(&vector, len).unwrap(); - #[expect(deprecated)] - let vortex_array = result.to_listview(); + let vortex_array = result.execute::(&mut ctx).unwrap(); assert_eq!(vortex_array.len(), len); @@ -927,10 +940,16 @@ mod tests { ); // Verify the null entry has sanitized offset/size (offset=2, size=0) rather than garbage. - #[expect(deprecated)] - let offsets = vortex_array.offsets().to_primitive(); - #[expect(deprecated)] - let sizes = vortex_array.sizes().to_primitive(); + let offsets = vortex_array + .offsets() + .clone() + .execute::(&mut ctx) + .unwrap(); + let sizes = vortex_array + .sizes() + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(offsets.as_slice::()[1], 2); // Previous end (0+2). assert_eq!(sizes.as_slice::()[1], 0); @@ -939,10 +958,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( - vortex_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx() - ) + .to_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); diff --git a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs index ed057eb709a..8e65d26ed6f 100644 --- a/vortex-duckdb/src/e2e_test/vortex_scan_test.rs +++ b/vortex-duckdb/src/e2e_test/vortex_scan_test.rs @@ -19,6 +19,8 @@ use jiff::tz::TimeZone; use num_traits::AsPrimitive; use tempfile::NamedTempFile; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::DictArray; @@ -823,7 +825,9 @@ async fn write_vortex_file_with_encodings() -> NamedTempFile { // 4. Run-End let run_ends = buffer![3u32, 5]; let run_values = buffer![100i32, 200]; - let rle_array = RunEnd::try_new(run_ends.into_array(), run_values.into_array()).unwrap(); + let mut rle_ctx = LEGACY_SESSION.create_execution_ctx(); + let rle_array = + RunEnd::try_new(run_ends.into_array(), run_values.into_array(), &mut rle_ctx).unwrap(); // 5. Sequence array let sequence_array = Sequence::try_new( diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index ff30dddc6c1..55234fc502d 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -77,8 +77,7 @@ pub(crate) fn new_exporter_with_flatten( let canonical = match canonical { Some(c) => c, None => { - #[expect(deprecated)] - let canonical = values.to_canonical()?; + let canonical = values.clone().execute::(ctx)?; cache .canonical_cache .insert(values_key, (values.clone(), canonical.clone())); diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 2b02bb6fcbc..bb4e464dea4 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -11,11 +11,10 @@ use paste::paste; use vortex::array::ArrayRef; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::NullArray; use vortex::array::arrays::PrimitiveArray; +use vortex::array::arrays::StructArray; use vortex::array::arrays::struct_::StructArrayExt; use vortex::array::validity::Validity; use vortex::buffer::Buffer; @@ -208,9 +207,10 @@ pub unsafe extern "C-unwind" fn vx_array_get_field( try_or_default(error_out, || { let array = vx_array::as_ref(array); - #[expect(deprecated)] + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let field_array = array - .to_struct() + .clone() + .execute::(&mut ctx)? .unmasked_fields() .get(index) .ok_or_else(|| vortex_err!("Field index out of bounds"))? @@ -435,6 +435,8 @@ mod tests { use std::ptr; use vortex::array::IntoArray; + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -764,8 +766,8 @@ mod tests { assert!(!res.is_null()); { let res = vx_array::as_ref(res); - #[expect(deprecated)] - let bool_array = res.to_bool(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let bool_array = res.clone().execute::(&mut ctx).unwrap(); let buffer = bool_array.to_bit_buffer(); let expected = BoolArray::from_iter(vec![false, false, true, true]); assert_eq!(buffer, expected.to_bit_buffer()); diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 442f9325959..69004415123 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -294,8 +294,8 @@ mod tests { use std::sync::Arc; use vortex::array::IntoArray; - #[expect(deprecated)] - use vortex::array::ToCanonical; + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; @@ -347,6 +347,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_get_item() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let (array, names_array, ages_array) = struct_array(); unsafe { let root = vx_expression_root(); @@ -362,8 +363,10 @@ mod tests { { let applied_array = vx_array::as_ref(applied_array); let expected: Buffer = ages_array.to_buffer(); - #[expect(deprecated)] - let prim = applied_array.to_primitive(); + let prim = applied_array + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(prim.to_buffer(), expected); } vx_array_free(applied_array); @@ -433,6 +436,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_and_or() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let col1 = BoolArray::from_iter([true, false, true, true]); let col2 = BoolArray::from_iter([false, true, true, false]); let col3 = BoolArray::from_iter([false, true, true, true]); @@ -468,8 +472,10 @@ mod tests { assert!(error.is_null()); assert!(!applied_array.is_null()); { - #[expect(deprecated)] - let array = vx_array::as_ref(applied_array).to_bool(); + let array = vx_array::as_ref(applied_array) + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = BoolArray::from_iter([false, false, true, false]); assert_eq!(array.to_bit_buffer(), expected.to_bit_buffer()); } @@ -482,8 +488,10 @@ mod tests { assert!(error.is_null()); assert!(!applied_array.is_null()); { - #[expect(deprecated)] - let array = vx_array::as_ref(applied_array).to_bool(); + let array = vx_array::as_ref(applied_array) + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = BoolArray::from_iter([true, true, true, false]); assert_eq!(array.to_bit_buffer(), expected.to_bit_buffer()); } @@ -520,6 +528,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_list_contains() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let elements = buffer![1i32, 2, 3, 4, 5].into_array(); let offsets = buffer![0u32, 2, 5, 5].into_array(); let array = ListArray::try_new(elements, offsets, Validity::NonNullable).unwrap(); @@ -537,8 +546,10 @@ mod tests { assert!(error.is_null()); assert!(!applied.is_null()); { - #[expect(deprecated)] - let applied = vx_array::as_ref(applied).to_bool(); + let applied = vx_array::as_ref(applied) + .clone() + .execute::(&mut ctx) + .unwrap(); let expected = BoolArray::from_iter([true, false, false]); assert_eq!(applied.to_bit_buffer(), expected.to_bit_buffer()); } diff --git a/vortex-ffi/src/struct_array.rs b/vortex-ffi/src/struct_array.rs index 19506524b4e..f3023625a82 100644 --- a/vortex-ffi/src/struct_array.rs +++ b/vortex-ffi/src/struct_array.rs @@ -133,8 +133,8 @@ mod tests { use std::sync::Arc; use vortex::array::IntoArray; - #[expect(deprecated)] - use vortex::array::ToCanonical; + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; @@ -171,6 +171,7 @@ mod tests { #[test] #[cfg_attr(miri, ignore)] fn test_many() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let names = ["age", "name"]; let age_field = PrimitiveArray::new(buffer![30u8, 25u8, 35u8], Validity::NonNullable); let name_field = VarBinViewArray::from_iter_str(["Alice", "Bob", "Charlie"]); @@ -238,8 +239,10 @@ mod tests { assert!(!array.is_null()); { - #[expect(deprecated)] - let array = vx_array::as_ref(array).to_struct(); + let array = vx_array::as_ref(array) + .clone() + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(array, struct_array); } diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index e42d5671a00..2ba10d96684 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -12,8 +12,6 @@ use futures::TryStreamExt; use futures::pin_mut; use vortex_array::ArrayRef; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::ChunkedArray; @@ -264,6 +262,7 @@ async fn test_read_simple_with_spawn() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_read_projection() { + let mut ctx = SESSION.create_execution_ctx(); let strings_expected = ["ab", "foo", "bar", "baz", "ab", "foo", "bar", "baz"]; let strings = ChunkedArray::from_iter([ VarBinArray::from(strings_expected[..4].to_vec()).into_array(), @@ -308,8 +307,11 @@ async fn test_read_projection() { ) ); - #[expect(deprecated)] - let actual = array.to_struct().unmasked_field(0).clone(); + let actual = array + .execute::(&mut ctx) + .unwrap() + .unmasked_field(0) + .clone(); let expected = VarBinArray::from(strings_expected.to_vec()).into_array(); assert_arrays_eq!(actual, expected); @@ -331,8 +333,11 @@ async fn test_read_projection() { ) ); - #[expect(deprecated)] - let actual = array.to_struct().unmasked_field(0).clone(); + let actual = array + .execute::(&mut ctx) + .unwrap() + .unmasked_field(0) + .clone(); let expected = Buffer::copy_from(numbers_expected).into_array(); assert_arrays_eq!(actual, expected); } @@ -340,6 +345,7 @@ async fn test_read_projection() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn unequal_batches() { + let mut ctx = SESSION.create_execution_ctx(); let strings = ChunkedArray::from_iter([ VarBinArray::from(vec!["ab", "foo", "bar", "bob"]).into_array(), VarBinArray::from(vec!["baz", "ab", "foo", "bar", "baz", "alice"]).into_array(), @@ -376,12 +382,14 @@ async fn unequal_batches() { let array = array.unwrap(); item_count += array.len(); - #[expect(deprecated)] let numbers = array - .to_struct() + .execute::(&mut ctx) + .unwrap() .unmasked_field_by_name("numbers") .unwrap() - .to_primitive(); + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(numbers.ptype(), PType::U32); } assert_eq!(item_count, 10); @@ -510,6 +518,7 @@ async fn issue_5385_filter_casted_column() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn filter_string() { + let mut ctx = SESSION.create_execution_ctx(); let names_orig = VarBinArray::from_iter( vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None], DType::Utf8(Nullability::Nullable), @@ -546,15 +555,23 @@ async fn filter_string() { .unwrap(); assert_eq!(result.len(), 1); - #[expect(deprecated)] - let names_actual = result[0].to_struct().unmasked_field(0).clone(); + let names_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(0) + .clone(); let names_expected = VarBinArray::from_iter(vec![Some("Joseph")], DType::Utf8(Nullability::Nullable)) .into_array(); assert_arrays_eq!(names_actual, names_expected); - #[expect(deprecated)] - let ages_actual = result[0].to_struct().unmasked_field(1).clone(); + let ages_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(1) + .clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32)]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); } @@ -562,6 +579,7 @@ async fn filter_string() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn filter_or() { + let mut ctx = SESSION.create_execution_ctx(); let names = VarBinArray::from_iter( vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None], DType::Utf8(Nullability::Nullable), @@ -603,8 +621,12 @@ async fn filter_or() { .unwrap(); assert_eq!(result.len(), 1); - #[expect(deprecated)] - let names_actual = result[0].to_struct().unmasked_field(0).clone(); + let names_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(0) + .clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), Some("Angela")], DType::Utf8(Nullability::Nullable), @@ -612,8 +634,12 @@ async fn filter_or() { .into_array(); assert_arrays_eq!(names_actual, names_expected); - #[expect(deprecated)] - let ages_actual = result[0].to_struct().unmasked_field(1).clone(); + let ages_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(1) + .clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), None]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); } @@ -621,6 +647,7 @@ async fn filter_or() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn filter_and() { + let mut ctx = SESSION.create_execution_ctx(); let names = VarBinArray::from_iter( vec![Some("Joseph"), None, Some("Angela"), Some("Mikhail"), None], DType::Utf8(Nullability::Nullable), @@ -659,8 +686,12 @@ async fn filter_and() { .unwrap(); assert_eq!(result.len(), 1); - #[expect(deprecated)] - let names_actual = result[0].to_struct().unmasked_field(0).clone(); + let names_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(0) + .clone(); let names_expected = VarBinArray::from_iter( vec![Some("Joseph"), None], DType::Utf8(Nullability::Nullable), @@ -668,8 +699,12 @@ async fn filter_and() { .into_array(); assert_arrays_eq!(names_actual, names_expected); - #[expect(deprecated)] - let ages_actual = result[0].to_struct().unmasked_field(1).clone(); + let ages_actual = result[0] + .clone() + .execute::(&mut ctx) + .unwrap() + .unmasked_field(1) + .clone(); let ages_expected = PrimitiveArray::from_option_iter([Some(25i32), Some(31i32)]).into_array(); assert_arrays_eq!(ages_actual, ages_expected); } @@ -677,6 +712,7 @@ async fn filter_and() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_with_indices_simple() { + let mut ctx = SESSION.create_execution_ctx(); let expected_numbers_split: Vec> = (0..5).map(|_| (0_i16..100).collect()).collect(); let expected_array = StructArray::from_fields(&[( "numbers", @@ -701,7 +737,6 @@ async fn test_with_indices_simple() { let file = SESSION.open_options().open_buffer(buf).unwrap(); // test no indices - #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -711,14 +746,14 @@ async fn test_with_indices_simple() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_kept_array.len(), 0); // test a few indices let kept_indices = [0_u64, 3, 99, 100, 101, 399, 400, 401, 499]; - #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -728,9 +763,13 @@ async fn test_with_indices_simple() { .read_all() .await .unwrap() - .to_struct(); - #[expect(deprecated)] - let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); + .execute::(&mut ctx) + .unwrap(); + let actual_kept_numbers_array = actual_kept_array + .unmasked_field(0) + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_kept_numbers: Vec = kept_indices .iter() @@ -740,7 +779,6 @@ async fn test_with_indices_simple() { assert_arrays_eq!(actual_kept_numbers_array, expected_array); // test all indices - #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -750,7 +788,8 @@ async fn test_with_indices_simple() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); let actual_numbers_array = actual_array.unmasked_field(0).clone(); let expected_array = Buffer::copy_from(&expected_numbers).into_array(); assert_arrays_eq!(actual_numbers_array, expected_array); @@ -759,6 +798,7 @@ async fn test_with_indices_simple() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_with_indices_on_two_columns() { + let mut ctx = SESSION.create_execution_ctx(); let strings_expected = ["ab", "foo", "bar", "baz", "ab", "foo", "bar", "baz"]; let strings = ChunkedArray::from_iter([ VarBinArray::from(strings_expected[..4].to_vec()).into_array(), @@ -784,7 +824,6 @@ async fn test_with_indices_on_two_columns() { let file = SESSION.open_options().open_buffer(buf).unwrap(); let kept_indices = [0_u64, 3, 7]; - #[expect(deprecated)] let array = file .scan() .unwrap() @@ -794,7 +833,8 @@ async fn test_with_indices_on_two_columns() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); let strings_actual = array.unmasked_field(0).clone(); let strings_expected_vec: Vec<&str> = kept_indices @@ -816,6 +856,7 @@ async fn test_with_indices_on_two_columns() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_with_indices_and_with_row_filter_simple() { + let mut ctx = SESSION.create_execution_ctx(); let expected_numbers_split: Vec> = (0..5).map(|_| (0_i16..100).collect()).collect(); let expected_array = StructArray::from_fields(&[( "numbers", @@ -839,7 +880,6 @@ async fn test_with_indices_and_with_row_filter_simple() { let file = SESSION.open_options().open_buffer(buf).unwrap(); - #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -850,14 +890,14 @@ async fn test_with_indices_and_with_row_filter_simple() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_kept_array.len(), 0); // test a few indices let kept_indices = [0u64, 3, 99, 100, 101, 399, 400, 401, 499]; - #[expect(deprecated)] let actual_kept_array = file .scan() .unwrap() @@ -868,10 +908,14 @@ async fn test_with_indices_and_with_row_filter_simple() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); - #[expect(deprecated)] - let actual_kept_numbers_array = actual_kept_array.unmasked_field(0).to_primitive(); + let actual_kept_numbers_array = actual_kept_array + .unmasked_field(0) + .clone() + .execute::(&mut ctx) + .unwrap(); let expected_kept_numbers: Buffer = kept_indices .iter() @@ -882,7 +926,6 @@ async fn test_with_indices_and_with_row_filter_simple() { assert_arrays_eq!(actual_kept_numbers_array, expected_array); // test all indices - #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -893,7 +936,8 @@ async fn test_with_indices_and_with_row_filter_simple() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); let actual_numbers_array = actual_array.unmasked_field(0).clone(); let expected_filtered: Buffer = expected_numbers @@ -908,6 +952,7 @@ async fn test_with_indices_and_with_row_filter_simple() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn filter_string_chunked() { + let mut ctx = SESSION.create_execution_ctx(); let name_chunk1 = VarBinViewArray::from_iter_nullable_str([Some("Joseph"), Some("James"), Some("Angela")]) .into_array(); @@ -943,7 +988,6 @@ async fn filter_string_chunked() { let file = SESSION.open_options().open_buffer(buf).unwrap(); - #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -953,7 +997,8 @@ async fn filter_string_chunked() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.len(), 1); let names_actual = actual_array.unmasked_field(0).clone(); @@ -970,6 +1015,7 @@ async fn filter_string_chunked() { #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_pruning_with_or() { + let mut ctx = SESSION.create_execution_ctx(); let letter_chunk1 = VarBinViewArray::from_iter_nullable_str([ Some("A".to_owned()), Some("B".to_owned()), @@ -1032,7 +1078,6 @@ async fn test_pruning_with_or() { let file = SESSION.open_options().open_buffer(buf).unwrap(); - #[expect(deprecated)] let actual_array = file .scan() .unwrap() @@ -1045,7 +1090,8 @@ async fn test_pruning_with_or() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); assert_eq!(actual_array.len(), 10); let letters_actual = actual_array.unmasked_field(0).clone(); @@ -1083,6 +1129,7 @@ async fn test_pruning_with_or() { #[tokio::test] async fn test_repeated_projection() { + let mut ctx = SESSION.create_execution_ctx(); let strings = ChunkedArray::from_iter([ VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), VarBinArray::from(vec!["ab", "foo", "bar", "baz"]).into_array(), @@ -1106,7 +1153,6 @@ async fn test_repeated_projection() { let file = SESSION.open_options().open_buffer(buf).unwrap(); - #[expect(deprecated)] let actual = file .scan() .unwrap() @@ -1116,7 +1162,8 @@ async fn test_repeated_projection() { .read_all() .await .unwrap() - .to_struct(); + .execute::(&mut ctx) + .unwrap(); assert_arrays_eq!(actual, expected); } @@ -1263,16 +1310,19 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { )? .into_array(); - #[expect(deprecated)] - let result = round_trip(&array, Ok).await?.to_struct(); + let mut ctx = SESSION.create_execution_ctx(); + let result = round_trip(&array, Ok) + .await? + .execute::(&mut ctx)?; assert_eq!(result.len(), 3); assert_eq!(result.struct_fields().nfields(), 1); - let mut ctx = SESSION.create_execution_ctx(); assert!(result.all_valid(&mut ctx)?); - #[expect(deprecated)] - let nested_struct = result.unmasked_field_by_name("struct")?.to_struct(); + let nested_struct = result + .unmasked_field_by_name("struct")? + .clone() + .execute::(&mut ctx)?; assert_eq!(nested_struct.dtype(), &nested_dtype); assert_eq!(nested_struct.len(), 3); assert!(nested_struct.all_invalid(&mut ctx)?); @@ -1382,6 +1432,7 @@ async fn test_writer_basic_push() -> VortexResult<()> { #[tokio::test] async fn test_writer_multiple_pushes() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let chunk1 = StructArray::from_fields(&[("numbers", buffer![1u32, 2, 3].into_array())])?.into_array(); let chunk2 = @@ -1405,9 +1456,8 @@ async fn test_writer_multiple_pushes() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 9); - #[expect(deprecated)] let numbers = result - .to_struct() + .execute::(&mut ctx)? .unmasked_field_by_name("numbers")? .clone(); let expected = buffer![1u32, 2, 3, 4, 5, 6, 7, 8, 9].into_array(); @@ -1418,6 +1468,7 @@ async fn test_writer_multiple_pushes() -> VortexResult<()> { #[tokio::test] async fn test_writer_push_stream() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let chunk1 = StructArray::from_fields(&[("numbers", buffer![1u32, 2, 3].into_array())])?.into_array(); let chunk2 = @@ -1440,9 +1491,8 @@ async fn test_writer_push_stream() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 6); - #[expect(deprecated)] let numbers = result - .to_struct() + .execute::(&mut ctx)? .unmasked_field_by_name("numbers")? .clone(); let expected = buffer![1u32, 2, 3, 4, 5, 6].into_array(); @@ -1481,6 +1531,7 @@ async fn test_writer_bytes_written() -> VortexResult<()> { #[tokio::test] async fn test_writer_empty_chunks() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let empty = StructArray::from_fields(&[( "numbers", PrimitiveArray::new::(buffer![], Validity::NonNullable).into_array(), @@ -1505,9 +1556,8 @@ async fn test_writer_empty_chunks() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 2); - #[expect(deprecated)] let numbers = result - .to_struct() + .execute::(&mut ctx)? .unmasked_field_by_name("numbers")? .clone(); let expected = buffer![1u32, 2].into_array(); @@ -1518,6 +1568,7 @@ async fn test_writer_empty_chunks() -> VortexResult<()> { #[tokio::test] async fn test_writer_mixed_push_and_stream() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let chunk1 = StructArray::from_fields(&[("numbers", buffer![1u32, 2].into_array())])?.into_array(); let chunk2 = @@ -1544,9 +1595,8 @@ async fn test_writer_mixed_push_and_stream() -> VortexResult<()> { let result = file.scan()?.into_array_stream()?.read_all().await?; assert_eq!(result.len(), 6); - #[expect(deprecated)] let numbers = result - .to_struct() + .execute::(&mut ctx)? .unmasked_field_by_name("numbers")? .clone(); let expected = buffer![1u32, 2, 3, 4, 5, 6].into_array(); @@ -1557,6 +1607,7 @@ async fn test_writer_mixed_push_and_stream() -> VortexResult<()> { #[tokio::test] async fn test_writer_with_complex_types() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); let strings = VarBinArray::from(vec!["hello", "world", "test"]).into_array(); let numbers = buffer![100i32, 200, 300].into_array(); let lists = ListArray::from_iter_slow::( @@ -1587,16 +1638,16 @@ async fn test_writer_with_complex_types() -> VortexResult<()> { assert_eq!(result.len(), 3); assert_eq!(result.dtype(), &dtype); - #[expect(deprecated)] let strings_field = result - .to_struct() + .execute::(&mut ctx)? .unmasked_field_by_name("strings") .cloned()?; - #[expect(deprecated)] - let strings = strings_field.to_varbinview().with_iterator(|iter| { - iter.map(|s| s.map(|st| unsafe { String::from_utf8_unchecked(st.to_vec()) })) - .collect::>() - }); + let strings = strings_field + .execute::(&mut ctx)? + .with_iterator(|iter| { + iter.map(|s| s.map(|st| unsafe { String::from_utf8_unchecked(st.to_vec()) })) + .collect::>() + }); assert_eq!( strings, vec![ diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index 9861d2c0c7f..cd8b1913482 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -165,6 +165,7 @@ impl VortexWriteOptions { stream, self.file_statistics.clone().into(), self.max_variable_length_statistics_size, + &self.session, ); // First, write the magic bytes. diff --git a/vortex-file/tests/test_write_table.rs b/vortex-file/tests/test_write_table.rs index 06f491e9491..8b051250e2e 100644 --- a/vortex-file/tests/test_write_table.rs +++ b/vortex-file/tests/test_write_table.rs @@ -9,8 +9,7 @@ use std::sync::LazyLock; use futures::StreamExt; use futures::pin_mut; use vortex_array::IntoArray; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DictArray; use vortex_array::arrays::ListViewArray; @@ -47,6 +46,7 @@ static SESSION: LazyLock = LazyLock::new(|| { #[tokio::test] async fn test_file_roundtrip() { + let mut ctx = SESSION.create_execution_ctx(); // Create a simple roundtrip let nums = PrimitiveArray::from_iter((0..1024).cycle().take(16_384)).into_array(); @@ -103,10 +103,13 @@ async fn test_file_roundtrip() { while let Some(next) = stream.next().await { let next = next.expect("next"); - #[expect(deprecated)] - let next = next.to_struct(); - #[expect(deprecated)] - let a = next.unmasked_field_by_name("a").unwrap().to_struct(); + let next = next.execute::(&mut ctx).unwrap(); + let a = next + .unmasked_field_by_name("a") + .unwrap() + .clone() + .execute::(&mut ctx) + .unwrap(); let b = next.unmasked_field_by_name("b").unwrap(); let raw = a.unmasked_field_by_name("raw").unwrap(); diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index 0ad60ae80ac..af74bf71ac7 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -28,10 +28,9 @@ use jni::sys::jshort; use jni::sys::jstring; use vortex::array::ArrayRef; use vortex::array::ArrayView; -use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; +use vortex::array::arrays::ExtensionArray; +use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBin; use vortex::array::arrays::VarBinView; use vortex::array::arrays::extension::ExtensionArrayExt; @@ -45,6 +44,7 @@ use vortex::error::VortexExpect; use vortex::error::vortex_err; use vortex::scalar::DecimalValue; +use crate::SESSION; use crate::errors::JNIError; use crate::errors::try_or_throw; @@ -235,8 +235,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getField( let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - #[expect(deprecated)] - let struct_array = array_ref.inner.to_struct(); + let mut ctx = SESSION.create_execution_ctx(); + let struct_array = array_ref.inner.clone().execute::(&mut ctx)?; let idx = index as usize; if idx >= struct_array.struct_fields().nfields() { return Err(vortex_err!("Field index out of bounds").into()); @@ -271,9 +271,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNull( ) -> jboolean { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let is_null = array_ref - .inner - .is_invalid(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let is_null = array_ref.inner.is_invalid(index as usize, &mut ctx)?; if is_null { Ok(JNI_TRUE) } else { Ok(JNI_FALSE) } }) } @@ -286,9 +285,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNullCount( ) -> jint { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let count = array_ref - .inner - .invalid_count(&mut LEGACY_SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let count = array_ref.inner.invalid_count(&mut ctx)?; Ok(jint::try_from(count).unwrap_or(-1)) }) } @@ -304,18 +302,16 @@ macro_rules! get_primitive { ) -> $jtype { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { + let mut ctx = SESSION.create_execution_ctx(); let scalar_value = if array_ref.is_extension { - #[expect(deprecated)] - let ext = array_ref.inner.to_extension(); - ext.storage_array().execute_scalar( - index as usize, - &mut LEGACY_SESSION.create_execution_ctx(), - )? + let ext = array_ref + .inner + .clone() + .execute::(&mut ctx)?; + ext.storage_array() + .execute_scalar(index as usize, &mut ctx)? } else { - array_ref.inner.execute_scalar( - index as usize, - &mut LEGACY_SESSION.create_execution_ctx(), - )? + array_ref.inner.execute_scalar(index as usize, &mut ctx)? }; Ok(scalar_value @@ -347,15 +343,16 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBigDecimal( ) -> jobject { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { + let mut ctx = SESSION.create_execution_ctx(); let scalar_value = if array_ref.is_extension { - #[expect(deprecated)] - let ext = array_ref.inner.to_extension(); + let ext = array_ref + .inner + .clone() + .execute::(&mut ctx)?; ext.storage_array() - .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())? + .execute_scalar(index as usize, &mut ctx)? } else { - array_ref - .inner - .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())? + array_ref.inner.execute_scalar(index as usize, &mut ctx)? }; let decimal_scalar = scalar_value.as_decimal(); @@ -417,9 +414,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBool( ) -> jboolean { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let value = array_ref - .inner - .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; match value.as_bool().value() { None => Ok(JNI_FALSE), Some(b) => { @@ -442,9 +438,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getUTF8<'local>( ) -> jstring { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { - let value = array_ref - .inner - .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; match value.as_utf8().value() { None => Ok(JObject::null().into_raw()), Some(buf_str) => Ok(env.new_string(buf_str.as_str())?.into_raw()), @@ -492,9 +487,8 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBinary<'local>( ) -> jbyteArray { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |env| { - let value = array_ref - .inner - .execute_scalar(index as usize, &mut LEGACY_SESSION.create_execution_ctx())?; + let mut ctx = SESSION.create_execution_ctx(); + let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; match value.as_binary().value() { None => Ok(JObject::null().into_raw()), Some(buf) => Ok(env.byte_array_from_slice(buf.as_slice())?.into_raw()), diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index 22494a81f23..c98f4c6e04f 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -408,7 +408,7 @@ impl core::clone::Clone for vortex_layout::layouts::file_stats::FileStatsAccumul pub fn vortex_layout::layouts::file_stats::FileStatsAccumulator::clone(&self) -> vortex_layout::layouts::file_stats::FileStatsAccumulator -pub fn vortex_layout::layouts::file_stats::accumulate_stats(stream: vortex_layout::sequence::SendableSequentialStream, stats: alloc::sync::Arc<[vortex_array::expr::stats::Stat]>, max_variable_length_statistics_size: usize) -> (vortex_layout::layouts::file_stats::FileStatsAccumulator, vortex_layout::sequence::SendableSequentialStream) +pub fn vortex_layout::layouts::file_stats::accumulate_stats(stream: vortex_layout::sequence::SendableSequentialStream, stats: alloc::sync::Arc<[vortex_array::expr::stats::Stat]>, max_variable_length_statistics_size: usize, session: &vortex_session::VortexSession) -> (vortex_layout::layouts::file_stats::FileStatsAccumulator, vortex_layout::sequence::SendableSequentialStream) pub mod vortex_layout::layouts::flat @@ -800,7 +800,7 @@ pub fn vortex_layout::layouts::zoned::zone_map::StatsAccumulator::as_stats_table pub fn vortex_layout::layouts::zoned::zone_map::StatsAccumulator::new(dtype: &vortex_array::dtype::DType, stats: &[vortex_array::expr::stats::Stat], max_variable_length_statistics_size: usize) -> Self -pub fn vortex_layout::layouts::zoned::zone_map::StatsAccumulator::push_chunk(&mut self, array: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult<()> +pub fn vortex_layout::layouts::zoned::zone_map::StatsAccumulator::push_chunk(&mut self, array: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_layout::layouts::zoned::zone_map::StatsAccumulator::push_chunk_without_compute(&mut self, array: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult<()> diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 446df56611d..5896c5b466f 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -263,6 +263,7 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; + use vortex_array::Canonical; use vortex_array::IntoArray as _; use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; @@ -466,6 +467,7 @@ mod tests { #[test] fn reading_is_null_works() { block_on(|handle| async move { + let mut ctx_exec = LEGACY_SESSION.create_execution_ctx(); let session = session_with_handle(handle); let strategy = DictStrategy::new( FlatLayoutStrategy::default(), @@ -525,12 +527,11 @@ mod tests { let expected = array .validity() .unwrap() - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(array.len(), &mut ctx_exec) .unwrap() .into_array(); - #[expect(deprecated)] let actual_canonical = actual - .to_canonical() + .execute::(&mut ctx_exec) .vortex_expect("to_canonical failed") .into_array(); assert_arrays_eq!(actual_canonical, expected); diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index d08c5933fb5..a970cfbedb3 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -8,10 +8,9 @@ use futures::StreamExt; use itertools::Itertools; use parking_lot::Mutex; use vortex_array::ArrayRef; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical as _; +use vortex_array::ExecutionCtx; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; @@ -20,6 +19,7 @@ use vortex_array::stats::StatsSet; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use vortex_session::VortexSession; use crate::layouts::zoned::zone_map::StatsAccumulator; use crate::sequence::SendableSequentialStream; @@ -31,9 +31,14 @@ pub fn accumulate_stats( stream: SendableSequentialStream, stats: Arc<[Stat]>, max_variable_length_statistics_size: usize, + session: &VortexSession, ) -> (FileStatsAccumulator, SendableSequentialStream) { - let accumulator = - FileStatsAccumulator::new(stream.dtype(), stats, max_variable_length_statistics_size); + let accumulator = FileStatsAccumulator::new( + stream.dtype(), + stats, + max_variable_length_statistics_size, + session, + ); let stream = SequentialStreamAdapter::new( stream.dtype().clone(), stream.scan(accumulator.clone(), |acc, item| { @@ -51,10 +56,16 @@ pub fn accumulate_stats( pub struct FileStatsAccumulator { stats: Arc<[Stat]>, accumulators: Arc>>, + ctx: Arc>, } impl FileStatsAccumulator { - fn new(dtype: &DType, stats: Arc<[Stat]>, max_variable_length_statistics_size: usize) -> Self { + fn new( + dtype: &DType, + stats: Arc<[Stat]>, + max_variable_length_statistics_size: usize, + session: &VortexSession, + ) -> Self { let accumulators = Arc::new(Mutex::new(match dtype.as_struct_fields_opt() { Some(struct_dtype) => { if dtype.nullability() == Nullability::Nullable { @@ -87,6 +98,7 @@ impl FileStatsAccumulator { Self { stats, accumulators, + ctx: Arc::new(Mutex::new(session.create_execution_ctx())), } } @@ -95,25 +107,25 @@ impl FileStatsAccumulator { chunk: VortexResult<(SequenceId, ArrayRef)>, ) -> VortexResult<(SequenceId, ArrayRef)> { let (sequence_id, chunk) = chunk?; + let mut ctx = self.ctx.lock(); if chunk.dtype().is_struct() { - #[expect(deprecated)] - let chunk = chunk.to_struct(); + let struct_chunk = chunk.clone().execute::(&mut ctx)?; for (acc, field) in self .accumulators .lock() .iter_mut() - .zip_eq(chunk.iter_unmasked_fields()) + .zip_eq(struct_chunk.iter_unmasked_fields()) { - acc.push_chunk(field)?; + acc.push_chunk(field, &mut ctx)?; } } else { - self.accumulators.lock()[0].push_chunk(&chunk)?; + self.accumulators.lock()[0].push_chunk(&chunk, &mut ctx)?; } Ok((sequence_id, chunk)) } pub fn stats_sets(&self) -> Vec { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = self.ctx.lock(); self.accumulators .lock() .iter_mut() diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 46cde10789f..30ec0fe9352 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -201,8 +201,6 @@ mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Dict; @@ -344,6 +342,7 @@ mod tests { #[test] fn struct_array_round_trip() { block_on(|handle| async { + let mut ctx_exec = LEGACY_SESSION.create_execution_ctx(); let session = SESSION.clone().with_handle(handle); let mut validity_builder = BitBufferMut::with_capacity(2); validity_builder.append(true); @@ -400,26 +399,29 @@ mod tests { result .validity() .unwrap() - .to_mask(result.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(result.len(), &mut ctx_exec) .unwrap() .bit_buffer(), AllOr::Some(&validity_boolean_buffer) ); - #[expect(deprecated)] - let result_struct = result.to_struct(); - #[expect(deprecated)] + let result_struct = result + .clone() + .execute::(&mut ctx_exec) + .unwrap(); let field_a = result_struct .unmasked_field_by_name("a") .unwrap() - .to_primitive(); + .clone() + .execute::(&mut ctx_exec) + .unwrap(); assert_eq!(field_a.as_slice::(), &[1, 2]); - #[expect(deprecated)] - let result_struct_b = result.to_struct(); - #[expect(deprecated)] + let result_struct_b = result.execute::(&mut ctx_exec).unwrap(); let field_b = result_struct_b .unmasked_field_by_name("b") .unwrap() - .to_primitive(); + .clone() + .execute::(&mut ctx_exec) + .unwrap(); assert_eq!(field_b.as_slice::(), &[3, 4]); }) } diff --git a/vortex-layout/src/layouts/repartition.rs b/vortex-layout/src/layouts/repartition.rs index f9bd3de86ac..c810078e55b 100644 --- a/vortex-layout/src/layouts/repartition.rs +++ b/vortex-layout/src/layouts/repartition.rs @@ -10,7 +10,9 @@ use futures::StreamExt as _; use futures::pin_mut; use vortex_array::ArrayContext; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::dtype::DType; use vortex_error::VortexExpect; @@ -103,12 +105,13 @@ impl LayoutStrategy for RepartitionStrategy { // canon_stream = stream.map(async {to_canonical}).map(spawn).buffered(parallelism) let dtype = stream.dtype().clone(); let stream = if self.options.canonicalize { + let canonicalize_session = session.clone(); SequentialStreamAdapter::new( dtype.clone(), - stream.map(|chunk| { + stream.map(move |chunk| { let (sequence_id, chunk) = chunk?; - #[expect(deprecated)] - let canonical = chunk.to_canonical()?.into_array(); + let mut ctx = canonicalize_session.create_execution_ctx(); + let canonical = chunk.execute::(&mut ctx)?.into_array(); VortexResult::Ok((sequence_id, canonical)) }), ) @@ -125,11 +128,13 @@ impl LayoutStrategy for RepartitionStrategy { // segments. let block_len = options.effective_block_len(&dtype); let block_size_minimum = options.block_size_minimum; + let repartition_session = session.clone(); let repartitioned_stream = try_stream! { let canonical_stream = stream.peekable(); pin_mut!(canonical_stream); + let mut ctx = repartition_session.create_execution_ctx(); let mut chunks = ChunksBuffer::new(block_size_minimum, block_len); while let Some(chunk) = canonical_stream.as_mut().next().await { let (sequence_id, chunk) = chunk?; @@ -147,8 +152,7 @@ impl LayoutStrategy for RepartitionStrategy { let chunked = ChunkedArray::try_new(output_chunks, dtype_clone.clone())?; if !chunked.is_empty() { - #[expect(deprecated)] - let canonical = chunked.into_array().to_canonical()?.into_array(); + let canonical = chunked.into_array().execute::(&mut ctx)?.into_array(); yield ( sequence_pointer.advance(), canonical, @@ -162,8 +166,7 @@ impl LayoutStrategy for RepartitionStrategy { dtype_clone.clone(), )?; if !to_flush.is_empty() { - #[expect(deprecated)] - let canonical = to_flush.into_array().to_canonical()?.into_array(); + let canonical = to_flush.into_array().execute::(&mut ctx)?.into_array(); yield ( sequence_pointer.advance(), canonical, @@ -273,6 +276,7 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; @@ -485,6 +489,7 @@ mod tests { /// `pop_front` subtracted the larger Cached-era values. #[test] fn chunks_buffer_pop_front_no_panic_after_shared_execution() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let n = 20_000usize; let block_len = 10_000usize; @@ -504,8 +509,8 @@ mod tests { // Transition SharedState from Source to Cached for ALL slices sharing this Arc. use vortex_array::arrays::shared::SharedArrayExt; - #[expect(deprecated)] - let _canonical = shared_handle.get_or_compute(|source| source.to_canonical())?; + let _canonical = + shared_handle.get_or_compute(|source| source.clone().execute::(&mut ctx))?; // Before the fix this panicked with "attempt to subtract with overflow". let _s2 = buf.pop_front().unwrap(); diff --git a/vortex-layout/src/layouts/row_idx/mod.rs b/vortex-layout/src/layouts/row_idx/mod.rs index 05218e0994d..33078617722 100644 --- a/vortex-layout/src/layouts/row_idx/mod.rs +++ b/vortex-layout/src/layouts/row_idx/mod.rs @@ -16,6 +16,7 @@ pub use expr::*; use futures::FutureExt; use futures::future::BoxFuture; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; @@ -218,9 +219,13 @@ impl LayoutReader for RowIdxLayoutReader { Partition::Child => self.child.filter_evaluation(row_range, expr, mask), }, |annotation, expr, mask| match annotation { - Partition::RowIdx => { - Ok(row_idx_array_future(self.row_offset, row_range, expr, mask)) - } + Partition::RowIdx => Ok(row_idx_array_future( + self.row_offset, + row_range, + expr, + mask, + self.session.clone(), + )), Partition::Child => self.child.projection_evaluation(row_range, expr, mask), }, self.session.clone(), @@ -235,15 +240,23 @@ impl LayoutReader for RowIdxLayoutReader { mask: MaskFuture, ) -> VortexResult>> { match &self.partition_expr(expr) { - Partitioning::RowIdx(expr) => { - Ok(row_idx_array_future(self.row_offset, row_range, expr, mask)) - } + Partitioning::RowIdx(expr) => Ok(row_idx_array_future( + self.row_offset, + row_range, + expr, + mask, + self.session.clone(), + )), Partitioning::Child(expr) => self.child.projection_evaluation(row_range, expr, mask), Partitioning::Partitioned(p) => { Arc::clone(p).into_array_future(mask, |annotation, expr, mask| match annotation { - Partition::RowIdx => { - Ok(row_idx_array_future(self.row_offset, row_range, expr, mask)) - } + Partition::RowIdx => Ok(row_idx_array_future( + self.row_offset, + row_range, + expr, + mask, + self.session.clone(), + )), Partition::Child => self.child.projection_evaluation(row_range, expr, mask), }) } @@ -292,14 +305,15 @@ fn row_idx_array_future( row_range: &Range, expr: &Expression, mask: MaskFuture, + session: VortexSession, ) -> ArrayFuture { let row_range = row_range.clone(); let expr = expr.clone(); async move { let array = idx_array(row_offset, &row_range).into_array(); let filtered = array.filter(mask.await?)?; - #[expect(deprecated)] - let array = filtered.to_canonical()?.into_array(); + let mut ctx = session.create_execution_ctx(); + let array = filtered.execute::(&mut ctx)?.into_array(); array.apply(&expr) } .boxed() diff --git a/vortex-layout/src/layouts/struct_/reader.rs b/vortex-layout/src/layouts/struct_/reader.rs index b0777f3ee89..94cf534ea16 100644 --- a/vortex-layout/src/layouts/struct_/reader.rs +++ b/vortex-layout/src/layouts/struct_/reader.rs @@ -11,8 +11,7 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::IntoArray; use vortex_array::MaskFuture; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::builtins::ArrayBuiltins; @@ -354,14 +353,15 @@ impl LayoutReader for StructReader { ), }; + let session = self.session.clone(); Ok(Box::pin(async move { if let Some(validity_fut) = validity_fut { let (array, validity) = try_join!(projected, validity_fut)?; // If root expression was a pack, then we apply the validity to each child field if is_pack_merge { - #[expect(deprecated)] - let struct_array = array.to_struct(); + let mut ctx = session.create_execution_ctx(); + let struct_array = array.execute::(&mut ctx)?; let masked_fields: Vec = struct_array .iter_unmasked_fields() .map(|a| a.clone().mask(validity.clone())) @@ -400,8 +400,6 @@ mod tests { use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; - #[expect(deprecated)] - use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; @@ -675,6 +673,7 @@ mod tests { fn test_struct_layout_select( #[from(struct_layout)] (segments, layout): (Arc, LayoutRef), ) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let reader = layout.new_reader("".into(), segments, &SESSION).unwrap(); let expr = pack( [("a", get_item("a", root())), ("b", get_item("b", root()))], @@ -695,16 +694,14 @@ mod tests { assert_eq!(result.len(), 2); let expected_a = PrimitiveArray::from_iter([7i32, 2]); - #[expect(deprecated)] - let result_struct_a = result.to_struct(); + let result_struct_a = result.clone().execute::(&mut ctx).unwrap(); assert_arrays_eq!( result_struct_a.unmasked_field_by_name("a").unwrap(), expected_a ); let expected_b = PrimitiveArray::from_iter([4i32, 5]); - #[expect(deprecated)] - let result_struct_b = result.to_struct(); + let result_struct_b = result.execute::(&mut ctx).unwrap(); assert_arrays_eq!( result_struct_b.unmasked_field_by_name("b").unwrap(), expected_b diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index ca7a9bb4c27..36567595835 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -16,10 +16,8 @@ use itertools::Itertools; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Field; @@ -234,18 +232,19 @@ impl LayoutStrategy for TableStrategy { } // stream -> stream> + let columns_session = session.clone(); let columns_vec_stream = stream.map(move |chunk| { let (sequence_id, chunk) = chunk?; let mut sequence_pointer = sequence_id.descend(); - #[expect(deprecated)] - let struct_chunk = chunk.to_struct(); + let mut ctx = columns_session.create_execution_ctx(); + let struct_chunk = chunk.clone().execute::(&mut ctx)?; let mut columns: Vec<(SequenceId, ArrayRef)> = Vec::new(); if is_nullable { columns.push(( sequence_pointer.advance(), chunk .validity()? - .to_mask(chunk.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_mask(chunk.len(), &mut ctx)? .into_array(), )); } diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 4485440515e..10f2c1ff598 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -16,8 +16,8 @@ use itertools::Itertools; use parking_lot::RwLock; use vortex_array::ArrayRef; use vortex_array::MaskFuture; -#[expect(deprecated)] -use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::FieldPath; @@ -135,10 +135,11 @@ impl ZonedReader { MaskFuture::new_true(nzones), ) .vortex_expect("Failed construct zone map evaluation"); + let session = self.session.clone(); async move { - #[expect(deprecated)] - let zones_array = zones_eval.await?.to_struct(); + let mut ctx = session.create_execution_ctx(); + let zones_array = zones_eval.await?.execute::(&mut ctx)?; // SAFETY: This is only fine to call because we perform validation above Ok(unsafe { ZoneMap::new_unchecked(zones_array, present_stats) }) } diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 083dc5ca787..00df7f72dd4 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -7,7 +7,6 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::sum::sum; use vortex_array::arrays::StructArray; @@ -219,12 +218,9 @@ impl StatsAccumulator { Ok(()) } - pub fn push_chunk(&mut self, array: &ArrayRef) -> VortexResult<()> { + pub fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { for builder in self.builders.iter_mut() { - if let Some(v) = array - .statistics() - .compute_stat(builder.stat(), &mut LEGACY_SESSION.create_execution_ctx())? - { + if let Some(v) = array.statistics().compute_stat(builder.stat(), ctx)? { builder.append_scalar(v.cast(&v.dtype().as_nullable())?)?; } else { builder.append_null(); @@ -279,8 +275,8 @@ mod tests { use rstest::rstest; use vortex_array::IntoArray; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; @@ -316,6 +312,7 @@ mod tests { #[case(DType::Utf8(Nullability::NonNullable))] #[case(DType::Binary(Nullability::NonNullable))] fn truncates_accumulated_stats(#[case] dtype: DType) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut builder = VarBinViewBuilder::with_capacity(dtype.clone(), 2); builder.append_value("Value to be truncated"); builder.append_value("untruncated"); @@ -324,9 +321,9 @@ mod tests { builder2.append_value("wait a minute"); let mut acc = StatsAccumulator::new(builder.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); - acc.push_chunk(&builder.finish()) + acc.push_chunk(&builder.finish(), &mut ctx) .vortex_expect("push_chunk should succeed for test data"); - acc.push_chunk(&builder2.finish()) + acc.push_chunk(&builder2.finish(), &mut ctx) .vortex_expect("push_chunk should succeed for test data"); let stats_table = acc .as_stats_table() @@ -341,14 +338,22 @@ mod tests { MIN_IS_TRUNCATED, ] ); - #[expect(deprecated)] - let field1_bool = stats_table.array.unmasked_field(1).to_bool(); + let field1_bool = stats_table + .array + .unmasked_field(1) + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( field1_bool.to_bit_buffer(), BitBuffer::from(vec![false, true]) ); - #[expect(deprecated)] - let field3_bool = stats_table.array.unmasked_field(3).to_bool(); + let field3_bool = stats_table + .array + .unmasked_field(3) + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!( field3_bool.to_bit_buffer(), BitBuffer::from(vec![true, false]) @@ -357,9 +362,10 @@ mod tests { #[test] fn always_adds_is_truncated_column() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = buffer![0, 1, 2].into_array(); let mut acc = StatsAccumulator::new(array.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); - acc.push_chunk(&array) + acc.push_chunk(&array, &mut ctx) .vortex_expect("push_chunk should succeed for test array"); let stats_table = acc .as_stats_table() @@ -375,11 +381,19 @@ mod tests { Stat::Sum.name(), ] ); - #[expect(deprecated)] - let field1_bool = stats_table.array.unmasked_field(1).to_bool(); + let field1_bool = stats_table + .array + .unmasked_field(1) + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(field1_bool.to_bit_buffer(), BitBuffer::from(vec![false])); - #[expect(deprecated)] - let field3_bool = stats_table.array.unmasked_field(3).to_bool(); + let field3_bool = stats_table + .array + .unmasked_field(3) + .clone() + .execute::(&mut ctx) + .unwrap(); assert_eq!(field3_bool.to_bit_buffer(), BitBuffer::from(vec![false])); } diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 6053cd6ebf5..84dfbdd93ce 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -465,9 +465,9 @@ mod test { use futures::task::noop_waker_ref; use parking_lot::Mutex; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; - #[expect(deprecated)] - use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; @@ -662,6 +662,7 @@ mod test { #[test] fn into_stream_executes_after_prepare() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let calls = Arc::new(AtomicUsize::new(0)); let reader = Arc::new(SplittingLayoutReader::new(Arc::clone(&calls))); @@ -673,8 +674,7 @@ mod test { let mut values = Vec::new(); for chunk in &mut iter { - #[expect(deprecated)] - let prim = chunk?.to_primitive(); + let prim = chunk?.execute::(&mut ctx)?; values.push(prim.into_buffer::()[0]); } diff --git a/vortex-python/src/arrays/compressed.rs b/vortex-python/src/arrays/compressed.rs index ca14c9a52b5..2bb7c1dd7a8 100644 --- a/vortex-python/src/arrays/compressed.rs +++ b/vortex-python/src/arrays/compressed.rs @@ -3,9 +3,10 @@ use pyo3::prelude::*; use vortex::array::IntoArray; -#[expect(deprecated)] -use vortex::array::ToCanonical; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::Dict; +use vortex::array::arrays::PrimitiveArray; use vortex::encodings::alp::ALP; use vortex::encodings::alp::ALPRD; use vortex::encodings::datetime_parts::DateTimeParts; @@ -90,8 +91,11 @@ impl EncodingSubclass for PyZigZagArray { impl PyZigZagArray { #[staticmethod] pub fn encode(array: PyArrayRef) -> PyVortexResult { - #[expect(deprecated)] - let primitive = array.inner().clone().to_primitive(); + // PyZigZagArray (and PyArrayRef) do not currently carry a VortexSession; + // threading one through would change the FromPyObject contract. Use + // LEGACY_SESSION until the wrappers are refactored. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let primitive = array.inner().clone().execute::(&mut ctx)?; Ok(PyVortex(zigzag_encode(primitive.as_view())?.into_array())) } } diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index af5f6afee6f..28ec1ac2020 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -23,11 +23,11 @@ use pyo3::types::PyRange; use pyo3::types::PyRangeMethods; use pyo3_bytes::PyBytes; use vortex::array::ArrayRef; +use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; +use vortex::array::arrays::BoolArray; use vortex::array::arrays::Chunked; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::chunked::ChunkedArrayExt; @@ -527,12 +527,16 @@ impl PyArray { /// ] /// ``` fn filter(slf: Bound, mask: PyArrayRef) -> PyVortexResult { + // PyArray/PyArrayRef do not currently carry a VortexSession; threading one + // through would change the FromPyObject contract. Use LEGACY_SESSION until + // the wrappers are refactored. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); - #[expect(deprecated)] - let mask_bool = (&*mask as &ArrayRef).to_bool(); - let mask = mask_bool.to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); - #[expect(deprecated)] - let canonical = slf.filter(mask)?.to_canonical()?; + let mask_bool = (&*mask as &ArrayRef) + .clone() + .execute::(&mut ctx)?; + let mask = mask_bool.to_mask_fill_null_false(&mut ctx); + let canonical = slf.filter(mask)?.execute::(&mut ctx)?; let inner = canonical.into_array(); Ok(PyArrayRef::from(inner)) } @@ -608,6 +612,10 @@ impl PyArray { /// ``` // TODO(ngates): return a vortex.Scalar fn scalar_at(slf: Bound, index: usize) -> PyVortexResult> { + // PyArray/PyArrayRef do not currently carry a VortexSession; threading one + // through would change the FromPyObject contract. Use LEGACY_SESSION until + // the wrappers are refactored. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let py = slf.py(); let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); if index >= slf.len() { @@ -617,10 +625,7 @@ impl PyArray { )) .into()); } - Ok(PyScalar::init( - py, - slf.execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx())?, - )?) + Ok(PyScalar::init(py, slf.execute_scalar(index, &mut ctx)?)?) } /// Filter, permute, and/or repeat elements by their index. diff --git a/vortex-python/src/dataset.rs b/vortex-python/src/dataset.rs index c9579043da6..bb1a99b8b48 100644 --- a/vortex-python/src/dataset.rs +++ b/vortex-python/src/dataset.rs @@ -11,8 +11,9 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyString; use vortex::array::ArrayRef; -#[expect(deprecated)] -use vortex::array::ToCanonical; +use vortex::array::ExecutionCtx; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::PrimitiveArray; use vortex::array::iter::ArrayIteratorExt; use vortex::dtype::FieldName; use vortex::dtype::FieldNames; @@ -23,6 +24,7 @@ use vortex::expr::select; use vortex::file::OpenOptionsSessionExt; use vortex::file::VortexFile; use vortex::layout::scan::split_by::SplitBy; +use vortex::session::VortexSession; use crate::RUNTIME; use crate::SESSION; @@ -54,6 +56,7 @@ pub fn read_array_from_reader( filter: Option, indices: Option, row_range: Option<(u64, u64)>, + ctx: &mut ExecutionCtx, ) -> VortexResult { let mut scan = vortex_file.scan()?.with_projection(projection); @@ -62,8 +65,7 @@ pub fn read_array_from_reader( } if let Some(indices) = indices { - #[expect(deprecated)] - let primitive = indices.to_primitive(); + let primitive = indices.execute::(ctx)?; let indices = primitive.into_buffer(); scan = scan.with_row_indices(indices); } @@ -106,12 +108,17 @@ fn filter_from_python(row_filter: Option<&Bound>) -> Option pub struct PyVortexDataset { vxf: VortexFile, schema: SchemaRef, + session: VortexSession, } impl PyVortexDataset { - pub fn try_new(vxf: VortexFile) -> VortexResult { + pub fn try_new(vxf: VortexFile, session: VortexSession) -> VortexResult { let schema = Arc::new(vxf.dtype().to_arrow_schema()?); - Ok(Self { vxf, schema }) + Ok(Self { + vxf, + schema, + session, + }) } pub async fn from_url( @@ -127,7 +134,7 @@ impl PyVortexDataset { } ResolvedStore::Path(path) => SESSION.open_options().open_path(path).await?, }; - PyVortexDataset::try_new(vxf) + PyVortexDataset::try_new(vxf, SESSION.clone()) } } @@ -145,12 +152,14 @@ impl PyVortexDataset { indices: Option, row_range: Option<(u64, u64)>, ) -> PyVortexResult { + let mut ctx = self.session.create_execution_ctx(); let array = read_array_from_reader( &self.vxf, projection_from_python(columns)?, filter_from_python(row_filter), indices.map(|i| i.into_inner()), row_range, + &mut ctx, )?; Ok(PyArrayRef::from(array)) } diff --git a/vortex-python/src/file.rs b/vortex-python/src/file.rs index 458736db0e5..0ba334ae7d4 100644 --- a/vortex-python/src/file.rs +++ b/vortex-python/src/file.rs @@ -9,8 +9,9 @@ use pyo3::prelude::*; use pyo3::types::PyList; use pyo3_object_store::PyObjectStore; use vortex::array::ArrayRef; -#[expect(deprecated)] -use vortex::array::ToCanonical; +use vortex::array::ExecutionCtx; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::PrimitiveArray; use vortex::array::builtins::ArrayBuiltins; use vortex::dtype::DType; use vortex::dtype::FieldNames; @@ -25,6 +26,7 @@ use vortex::file::VortexFile; use vortex::layout::scan::scan_builder::ScanBuilder; use vortex::layout::scan::split_by::SplitBy; use vortex::layout::segments::MokaSegmentCache; +use vortex::session::VortexSession; use crate::RUNTIME; use crate::SESSION; @@ -81,12 +83,16 @@ pub fn open( }) })?; - Ok(PyVortexFile { vxf }) + Ok(PyVortexFile { + vxf, + session: SESSION.clone(), + }) } #[pyclass(name = "VortexFile", module = "vortex", frozen)] pub struct PyVortexFile { vxf: VortexFile, + session: VortexSession, } #[pymethods] @@ -109,12 +115,14 @@ impl PyVortexFile { indices: Option, batch_size: Option, ) -> PyVortexResult { + let mut ctx = slf.get().session.create_execution_ctx(); let builder = slf.get().scan_builder( projection.map(|p| p.0), expr.map(|e| e.into_inner()), limit, indices.map(|i| i.into_inner()), batch_size, + &mut ctx, )?; Ok(PyArrayIterator::new(Box::new( @@ -131,12 +139,14 @@ impl PyVortexFile { indices: Option, batch_size: Option, ) -> PyVortexResult { + let mut ctx = slf.get().session.create_execution_ctx(); let builder = slf.get().scan_builder( projection.map(|p| p.0), expr.map(|e| e.into_inner()), limit, indices.map(|i| i.into_inner()), batch_size, + &mut ctx, )?; let scan = builder.prepare()?; @@ -180,7 +190,10 @@ impl PyVortexFile { } fn to_dataset(slf: Bound) -> PyVortexResult { - Ok(PyVortexDataset::try_new(slf.get().vxf.clone())?) + Ok(PyVortexDataset::try_new( + slf.get().vxf.clone(), + slf.get().session.clone(), + )?) } #[pyo3(signature = (*))] @@ -202,6 +215,7 @@ impl PyVortexFile { limit: Option, indices: Option, batch_size: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut builder = self .vxf @@ -215,8 +229,7 @@ impl PyVortexFile { if let Some(indices) = indices { let casted = indices.cast(DType::Primitive(PType::U64, NonNullable))?; - #[expect(deprecated)] - let indices = casted.to_primitive().into_buffer::(); + let indices = casted.execute::(ctx)?.into_buffer::(); builder = builder.with_row_indices(indices); } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs index 04b5115e94d..97af909b966 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -46,6 +48,9 @@ impl FlatLayoutFixture for AlprdFixture { } fn build(&self) -> VortexResult { + // NOTE: `FlatLayoutFixture::build` has a fixed trait signature without `ExecutionCtx`, so + // we construct a legacy ctx locally at this trait boundary. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let sensor: PrimitiveArray = (0..N) .map(|i| { let noise = ((i * 7 + 13) % 100) as f64 / 1000.0; @@ -150,21 +155,31 @@ impl FlatLayoutFixture for AlprdFixture { "nullable_specials", ]), vec![ - sensor_enc.encode(sensor.as_view()).into_array(), - drift_enc.encode(drift.as_view()).into_array(), - constant_enc.encode(constant_series.as_view()).into_array(), - decreasing_enc.encode(decreasing.as_view()).into_array(), - oscillating_enc.encode(oscillating.as_view()).into_array(), + sensor_enc.encode(sensor.as_view(), &mut ctx).into_array(), + drift_enc.encode(drift.as_view(), &mut ctx).into_array(), + constant_enc + .encode(constant_series.as_view(), &mut ctx) + .into_array(), + decreasing_enc + .encode(decreasing.as_view(), &mut ctx) + .into_array(), + oscillating_enc + .encode(oscillating.as_view(), &mut ctx) + .into_array(), periodic_resets_enc - .encode(periodic_resets.as_view()) + .encode(periodic_resets.as_view(), &mut ctx) + .into_array(), + nullable_enc + .encode(sensor_nullable.as_view(), &mut ctx) + .into_array(), + special_enc + .encode(special_values.as_view(), &mut ctx) .into_array(), - nullable_enc.encode(sensor_nullable.as_view()).into_array(), - special_enc.encode(special_values.as_view()).into_array(), boundary_enc - .encode(boundary_specials.as_view()) + .encode(boundary_specials.as_view(), &mut ctx) .into_array(), nullable_special_enc - .encode(nullable_specials.as_view()) + .encode(nullable_specials.as_view(), &mut ctx) .into_array(), ], N, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs index 9f3b67291d9..f242ebc4e0a 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bitpacked.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -32,6 +34,9 @@ impl FlatLayoutFixture for BitPackedFixture { } fn build(&self) -> VortexResult { + // NOTE: `FlatLayoutFixture::build` has a fixed `(&self)` trait signature, so we cannot + // plumb `ExecutionCtx` from a caller; construct one from `LEGACY_SESSION` here. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let u32_8bit: PrimitiveArray = (0..N as u32).map(|i| i % 256).collect(); let u64_12bit: PrimitiveArray = (0..N as u64).map(|i| i % 4096).collect(); let u16_4bit: PrimitiveArray = (0..N as u16).map(|i| i % 16).collect(); @@ -80,21 +85,21 @@ impl FlatLayoutFixture for BitPackedFixture { "u16_head_tail_nulls", ]), vec![ - bitpack_encode(&u32_8bit, 8, None)?.into_array(), - bitpack_encode(&u64_12bit, 12, None)?.into_array(), - bitpack_encode(&u16_4bit, 4, None)?.into_array(), - bitpack_encode(&u16_1bit, 1, None)?.into_array(), - bitpack_encode(&u32_nullable, 7, None)?.into_array(), - bitpack_encode(&u32_all_zero, 1, None)?.into_array(), - bitpack_encode(&u16_all_equal, 3, None)?.into_array(), - bitpack_encode(&u16_15bit, 15, None)?.into_array(), - bitpack_encode(&u32_31bit, 31, None)?.into_array(), - bitpack_encode(&u64_63bit, 63, None)?.into_array(), - bitpack_encode(&u8_3bit, 3, None)?.into_array(), - bitpack_encode(&u8_5bit, 5, None)?.into_array(), - bitpack_encode(&u16_9bit, 9, None)?.into_array(), - bitpack_encode(&u32_17bit, 17, None)?.into_array(), - bitpack_encode(&u16_head_tail_nulls, 5, None)?.into_array(), + bitpack_encode(&u32_8bit, 8, None, &mut ctx)?.into_array(), + bitpack_encode(&u64_12bit, 12, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_4bit, 4, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_1bit, 1, None, &mut ctx)?.into_array(), + bitpack_encode(&u32_nullable, 7, None, &mut ctx)?.into_array(), + bitpack_encode(&u32_all_zero, 1, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_all_equal, 3, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_15bit, 15, None, &mut ctx)?.into_array(), + bitpack_encode(&u32_31bit, 31, None, &mut ctx)?.into_array(), + bitpack_encode(&u64_63bit, 63, None, &mut ctx)?.into_array(), + bitpack_encode(&u8_3bit, 3, None, &mut ctx)?.into_array(), + bitpack_encode(&u8_5bit, 5, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_9bit, 9, None, &mut ctx)?.into_array(), + bitpack_encode(&u32_17bit, 17, None, &mut ctx)?.into_array(), + bitpack_encode(&u16_head_tail_nulls, 5, None, &mut ctx)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs index c77b6c6a731..0bfb9985e1d 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/datetimeparts.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::TemporalArray; @@ -22,7 +24,7 @@ pub struct DateTimePartsFixture; fn encode_temporal(temporal: TemporalArray) -> VortexResult { let dtype = temporal.dtype().clone(); - let parts = split_temporal(temporal)?; + let parts = split_temporal(temporal, &mut LEGACY_SESSION.create_execution_ctx())?; Ok(DateTimeParts::try_new(dtype, parts.days, parts.seconds, parts.subseconds)?.into_array()) } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs index 4365fb1a01c..17314629737 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinArray; use vortex::array::dtype::FieldNames; @@ -109,6 +111,7 @@ impl FlatLayoutFixture for FsstFixture { let high_entropy_comp = fsst_train_compressor(&high_entropy_col); let all_null_clustered_comp = fsst_train_compressor(&all_null_clustered); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let arr = StructArray::try_new( FieldNames::from([ "urls", @@ -121,22 +124,44 @@ impl FlatLayoutFixture for FsstFixture { "all_null_clustered", ]), vec![ - fsst_compress(&url_col, url_col.len(), url_col.dtype(), &url_comp).into_array(), - fsst_compress(&log_col, log_col.len(), log_col.dtype(), &log_comp).into_array(), + fsst_compress( + &url_col, + url_col.len(), + url_col.dtype(), + &url_comp, + &mut ctx, + ) + .into_array(), + fsst_compress( + &log_col, + log_col.len(), + log_col.dtype(), + &log_comp, + &mut ctx, + ) + .into_array(), fsst_compress( &nullable_col, nullable_col.len(), nullable_col.dtype(), &nullable_comp, + &mut ctx, + ) + .into_array(), + fsst_compress( + &short_col, + short_col.len(), + short_col.dtype(), + &short_comp, + &mut ctx, ) .into_array(), - fsst_compress(&short_col, short_col.len(), short_col.dtype(), &short_comp) - .into_array(), fsst_compress( &empty_and_unicode_col, empty_and_unicode_col.len(), empty_and_unicode_col.dtype(), &empty_and_unicode_comp, + &mut ctx, ) .into_array(), fsst_compress( @@ -144,6 +169,7 @@ impl FlatLayoutFixture for FsstFixture { suffix_shared_col.len(), suffix_shared_col.dtype(), &suffix_shared_comp, + &mut ctx, ) .into_array(), fsst_compress( @@ -151,6 +177,7 @@ impl FlatLayoutFixture for FsstFixture { high_entropy_col.len(), high_entropy_col.dtype(), &high_entropy_comp, + &mut ctx, ) .into_array(), fsst_compress( @@ -158,6 +185,7 @@ impl FlatLayoutFixture for FsstFixture { all_null_clustered.len(), all_null_clustered.dtype(), &all_null_clustered_comp, + &mut ctx, ) .into_array(), ], diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs index f6902fdc2c5..7c3f4edaf72 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -31,6 +33,9 @@ impl FlatLayoutFixture for PcoFixture { } fn build(&self) -> VortexResult { + // NOTE: `FlatLayoutFixture::build` has a fixed trait signature without `ExecutionCtx`, so + // we construct a legacy ctx locally at this trait boundary. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let irregular_i64: PrimitiveArray = (0..N as i64).map(|i| i * i + (i % 17) * 1000).collect(); let smooth_f64: PrimitiveArray = (0..N) @@ -70,14 +75,14 @@ impl FlatLayoutFixture for PcoFixture { "narrow_i16", ]), vec![ - Pco::from_primitive(irregular_i64.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(smooth_f64.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(pattern_u32.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(nullable_f32.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(negative_i32.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(constant_u16.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(spike_outliers.as_view(), 8, 0)?.into_array(), - Pco::from_primitive(narrow_i16.as_view(), 8, 0)?.into_array(), + Pco::from_primitive(irregular_i64.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(smooth_f64.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(pattern_u32.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(nullable_f32.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(negative_i32.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(constant_u16.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(spike_outliers.as_view(), 8, 0, &mut ctx)?.into_array(), + Pco::from_primitive(narrow_i16.as_view(), 8, 0, &mut ctx)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs index bebcb9fa12b..5c077386516 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/rle.rs @@ -1,16 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex::VortexSessionDefault; use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; use vortex::array::validity::Validity; use vortex::encodings::fastlanes::RLE; use vortex::error::VortexResult; +use vortex_session::VortexSession; use super::N; use crate::fixtures::FlatLayoutFixture; @@ -31,6 +34,9 @@ impl FlatLayoutFixture for RleFixture { } fn build(&self) -> VortexResult { + let session = VortexSession::default(); + let mut ctx = session.create_execution_ctx(); + let runs_i32: PrimitiveArray = (0..N as i32).map(|i| i / 64).collect(); let single_run: PrimitiveArray = std::iter::repeat_n(42u64, N).collect(); let nullable_runs = PrimitiveArray::from_option_iter( @@ -69,14 +75,14 @@ impl FlatLayoutFixture for RleFixture { "short_runs_u8", ]), vec![ - RLE::encode(runs_i32.as_view())?.into_array(), - RLE::encode(single_run.as_view())?.into_array(), - RLE::encode(nullable_runs.as_view())?.into_array(), - RLE::encode(alternating_singletons.as_view())?.into_array(), - RLE::encode(exact_boundary_runs.as_view())?.into_array(), - RLE::encode(giant_final_run.as_view())?.into_array(), - RLE::encode(all_null_i32.as_view())?.into_array(), - RLE::encode(short_runs_u8.as_view())?.into_array(), + RLE::encode(runs_i32.as_view(), &mut ctx)?.into_array(), + RLE::encode(single_run.as_view(), &mut ctx)?.into_array(), + RLE::encode(nullable_runs.as_view(), &mut ctx)?.into_array(), + RLE::encode(alternating_singletons.as_view(), &mut ctx)?.into_array(), + RLE::encode(exact_boundary_runs.as_view(), &mut ctx)?.into_array(), + RLE::encode(giant_final_run.as_view(), &mut ctx)?.into_array(), + RLE::encode(all_null_i32.as_view(), &mut ctx)?.into_array(), + RLE::encode(short_runs_u8.as_view(), &mut ctx)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs index da81a2b07c4..c14553ea11a 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/runend.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; @@ -34,6 +36,7 @@ impl FlatLayoutFixture for RunEndFixture { } fn build(&self) -> VortexResult { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let run_lengths = [1usize, 5, 10, 50, 100]; let mut values = Vec::with_capacity(N); let mut run_idx = 0i64; @@ -47,8 +50,8 @@ impl FlatLayoutFixture for RunEndFixture { rl_idx += 1; } let run_prim: PrimitiveArray = values.into_iter().collect(); - let (run_ends, run_values) = runend_encode(run_prim.as_view()); - let run_col = RunEnd::try_new(run_ends.into_array(), run_values)?; + let (run_ends, run_values) = runend_encode(run_prim.as_view(), &mut ctx); + let run_col = RunEnd::try_new(run_ends.into_array(), run_values, &mut ctx)?; let statuses = ["open", "closed", "pending", "cancelled"]; let mut status_values = Vec::new(); @@ -68,16 +71,17 @@ impl FlatLayoutFixture for RunEndFixture { let status_col = RunEnd::try_new( status_ends_prim.into_array(), VarBinArray::from_strs(status_values).into_array(), + &mut ctx, )?; let uniform_prim: PrimitiveArray = (0..N as i32).map(|i| i / 64).collect(); - let (uniform_ends, uniform_values) = runend_encode(uniform_prim.as_view()); - let uniform_col = RunEnd::try_new(uniform_ends.into_array(), uniform_values)?; + let (uniform_ends, uniform_values) = runend_encode(uniform_prim.as_view(), &mut ctx); + let uniform_col = RunEnd::try_new(uniform_ends.into_array(), uniform_values, &mut ctx)?; let bool_ends: PrimitiveArray = (1..=N / 32).map(|i| (i * 32) as u16).collect(); let bool_values = BoolArray::from_iter((0..bool_ends.len()).map(|i| i % 2 == 0)).into_array(); - let bool_runs = RunEnd::try_new(bool_ends.into_array(), bool_values)?; + let bool_runs = RunEnd::try_new(bool_ends.into_array(), bool_values, &mut ctx)?; let nullable_run_values = PrimitiveArray::from_option_iter([ Some(10i32), None, @@ -89,15 +93,20 @@ impl FlatLayoutFixture for RunEndFixture { let nullable_runs = RunEnd::try_new( PrimitiveArray::from_iter([16u16, 64, 128, 256, 512, N as u16]).into_array(), nullable_run_values.into_array(), + &mut ctx, )?; let single_run = RunEnd::try_new( PrimitiveArray::from_iter([N as u64]).into_array(), PrimitiveArray::from_iter([1234i64]).into_array(), + &mut ctx, )?; let singleton_values: PrimitiveArray = (0..N as i16).map(|i| i - 512).collect(); let singleton_ends: PrimitiveArray = (1..=N as u16).collect(); - let alternating_singletons = - RunEnd::try_new(singleton_ends.into_array(), singleton_values.into_array())?; + let alternating_singletons = RunEnd::try_new( + singleton_ends.into_array(), + singleton_values.into_array(), + &mut ctx, + )?; let arr = StructArray::try_new( FieldNames::from([ diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs index 724b41abf30..951f9a50a23 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; @@ -36,6 +38,9 @@ impl FlatLayoutFixture for SparseFixture { } fn build(&self) -> VortexResult { + // `FlatLayoutFixture::build` has a fixed trait signature that can't accept a ctx, + // so we instantiate one here. + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let sparse_i64_col = PrimitiveArray::from_option_iter( (0..N as i64).map(|i| (i % 50 == 0).then_some(i * 1000)), ); @@ -89,22 +94,28 @@ impl FlatLayoutFixture for SparseFixture { "mixed_null_and_values", ]), vec![ - Sparse::encode(&sparse_i64_col.into_array(), None)?, - Sparse::encode(&sparse_str_col.into_array(), None)?, - Sparse::encode(&sparse_bool_col.into_array(), None)?, - Sparse::encode(&sparse_f64.into_array(), None)?, - Sparse::encode(&sparse_boundary.into_array(), None)?, + Sparse::encode(&sparse_i64_col.into_array(), None, &mut ctx)?, + Sparse::encode(&sparse_str_col.into_array(), None, &mut ctx)?, + Sparse::encode(&sparse_bool_col.into_array(), None, &mut ctx)?, + Sparse::encode(&sparse_f64.into_array(), None, &mut ctx)?, + Sparse::encode(&sparse_boundary.into_array(), None, &mut ctx)?, Sparse::encode( &explicit_fill_values.into_array(), Some(Scalar::primitive(10i32, Nullability::Nullable)), + &mut ctx, )?, - Sparse::encode(&all_default, Some(Scalar::from(10i32)))?, - Sparse::encode(&clustered_edges.into_array(), None)?, + Sparse::encode(&all_default, Some(Scalar::from(10i32)), &mut ctx)?, + Sparse::encode(&clustered_edges.into_array(), None, &mut ctx)?, Sparse::encode( &almost_dense.into_array(), Some(Scalar::primitive(0i32, Nullability::Nullable)), + &mut ctx, + )?, + Sparse::encode( + &mixed_null_and_values.into_array(), + Some(mixed_null_fill), + &mut ctx, )?, - Sparse::encode(&mixed_null_and_values.into_array(), Some(mixed_null_fill))?, ], N, Validity::NonNullable, diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs index 16736672878..aa55bb78eb9 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; @@ -32,6 +34,7 @@ impl FlatLayoutFixture for ZstdFixture { } fn build(&self) -> VortexResult { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let ints: PrimitiveArray = (0..N as i32).map(|i| i / 8).collect(); let floats: PrimitiveArray = (0..N) .map(|i| { @@ -87,14 +90,14 @@ impl FlatLayoutFixture for ZstdFixture { "pseudo_random", ]), vec![ - Zstd::from_primitive(&ints, 3, 128)?.into_array(), - Zstd::from_primitive(&floats, 3, 128)?.into_array(), - Zstd::from_primitive(&nullable_i64, 3, 128)?.into_array(), - Zstd::from_var_bin_view(&utf8, 3, 128)?.into_array(), - Zstd::from_var_bin_view(&nullable_utf8, 3, 128)?.into_array(), - Zstd::from_primitive(&all_zeros, 3, 128)?.into_array(), - Zstd::from_primitive(&all_null_i32, 3, 128)?.into_array(), - Zstd::from_primitive(&pseudo_random, 3, 128)?.into_array(), + Zstd::from_primitive(&ints, 3, 128, &mut ctx)?.into_array(), + Zstd::from_primitive(&floats, 3, 128, &mut ctx)?.into_array(), + Zstd::from_primitive(&nullable_i64, 3, 128, &mut ctx)?.into_array(), + Zstd::from_var_bin_view(&utf8, 3, 128, &mut ctx)?.into_array(), + Zstd::from_var_bin_view(&nullable_utf8, 3, 128, &mut ctx)?.into_array(), + Zstd::from_primitive(&all_zeros, 3, 128, &mut ctx)?.into_array(), + Zstd::from_primitive(&all_null_i32, 3, 128, &mut ctx)?.into_array(), + Zstd::from_primitive(&pseudo_random, 3, 128, &mut ctx)?.into_array(), ], N, Validity::NonNullable, diff --git a/vortex-tui/src/browse/ui/layouts.rs b/vortex-tui/src/browse/ui/layouts.rs index 845a5988c78..c3245446197 100644 --- a/vortex-tui/src/browse/ui/layouts.rs +++ b/vortex-tui/src/browse/ui/layouts.rs @@ -27,10 +27,8 @@ use ratatui::widgets::Table; use ratatui::widgets::Widget; use ratatui::widgets::Wrap; use vortex::array::ArrayRef; -use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; +use vortex::array::arrays::StructArray; use vortex::array::arrays::struct_::StructArrayExt; use vortex::error::VortexExpect; use vortex::layout::layouts::flat::Flat; @@ -143,8 +141,11 @@ fn render_array(app: &AppState, area: Rect, buf: &mut Buffer, is_stats_table: bo if is_stats_table { // Render the stats table horizontally - #[expect(deprecated)] - let struct_array = array.to_struct(); + let mut ctx = app.session.create_execution_ctx(); + let struct_array = array + .clone() + .execute::(&mut ctx) + .vortex_expect("failed to canonicalize stats array to StructArray"); // add 1 for the chunk column let field_count = struct_array.struct_fields().nfields() + 1; let header = std::iter::once("chunk") @@ -163,17 +164,18 @@ fn render_array(app: &AppState, area: Rect, buf: &mut Buffer, is_stats_table: bo let field_arrays: Vec = struct_array.unmasked_fields().to_vec(); // TODO: trim the number of displayed rows and allow paging through column stats. - let rows = (0..array.len()).map(|chunk_id| { - std::iter::once(Cell::from(Text::from(format!("{chunk_id}")))) - .chain(field_arrays.iter().map(|arr| { - Cell::from(Text::from( - arr.execute_scalar(chunk_id, &mut LEGACY_SESSION.create_execution_ctx()) - .vortex_expect("scalar_at failed") - .to_string(), - )) - })) - .collect::() - }); + let mut rows = Vec::with_capacity(array.len()); + for chunk_id in 0..array.len() { + let mut cells: Vec = Vec::with_capacity(field_count); + cells.push(Cell::from(Text::from(format!("{chunk_id}")))); + for arr in &field_arrays { + let scalar = arr + .execute_scalar(chunk_id, &mut ctx) + .vortex_expect("scalar_at failed"); + cells.push(Cell::from(Text::from(scalar.to_string()))); + } + rows.push(cells.into_iter().collect::()); + } Widget::render( Table::new(rows, (0..field_count).map(|_| Constraint::Min(6))).header(header), diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 9a96c478e5c..552f66c321f 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -16,8 +16,6 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; @@ -74,32 +72,34 @@ mod setup { use super::*; fn setup_primitive_arrays() -> (PrimitiveArray, PrimitiveArray, PrimitiveArray) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut rng = StdRng::seed_from_u64(0); let uint_array = PrimitiveArray::from_iter((0..NUM_VALUES).map(|_| rng.random_range(42u32..256))); - #[expect(deprecated)] let int_array = uint_array .clone() .into_array() .cast(PType::I32.into()) .unwrap() - .to_primitive(); - #[expect(deprecated)] + .execute::(&mut ctx) + .unwrap(); let float_array = uint_array .clone() .into_array() .cast(PType::F64.into()) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); (uint_array, int_array, float_array) } /// Create FoR <- BitPacked encoding tree for u64 pub fn for_bp_u64() -> ArrayRef { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let (uint_array, ..) = setup_primitive_arrays(); let compressed = FoR::encode(uint_array).unwrap(); let inner = compressed.encoded(); - let bp = BitPacked::encode(inner, 8).unwrap(); + let bp = BitPacked::encode(inner, 8, &mut ctx).unwrap(); FoR::try_new(bp.into_array(), compressed.reference_scalar().clone()) .unwrap() .into_array() @@ -107,20 +107,19 @@ mod setup { /// Create ALP <- FoR <- BitPacked encoding tree for f64 pub fn alp_for_bp_f64() -> ArrayRef { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let (_, _, float_array) = setup_primitive_arrays(); - let alp_compressed = alp_encode( - float_array.as_view(), - None, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let alp_compressed = alp_encode(float_array.as_view(), None, &mut ctx).unwrap(); // Manually construct ALP <- FoR <- BitPacked tree - #[expect(deprecated)] - let alp_encoded_prim = alp_compressed.encoded().to_primitive(); + let alp_encoded_prim = alp_compressed + .encoded() + .clone() + .execute::(&mut ctx) + .unwrap(); let for_array = FoR::encode(alp_encoded_prim).unwrap(); let inner = for_array.encoded(); - let bp = BitPacked::encode(inner, 8).unwrap(); + let bp = BitPacked::encode(inner, 8, &mut ctx).unwrap(); let for_with_bp = FoR::try_new(bp.into_array(), for_array.reference_scalar().clone()).unwrap(); @@ -155,7 +154,8 @@ mod setup { let codes_prim = PrimitiveArray::from_iter(codes); // Compress codes with BitPacked (6 bits should be enough for ~50 unique values) - let codes_bp = BitPacked::encode(&codes_prim.into_array(), 6) + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let codes_bp = BitPacked::encode(&codes_prim.into_array(), 6, &mut ctx) .unwrap() .into_array(); @@ -185,28 +185,35 @@ mod setup { run_length -= 1; } + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let prim_array = PrimitiveArray::from_iter(values); - let runend = RunEnd::encode(prim_array.into_array()).unwrap(); + let runend = RunEnd::encode(prim_array.into_array(), &mut ctx).unwrap(); // Compress the ends with FoR <- BitPacked - #[expect(deprecated)] - let ends_prim = runend.ends().to_primitive(); + let ends_prim = runend + .ends() + .clone() + .execute::(&mut ctx) + .unwrap(); let ends_for = FoR::encode(ends_prim).unwrap(); let ends_inner = ends_for.encoded(); - let ends_bp = BitPacked::encode(ends_inner, 8).unwrap(); + let ends_bp = BitPacked::encode(ends_inner, 8, &mut ctx).unwrap(); let compressed_ends = FoR::try_new(ends_bp.into_array(), ends_for.reference_scalar().clone()) .unwrap() .into_array(); // Compress the values with BitPacked - #[expect(deprecated)] - let values_prim = runend.values().to_primitive(); - let compressed_values = BitPacked::encode(&values_prim.into_array(), 8) + let values_prim = runend + .values() + .clone() + .execute::(&mut ctx) + .unwrap(); + let compressed_values = BitPacked::encode(&values_prim.into_array(), 8, &mut ctx) .unwrap() .into_array(); - RunEnd::try_new(compressed_ends, compressed_values) + RunEnd::try_new(compressed_ends, compressed_values, &mut ctx) .unwrap() .into_array() } @@ -227,6 +234,7 @@ mod setup { .collect(); // Train and compress unique values with FSST + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unique_varbinview = VarBinViewArray::from_iter_str(unique_strings); let fsst_compressor = fsst_train_compressor(&unique_varbinview); let fsst_values = fsst_compress( @@ -234,6 +242,7 @@ mod setup { unique_varbinview.len(), unique_varbinview.dtype(), &fsst_compressor, + &mut ctx, ); // Create codes array (random indices into unique values) @@ -264,6 +273,7 @@ mod setup { .collect(); // Train and compress unique values with FSST + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let unique_varbinview = VarBinViewArray::from_iter_str(unique_strings); let fsst_compressor = fsst_train_compressor(&unique_varbinview); let fsst = fsst_compress( @@ -271,13 +281,17 @@ mod setup { unique_varbinview.len(), unique_varbinview.dtype(), &fsst_compressor, + &mut ctx, ); // Compress the VarBin offsets with BitPacked let codes = fsst.codes(); - #[expect(deprecated)] - let offsets_prim = codes.offsets().to_primitive(); - let offsets_bp = BitPacked::encode(&offsets_prim.into_array(), 20).unwrap(); + let offsets_prim = codes + .offsets() + .clone() + .execute::(&mut ctx) + .unwrap(); + let offsets_bp = BitPacked::encode(&offsets_prim.into_array(), 20, &mut ctx).unwrap(); // Rebuild VarBin with compressed offsets let compressed_codes = VarBinArray::try_new( @@ -297,6 +311,7 @@ mod setup { fsst.symbol_lengths().clone(), compressed_codes, fsst.uncompressed_lengths().clone(), + &mut ctx, ) .unwrap(); @@ -326,25 +341,32 @@ mod setup { let temporal_array = TemporalArray::new_timestamp(ts_array, TimeUnit::Microseconds, None); // Split into days, seconds, subseconds - let parts = split_temporal(temporal_array.clone()).unwrap(); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let parts = split_temporal(temporal_array.clone(), &mut ctx).unwrap(); // Compress days with FoR <- BitPacked - #[expect(deprecated)] - let days_prim = parts.days.to_primitive(); + let days_prim = parts + .days + .clone() + .execute::(&mut ctx) + .unwrap(); let days_for = FoR::encode(days_prim).unwrap(); let days_inner = days_for.encoded(); - let days_bp = BitPacked::encode(days_inner, 16).unwrap(); + let days_bp = BitPacked::encode(days_inner, 16, &mut ctx).unwrap(); let compressed_days = FoR::try_new(days_bp.into_array(), days_for.reference_scalar().clone()) .unwrap() .into_array(); // Compress seconds with FoR <- BitPacked - #[expect(deprecated)] - let seconds_prim = parts.seconds.to_primitive(); + let seconds_prim = parts + .seconds + .clone() + .execute::(&mut ctx) + .unwrap(); let seconds_for = FoR::encode(seconds_prim).unwrap(); let seconds_inner = seconds_for.encoded(); - let seconds_bp = BitPacked::encode(seconds_inner, 17).unwrap(); + let seconds_bp = BitPacked::encode(seconds_inner, 17, &mut ctx).unwrap(); let compressed_seconds = FoR::try_new( seconds_bp.into_array(), seconds_for.reference_scalar().clone(), @@ -353,11 +375,13 @@ mod setup { .into_array(); // Compress subseconds with FoR <- BitPacked - #[expect(deprecated)] - let subseconds_prim = parts.subseconds.to_primitive(); + let subseconds_prim = parts + .subseconds + .execute::(&mut ctx) + .unwrap(); let subseconds_for = FoR::encode(subseconds_prim).unwrap(); let subseconds_inner = subseconds_for.encoded(); - let subseconds_bp = BitPacked::encode(subseconds_inner, 20).unwrap(); + let subseconds_bp = BitPacked::encode(subseconds_inner, 20, &mut ctx).unwrap(); let compressed_subseconds = FoR::try_new( subseconds_bp.into_array(), subseconds_for.reference_scalar().clone(), diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index e74301cea9f..e0caed0d84f 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -18,8 +18,6 @@ use vortex::array::Canonical; use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex::array::ToCanonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::builders::dict::dict_encode; @@ -75,23 +73,24 @@ fn canonicalize(array: impl IntoArray, ctx: &mut ExecutionCtx) -> VortexResult (PrimitiveArray, PrimitiveArray, PrimitiveArray) { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut rng = StdRng::seed_from_u64(0); let uint_array = PrimitiveArray::from_iter((0..NUM_VALUES).map(|_| rng.random_range(42u32..256))); - #[expect(deprecated)] let int_array = uint_array .clone() .into_array() .cast(PType::I32.into()) .unwrap() - .to_primitive(); - #[expect(deprecated)] + .execute::(&mut ctx) + .unwrap(); let float_array = uint_array .clone() .into_array() .cast(PType::F64.into()) .unwrap() - .to_primitive(); + .execute::(&mut ctx) + .unwrap(); (uint_array, int_array, float_array) } @@ -130,9 +129,14 @@ fn bench_bitpacked_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); let bit_width = 8; - let compressed = bitpack_encode(&uint_array, bit_width, None) - .unwrap() - .into_array(); + let compressed = bitpack_encode( + &uint_array, + bit_width, + None, + &mut SESSION.create_execution_ctx(), + ) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) @@ -144,14 +148,18 @@ fn bench_runend_compress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| uint_array.clone()) - .bench_values(|a| RunEnd::encode(a.into_array()).unwrap()); + .with_inputs(|| (uint_array.clone(), LEGACY_SESSION.create_execution_ctx())) + .bench_values(|(a, mut ctx)| RunEnd::encode(a.into_array(), &mut ctx).unwrap()); } #[divan::bench(name = "runend_decompress_u32")] fn bench_runend_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); - let compressed = RunEnd::encode(uint_array.into_array()).unwrap(); + let compressed = RunEnd::encode( + uint_array.into_array(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) @@ -246,8 +254,8 @@ fn bench_sequence_compress_u32(bencher: Bencher) { let seq_array = PrimitiveArray::from_iter(0..NUM_VALUES as u32); with_byte_counter(bencher, NUM_VALUES * 4) - .with_inputs(|| seq_array.clone()) - .bench_values(|a| sequence_encode(a.as_view()).unwrap().unwrap()); + .with_inputs(|| (seq_array.clone(), SESSION.create_execution_ctx())) + .bench_values(|(a, mut ctx)| sequence_encode(a.as_view(), &mut ctx).unwrap().unwrap()); } #[expect(clippy::cast_possible_truncation)] @@ -298,10 +306,10 @@ fn bench_alp_rd_compress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); with_byte_counter(bencher, NUM_VALUES * 8) - .with_inputs(|| &float_array) - .bench_refs(|a| { + .with_inputs(|| (&float_array, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| { let encoder = RDEncoder::new(a.as_slice::()); - encoder.encode(a.as_view()) + encoder.encode(a.as_view(), ctx) }); } @@ -309,7 +317,7 @@ fn bench_alp_rd_compress_f64(bencher: Bencher) { fn bench_alp_rd_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); let encoder = RDEncoder::new(float_array.as_slice::()); - let compressed = encoder.encode(float_array.as_view()); + let compressed = encoder.encode(float_array.as_view(), &mut SESSION.create_execution_ctx()); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) @@ -321,14 +329,20 @@ fn bench_pcodec_compress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); with_byte_counter(bencher, NUM_VALUES * 8) - .with_inputs(|| &float_array) - .bench_refs(|a| Pco::from_primitive(a.as_view(), 3, 0).unwrap()); + .with_inputs(|| (&float_array, SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| Pco::from_primitive(a.as_view(), 3, 0, ctx).unwrap()); } #[divan::bench(name = "pcodec_decompress_f64")] fn bench_pcodec_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); - let compressed = Pco::from_primitive(float_array.as_view(), 3, 0).unwrap(); + let compressed = Pco::from_primitive( + float_array.as_view(), + 3, + 0, + &mut SESSION.create_execution_ctx(), + ) + .unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| (&compressed, SESSION.create_execution_ctx())) @@ -343,7 +357,9 @@ fn bench_zstd_compress_u32(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| array.clone()) - .bench_values(|a| ZstdData::from_array(a, 3, 8192).unwrap()); + .bench_values(|a| { + ZstdData::from_array(a, 3, 8192, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() + }); } #[cfg(feature = "zstd")] @@ -354,7 +370,13 @@ fn bench_zstd_decompress_u32(bencher: Bencher) { let validity = uint_array.validity().unwrap(); let compressed = Zstd::try_new( dtype, - ZstdData::from_array(uint_array.into_array(), 3, 8192).unwrap(), + ZstdData::from_array( + uint_array.into_array(), + 3, + 8192, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(), validity, ) .unwrap() @@ -397,8 +419,8 @@ fn bench_fsst_compress_string(bencher: Bencher) { let nbytes = varbinview_arr.nbytes() as u64; with_byte_counter(bencher, nbytes) - .with_inputs(|| &varbinview_arr) - .bench_refs(|a| fsst_compress(*a, a.len(), a.dtype(), &fsst_compressor)); + .with_inputs(|| (&varbinview_arr, LEGACY_SESSION.create_execution_ctx())) + .bench_refs(|(a, ctx)| fsst_compress(*a, a.len(), a.dtype(), &fsst_compressor, ctx)); } #[divan::bench(name = "fsst_decompress_string")] @@ -411,6 +433,7 @@ fn bench_fsst_decompress_string(bencher: Bencher) { varbinview_arr.len(), varbinview_arr.dtype(), &fsst_compressor, + &mut LEGACY_SESSION.create_execution_ctx(), ); let nbytes = varbinview_arr.into_array().nbytes() as u64; @@ -429,7 +452,9 @@ fn bench_zstd_compress_string(bencher: Bencher) { with_byte_counter(bencher, nbytes) .with_inputs(|| array.clone()) - .bench_values(|a| ZstdData::from_array(a, 3, 8192).unwrap()); + .bench_values(|a| { + ZstdData::from_array(a, 3, 8192, &mut LEGACY_SESSION.create_execution_ctx()).unwrap() + }); } #[cfg(feature = "zstd")] @@ -441,7 +466,13 @@ fn bench_zstd_decompress_string(bencher: Bencher) { let validity = varbinview_arr.validity().unwrap(); let compressed = Zstd::try_new( dtype, - ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192).unwrap(), + ZstdData::from_array( + varbinview_arr.clone().into_array(), + 3, + 8192, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(), validity, ) .unwrap() From 09601afa16cb6c0513514920a5112fe25f4fd4bd Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Mon, 20 Apr 2026 13:56:47 -0400 Subject: [PATCH 143/250] avoid including experimental encoding in footer (#7569) ## Summary We have been publishing a footer reference to `vortex.patched` experimental encoding in the last few releases. The issue is that we should not be including the Patched encoding in the default array context This removes Patched from the default array context, instead registering it conditionally where we register the other unstable encodings. I've verified that after this change writing a file with default settings no longer includes Patched encoding image --------- Signed-off-by: Andrew Duffy --- vortex-array/src/arrays/patched/vtable/mod.rs | 5 ++++- vortex-array/src/session/mod.rs | 2 -- vortex-file/src/lib.rs | 5 +++++ vortex-file/src/strategy.rs | 10 ++++++---- vortex-file/src/writer.rs | 3 ++- vortex-python/src/io.rs | 4 ++-- .../src/fixtures/arrays/synthetic/encodings/mod.rs | 3 ++- .../src/fixtures/arrays/synthetic/encodings/patched.rs | 4 ++++ 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 5a1cdef3edf..ad2fc9aadc7 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -373,6 +373,7 @@ mod tests { use crate::patches::Patches; use crate::serde::SerializeOptions; use crate::serde::SerializedArray; + use crate::session::ArraySessionExt; use crate::validity::Validity; #[test] @@ -588,7 +589,9 @@ mod tests { let dtype = array.dtype().clone(); let len = array.len(); - let ctx = ArrayContext::empty(); + LEGACY_SESSION.arrays().register(Patched); + + let ctx = ArrayContext::empty().with_registry(LEGACY_SESSION.arrays().registry().clone()); let serialized = array .serialize(&ctx, &LEGACY_SESSION, &SerializeOptions::default()) .unwrap(); diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index 034a369694f..4b30257d981 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -23,7 +23,6 @@ use crate::arrays::List; use crate::arrays::ListView; use crate::arrays::Masked; use crate::arrays::Null; -use crate::arrays::Patched; use crate::arrays::Primitive; use crate::arrays::Struct; use crate::arrays::VarBin; @@ -80,7 +79,6 @@ impl Default for ArraySession { this.register(Dict); this.register(List); this.register(Masked); - this.register(Patched); this.register(VarBin); this diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 3cc3b6d0858..ce6598173a6 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -110,6 +110,8 @@ pub use forever_constant::*; pub use open::*; pub use strategy::*; use vortex_array::arrays::Dict; +use vortex_array::arrays::Patched; +use vortex_array::arrays::patched::use_experimental_patches; use vortex_array::session::ArraySessionExt; use vortex_bytebool::ByteBool; use vortex_fsst::FSST; @@ -168,6 +170,9 @@ pub fn register_default_encodings(session: &VortexSession) { arrays.register(vortex_zstd::Zstd); #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] arrays.register(vortex_zstd::ZstdBuffers); + if use_experimental_patches() { + arrays.register(Patched); + } } // Eventually all encodings crates should expose an initialize function. For now it's only diff --git a/vortex-file/src/strategy.rs b/vortex-file/src/strategy.rs index 41d9e3c2ad2..71c72ffc904 100644 --- a/vortex-file/src/strategy.rs +++ b/vortex-file/src/strategy.rs @@ -90,10 +90,6 @@ pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { allowed.insert(Masked.id()); allowed.insert(Dict.id()); - if use_experimental_patches() { - allowed.insert(Patched.id()); - } - // Compressed encodings from encoding crates allowed.insert(ALP.id()); allowed.insert(ALPRD.id()); @@ -111,6 +107,12 @@ pub static ALLOWED_ENCODINGS: LazyLock> = LazyLock::new(|| { allowed.insert(Sparse.id()); allowed.insert(ZigZag.id()); + // Experimental encodings + + if use_experimental_patches() { + allowed.insert(Patched.id()); + } + #[cfg(feature = "zstd")] allowed.insert(Zstd.id()); #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] diff --git a/vortex-file/src/writer.rs b/vortex-file/src/writer.rs index cd8b1913482..82720c25b29 100644 --- a/vortex-file/src/writer.rs +++ b/vortex-file/src/writer.rs @@ -47,6 +47,7 @@ use vortex_session::SessionExt; use vortex_session::VortexSession; use vortex_session::registry::ReadContext; +use crate::ALLOWED_ENCODINGS; use crate::Footer; use crate::MAGIC_BYTES; use crate::WriteStrategyBuilder; @@ -147,7 +148,7 @@ impl VortexWriteOptions { // serialised array order is deterministic. The serialisation of arrays are done // parallel and with an empty context they can register their encodings to the context // in different order, changing the written bytes from run to run. - let ctx = ArrayContext::new(self.session.arrays().registry().ids().sorted().collect()) + let ctx = ArrayContext::new(ALLOWED_ENCODINGS.iter().cloned().sorted().collect()) // Configure a registry just to ensure only known encodings are interned. .with_registry(self.session.arrays().registry().clone()); let dtype = stream.dtype().clone(); diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index 1d1cb63cce7..81bfdc750f6 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -280,7 +280,7 @@ impl PyVortexWriteOptions { /// >>> vx.io.VortexWriteOptions.default().write(sprl, "chonky.vortex") /// >>> import os /// >>> os.path.getsize('chonky.vortex') - /// 216036 + /// 215972 /// ``` /// /// Wow, Vortex manages to use about two bytes per integer! So advanced. So tiny. @@ -292,7 +292,7 @@ impl PyVortexWriteOptions { /// ```python /// >>> vx.io.VortexWriteOptions.compact().write(sprl, "tiny.vortex") /// >>> os.path.getsize('tiny.vortex') - /// 55152 + /// 55088 /// ``` /// /// Random numbers are not (usually) composed of random bytes! diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/mod.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/mod.rs index b6c18064e0f..830b50450da 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/mod.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/mod.rs @@ -43,7 +43,8 @@ pub fn fixtures() -> Vec> { Box::new(dict::DictFixture), Box::new(fsst::FsstFixture), Box::new(for_::FoRFixture), - Box::new(patched::PatchedFixture), + // TODO(aduffy): add back once we stabilized Patched array + // Box::new(patched::PatchedFixture), Box::new(pco::PcoFixture), Box::new(rle::RleFixture), Box::new(runend::RunEndFixture), diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs index b9a43dfb1e2..fea0c990173 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/patched.rs @@ -14,6 +14,10 @@ use vortex_session::VortexSession; use crate::fixtures::FlatLayoutFixture; +#[expect( + dead_code, + reason = "This will be unused until we stabilize Patched array" +)] pub struct PatchedFixture; impl FlatLayoutFixture for PatchedFixture { From 36c134c378b919b4868b3249cf4452c348c2c1ef Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 20 Apr 2026 14:10:47 -0400 Subject: [PATCH 144/250] Implement validity for Between scalar function (#7519) Avoid using IsNotNull(expr) fallback which can lead to infinite recursion. As a followup we should make validity required thing for scalar functions --------- Signed-off-by: Robert Kruszewski Co-authored-by: Joe Isaacs --- vortex-array/public-api.lock | 4 ++-- vortex-array/src/scalar_fn/fns/between/mod.rs | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 859ee6829e2..933d1ea2310 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -15504,7 +15504,7 @@ pub fn vortex_array::scalar_fn::fns::between::Between::stat_expression(&self, op pub fn vortex_array::scalar_fn::fns::between::Between::stat_falsification(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option -pub fn vortex_array::scalar_fn::fns::between::Between::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> +pub fn vortex_array::scalar_fn::fns::between::Between::validity(&self, _options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> pub struct vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor(pub V) @@ -17738,7 +17738,7 @@ pub fn vortex_array::scalar_fn::fns::between::Between::stat_expression(&self, op pub fn vortex_array::scalar_fn::fns::between::Between::stat_falsification(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option -pub fn vortex_array::scalar_fn::fns::between::Between::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> +pub fn vortex_array::scalar_fn::fns::between::Between::validity(&self, _options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::binary::Binary diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index 127985af93a..d23c5363366 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -8,6 +8,7 @@ use std::fmt::Formatter; pub use kernel::*; use prost::Message; +use vortex_array::expr::and; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_proto::expr as pb; @@ -308,9 +309,18 @@ impl ScalarFnVTable for Between { let lhs = Binary.new_expr(options.lower_strict.to_operator(), [lower, arr.clone()]); let rhs = Binary.new_expr(options.upper_strict.to_operator(), [arr, upper]); - Binary - .new_expr(Operator::And, [lhs, rhs]) - .stat_falsification(catalog) + and(lhs, rhs).stat_falsification(catalog) + } + + fn validity( + &self, + _options: &Self::Options, + expression: &Expression, + ) -> VortexResult> { + let arr = expression.child(0).validity()?; + let lower = expression.child(1).validity()?; + let upper = expression.child(2).validity()?; + Ok(Some(and(and(arr, lower), upper))) } fn is_null_sensitive(&self, _instance: &Self::Options) -> bool { From b213af14664d3b85518b493a0df6aec8254d470e Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:21:36 -0400 Subject: [PATCH 145/250] Move tensor types under `types/` submodule (#7573) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 Moves the logical types in `vortex-tensor` into a `types` submodule. We will be adding more types soon, so it makes sense to group them. There are no API changes as we re-export through this module. ## Testing N/A as this is a purely cosmetic change. Signed-off-by: Connor Tsui --- vortex-tensor/src/encodings/turboquant/compress.rs | 4 ++-- vortex-tensor/src/encodings/turboquant/mod.rs | 4 ++-- vortex-tensor/src/encodings/turboquant/tests/mod.rs | 2 +- .../src/encodings/turboquant/tests/structural.rs | 2 +- vortex-tensor/src/lib.rs | 10 ++++++---- vortex-tensor/src/matcher.rs | 8 ++++---- vortex-tensor/src/scalar_fns/cosine_similarity.rs | 2 +- vortex-tensor/src/scalar_fns/inner_product.rs | 4 ++-- vortex-tensor/src/scalar_fns/l2_denorm.rs | 2 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 2 +- vortex-tensor/src/scalar_fns/sorf_transform/tests.rs | 7 +++++-- vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs | 4 ++-- vortex-tensor/src/{ => types}/fixed_shape/matcher.rs | 6 +++--- vortex-tensor/src/{ => types}/fixed_shape/metadata.rs | 0 vortex-tensor/src/{ => types}/fixed_shape/mod.rs | 0 vortex-tensor/src/{ => types}/fixed_shape/proto.rs | 2 +- vortex-tensor/src/{ => types}/fixed_shape/vtable.rs | 10 +++++----- vortex-tensor/src/types/mod.rs | 7 +++++++ vortex-tensor/src/{ => types}/vector/matcher.rs | 6 +++--- vortex-tensor/src/{ => types}/vector/mod.rs | 0 vortex-tensor/src/{ => types}/vector/vtable.rs | 4 ++-- vortex-tensor/src/utils.rs | 6 +++--- vortex-tensor/src/vector_search.rs | 2 +- 23 files changed, 53 insertions(+), 41 deletions(-) rename vortex-tensor/src/{ => types}/fixed_shape/matcher.rs (96%) rename vortex-tensor/src/{ => types}/fixed_shape/metadata.rs (100%) rename vortex-tensor/src/{ => types}/fixed_shape/mod.rs (100%) rename vortex-tensor/src/{ => types}/fixed_shape/proto.rs (98%) rename vortex-tensor/src/{ => types}/fixed_shape/vtable.rs (94%) create mode 100644 vortex-tensor/src/types/mod.rs rename vortex-tensor/src/{ => types}/vector/matcher.rs (96%) rename vortex-tensor/src/{ => types}/vector/mod.rs (100%) rename vortex-tensor/src/{ => types}/vector/vtable.rs (98%) diff --git a/vortex-tensor/src/encodings/turboquant/compress.rs b/vortex-tensor/src/encodings/turboquant/compress.rs index 40e24807091..58c5666525a 100644 --- a/vortex-tensor/src/encodings/turboquant/compress.rs +++ b/vortex-tensor/src/encodings/turboquant/compress.rs @@ -39,9 +39,9 @@ use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; +use crate::types::vector::AnyVector; +use crate::types::vector::Vector; use crate::utils::cast_to_f32; -use crate::vector::AnyVector; -use crate::vector::Vector; /// Configuration for TurboQuant encoding. #[derive(Clone, Debug)] diff --git a/vortex-tensor/src/encodings/turboquant/mod.rs b/vortex-tensor/src/encodings/turboquant/mod.rs index 51480955594..463e8bc7b25 100644 --- a/vortex-tensor/src/encodings/turboquant/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/mod.rs @@ -154,8 +154,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use crate::vector::AnyVector; -use crate::vector::VectorMatcherMetadata; +use crate::types::vector::AnyVector; +use crate::types::vector::VectorMatcherMetadata; /// Validates that `dtype` is a [`Vector`](crate::vector::Vector) extension type with /// dimension >= [`MIN_DIMENSION`]. diff --git a/vortex-tensor/src/encodings/turboquant/tests/mod.rs b/vortex-tensor/src/encodings/turboquant/tests/mod.rs index 5b7e325b9b7..7dfe53c444a 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/mod.rs @@ -32,7 +32,7 @@ use vortex_error::VortexResult; use crate::encodings::turboquant::TurboQuantConfig; use crate::encodings::turboquant::turboquant_encode; use crate::tests::SESSION; -use crate::vector::Vector; +use crate::types::vector::Vector; /// Create a FixedSizeListArray of random f32 vectors with the given validity. fn make_fsl_with_validity( diff --git a/vortex-tensor/src/encodings/turboquant/tests/structural.rs b/vortex-tensor/src/encodings/turboquant/tests/structural.rs index 7cd2cdfcf66..193d340285a 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/structural.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/structural.rs @@ -236,7 +236,7 @@ fn sorf_transform_roundtrip_isolation() -> VortexResult<()> { use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; - use crate::vector::Vector; + use crate::types::vector::Vector; let dim = 128usize; let seed = 99u64; diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index 68269407473..c748bdd9f43 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -11,19 +11,21 @@ use vortex_array::scalar_fn::session::ScalarFnSessionExt; use vortex_array::session::ArraySessionExt; use vortex_session::VortexSession; -use crate::fixed_shape::FixedShapeTensor; use crate::scalar_fns::cosine_similarity::CosineSimilarity; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_norm::L2Norm; use crate::scalar_fns::sorf_transform::SorfTransform; -use crate::vector::Vector; +use crate::types::fixed_shape::FixedShapeTensor; +use crate::types::vector::Vector; pub mod matcher; pub mod scalar_fns; -pub mod fixed_shape; -pub mod vector; +mod types; + +pub use types::fixed_shape; +pub use types::vector; pub mod encodings; diff --git a/vortex-tensor/src/matcher.rs b/vortex-tensor/src/matcher.rs index 9ff58d17c4f..4566dcb3a38 100644 --- a/vortex-tensor/src/matcher.rs +++ b/vortex-tensor/src/matcher.rs @@ -7,10 +7,10 @@ use vortex_array::dtype::PType; use vortex_array::dtype::extension::ExtDTypeRef; use vortex_array::dtype::extension::Matcher; -use crate::fixed_shape::AnyFixedShapeTensor; -use crate::fixed_shape::FixedShapeTensorMatcherMetadata; -use crate::vector::AnyVector; -use crate::vector::VectorMatcherMetadata; +use crate::types::fixed_shape::AnyFixedShapeTensor; +use crate::types::fixed_shape::FixedShapeTensorMatcherMetadata; +use crate::types::vector::AnyVector; +use crate::types::vector::VectorMatcherMetadata; /// Matcher for any tensor-like extension type. /// diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 85d16236c8c..9f1cd39fa1a 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -327,12 +327,12 @@ mod tests { use crate::scalar_fns::cosine_similarity::CosineSimilarity; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::tests::SESSION; + use crate::types::vector::Vector; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; use crate::utils::test_helpers::l2_denorm_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - use crate::vector::Vector; /// Evaluates cosine similarity between two tensor arrays and returns the result as `Vec`. fn eval_cosine_similarity(lhs: ArrayRef, rhs: ArrayRef, len: usize) -> VortexResult> { diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 22b28f9e4d5..f001c74498e 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -55,11 +55,11 @@ use crate::matcher::AnyTensor; use crate::scalar_fns::l2_denorm::DenormOrientation; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfTransform; +use crate::types::vector::Vector; use crate::utils::extract_constant_flat_row; use crate::utils::extract_flat_elements; use crate::utils::extract_l2_denorm_children; use crate::utils::validate_binary_tensor_float_inputs; -use crate::vector::Vector; /// Inner product (dot product) between two columns. /// @@ -906,10 +906,10 @@ mod tests { use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; use crate::tests::SESSION; + use crate::types::vector::Vector; use crate::utils::extract_flat_elements; use crate::utils::test_helpers::literal_vector_array; use crate::utils::test_helpers::vector_array; - use crate::vector::Vector; /// Build a `Vector` whose storage is `FSL(DictArray(codes: u8, values: /// f32))`. This mirrors the shape that TurboQuant produces as the SorfTransform child. diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 875d7f57e2f..8474216156a 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -767,11 +767,11 @@ mod tests { use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::l2_denorm::validate_l2_normalized_rows_against_norms; use crate::tests::SESSION; + use crate::types::vector::Vector; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::constant_tensor_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - use crate::vector::Vector; /// Evaluates L2 denorm on a tensor/vector array and returns the executed array. fn eval_l2_denorm(normalized: ArrayRef, norms: ArrayRef, len: usize) -> VortexResult { diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 47ba68a35c4..d9341b282db 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -286,11 +286,11 @@ mod tests { use crate::scalar_fns::l2_norm::L2Norm; use crate::tests::SESSION; + use crate::types::vector::Vector; use crate::utils::test_helpers::assert_close; use crate::utils::test_helpers::literal_vector_array; use crate::utils::test_helpers::tensor_array; use crate::utils::test_helpers::vector_array; - use crate::vector::Vector; /// Evaluates L2 norm on a tensor/vector array and returns the result as `Vec`. fn eval_l2_norm(input: ArrayRef, len: usize) -> VortexResult> { diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs index 64308c39562..4bc93871594 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs @@ -36,7 +36,7 @@ use crate::encodings::turboquant::centroids::compute_centroid_boundaries; use crate::encodings::turboquant::centroids::find_nearest_centroid; use crate::encodings::turboquant::centroids::get_centroids; use crate::tests::SESSION; -use crate::vector::Vector; +use crate::types::vector::Vector; /// Build a unit-normalized input vector array and forward-transform + quantize it, returning /// `(input_f32, Vector(FSL(Dict(codes, centroids))), padded_dim)`. @@ -300,7 +300,10 @@ fn return_dtype_is_vector_extension() -> VortexResult<()> { let ext = return_dtype .as_extension_opt() .expect("return dtype should be an extension type"); - assert!(ext.metadata_opt::().is_some()); + assert!( + ext.metadata_opt::() + .is_some() + ); // Inner FSL should have the original (unpadded) dimension. let DType::FixedSizeList(_, inner_dim, _) = ext.storage_dtype() else { diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs index b54158a753f..81e17f665fa 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -47,8 +47,8 @@ use super::SorfOptions; use super::SorfTransform; use super::rotation::SorfMatrix; use super::validate_sorf_options; -use crate::vector::AnyVector; -use crate::vector::Vector; +use crate::types::vector::AnyVector; +use crate::types::vector::Vector; impl ScalarFnVTable for SorfTransform { type Options = SorfOptions; diff --git a/vortex-tensor/src/fixed_shape/matcher.rs b/vortex-tensor/src/types/fixed_shape/matcher.rs similarity index 96% rename from vortex-tensor/src/fixed_shape/matcher.rs rename to vortex-tensor/src/types/fixed_shape/matcher.rs index f703ccba978..32c0dd55136 100644 --- a/vortex-tensor/src/fixed_shape/matcher.rs +++ b/vortex-tensor/src/types/fixed_shape/matcher.rs @@ -8,8 +8,8 @@ use vortex_array::dtype::extension::Matcher; use vortex_error::VortexExpect; use vortex_error::vortex_panic; -use crate::fixed_shape::FixedShapeTensor; -use crate::fixed_shape::FixedShapeTensorMetadata; +use crate::types::fixed_shape::FixedShapeTensor; +use crate::types::fixed_shape::FixedShapeTensorMetadata; pub struct AnyFixedShapeTensor; @@ -98,7 +98,7 @@ mod tests { use vortex_error::VortexResult; use super::*; - use crate::vector::Vector; + use crate::types::vector::Vector; fn tensor_storage_dtype(element_ptype: PType, list_size: u32) -> DType { DType::FixedSizeList( diff --git a/vortex-tensor/src/fixed_shape/metadata.rs b/vortex-tensor/src/types/fixed_shape/metadata.rs similarity index 100% rename from vortex-tensor/src/fixed_shape/metadata.rs rename to vortex-tensor/src/types/fixed_shape/metadata.rs diff --git a/vortex-tensor/src/fixed_shape/mod.rs b/vortex-tensor/src/types/fixed_shape/mod.rs similarity index 100% rename from vortex-tensor/src/fixed_shape/mod.rs rename to vortex-tensor/src/types/fixed_shape/mod.rs diff --git a/vortex-tensor/src/fixed_shape/proto.rs b/vortex-tensor/src/types/fixed_shape/proto.rs similarity index 98% rename from vortex-tensor/src/fixed_shape/proto.rs rename to vortex-tensor/src/types/fixed_shape/proto.rs index 9d89c56bcec..89b3db4289d 100644 --- a/vortex-tensor/src/fixed_shape/proto.rs +++ b/vortex-tensor/src/types/fixed_shape/proto.rs @@ -8,7 +8,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_err; -use crate::fixed_shape::FixedShapeTensorMetadata; +use crate::types::fixed_shape::FixedShapeTensorMetadata; /// Protobuf representation of [`FixedShapeTensorMetadata`]. /// diff --git a/vortex-tensor/src/fixed_shape/vtable.rs b/vortex-tensor/src/types/fixed_shape/vtable.rs similarity index 94% rename from vortex-tensor/src/fixed_shape/vtable.rs rename to vortex-tensor/src/types/fixed_shape/vtable.rs index 52e32ef0cf6..7d69b9a7c44 100644 --- a/vortex-tensor/src/fixed_shape/vtable.rs +++ b/vortex-tensor/src/types/fixed_shape/vtable.rs @@ -11,9 +11,9 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; -use crate::fixed_shape::FixedShapeTensor; -use crate::fixed_shape::FixedShapeTensorMetadata; -use crate::fixed_shape::proto; +use crate::types::fixed_shape::FixedShapeTensor; +use crate::types::fixed_shape::FixedShapeTensorMetadata; +use crate::types::fixed_shape::proto; impl ExtVTable for FixedShapeTensor { type Metadata = FixedShapeTensorMetadata; @@ -80,8 +80,8 @@ mod tests { use vortex_array::dtype::extension::ExtVTable; use vortex_error::VortexResult; - use crate::fixed_shape::FixedShapeTensor; - use crate::fixed_shape::FixedShapeTensorMetadata; + use crate::types::fixed_shape::FixedShapeTensor; + use crate::types::fixed_shape::FixedShapeTensorMetadata; /// Serializes and deserializes the given metadata through protobuf, asserting equality. fn assert_roundtrip(metadata: &FixedShapeTensorMetadata) -> VortexResult<()> { diff --git a/vortex-tensor/src/types/mod.rs b/vortex-tensor/src/types/mod.rs new file mode 100644 index 00000000000..3ecd2826743 --- /dev/null +++ b/vortex-tensor/src/types/mod.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Internal homes for tensor extension types. + +pub mod fixed_shape; +pub mod vector; diff --git a/vortex-tensor/src/vector/matcher.rs b/vortex-tensor/src/types/vector/matcher.rs similarity index 96% rename from vortex-tensor/src/vector/matcher.rs rename to vortex-tensor/src/types/vector/matcher.rs index e4838b77426..7ac75f097db 100644 --- a/vortex-tensor/src/vector/matcher.rs +++ b/vortex-tensor/src/types/vector/matcher.rs @@ -10,7 +10,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use crate::vector::Vector; +use crate::types::vector::Vector; pub struct AnyVector; @@ -101,8 +101,8 @@ mod tests { use vortex_error::VortexResult; use super::*; - use crate::fixed_shape::FixedShapeTensor; - use crate::fixed_shape::FixedShapeTensorMetadata; + use crate::types::fixed_shape::FixedShapeTensor; + use crate::types::fixed_shape::FixedShapeTensorMetadata; fn vector_storage_dtype(element_ptype: PType, dimensions: u32) -> DType { DType::FixedSizeList( diff --git a/vortex-tensor/src/vector/mod.rs b/vortex-tensor/src/types/vector/mod.rs similarity index 100% rename from vortex-tensor/src/vector/mod.rs rename to vortex-tensor/src/types/vector/mod.rs diff --git a/vortex-tensor/src/vector/vtable.rs b/vortex-tensor/src/types/vector/vtable.rs similarity index 98% rename from vortex-tensor/src/vector/vtable.rs rename to vortex-tensor/src/types/vector/vtable.rs index 19fe57fed41..d870e7a8e0d 100644 --- a/vortex-tensor/src/vector/vtable.rs +++ b/vortex-tensor/src/types/vector/vtable.rs @@ -11,7 +11,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use crate::vector::Vector; +use crate::types::vector::Vector; impl ExtVTable for Vector { type Metadata = EmptyMetadata; @@ -70,7 +70,7 @@ mod tests { use vortex_array::extension::EmptyMetadata; use vortex_error::VortexResult; - use crate::vector::Vector; + use crate::types::vector::Vector; /// Constructs a `FixedSizeList` storage dtype with the given float [`PType`], list size, and /// [`Nullability`]. diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 3434058f082..92ead274bcb 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -242,10 +242,10 @@ pub mod test_helpers { use vortex_buffer::Buffer; use vortex_error::VortexResult; - use crate::fixed_shape::FixedShapeTensor; - use crate::fixed_shape::FixedShapeTensorMetadata; use crate::scalar_fns::l2_denorm::L2Denorm; - use crate::vector::Vector; + use crate::types::fixed_shape::FixedShapeTensor; + use crate::types::fixed_shape::FixedShapeTensorMetadata; + use crate::types::vector::Vector; /// Builds a `FixedSizeList` storage array from flat `elements`. The row count is /// inferred from `elements.len() / list_size`. diff --git a/vortex-tensor/src/vector_search.rs b/vortex-tensor/src/vector_search.rs index c5551c99f2d..2a036accdfe 100644 --- a/vortex-tensor/src/vector_search.rs +++ b/vortex-tensor/src/vector_search.rs @@ -48,7 +48,7 @@ use vortex_array::scalar_fn::fns::operators::Operator; use vortex_error::VortexResult; use crate::scalar_fns::cosine_similarity::CosineSimilarity; -use crate::vector::Vector; +use crate::types::vector::Vector; /// Build the lazy similarity-search expression tree for a prepared database array and a /// single query vector. From be2a14b60d8d17ea0f6f65f29796af058d087800 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 20 Apr 2026 20:07:03 +0100 Subject: [PATCH 146/250] deprecate: execute_mask over to_mask (#7574) deprecate to mask to allow execute_mask to take a context --------- Signed-off-by: Joe Isaacs --- encodings/alp/src/alp/array.rs | 2 +- encodings/alp/src/alp/compress.rs | 2 +- encodings/alp/src/alp_rd/array.rs | 2 +- encodings/datetime-parts/src/canonical.rs | 7 +-- .../src/bitpacking/array/bitpack_compress.rs | 9 ++- .../bitpacking/array/bitpack_decompress.rs | 2 +- .../fastlanes/src/for/array/for_decompress.rs | 2 +- encodings/fastlanes/src/rle/array/mod.rs | 2 +- .../fastlanes/src/rle/array/rle_decompress.rs | 4 +- encodings/fsst/src/array.rs | 2 +- encodings/fsst/src/compute/compare.rs | 4 +- encodings/pco/src/array.rs | 2 +- encodings/pco/src/tests.rs | 4 +- encodings/runend/src/compress.rs | 4 +- encodings/runend/src/decompress_bool.rs | 8 +-- encodings/sequence/src/compute/take.rs | 2 +- encodings/sparse/src/canonical.rs | 8 +-- encodings/sparse/src/lib.rs | 14 ++--- encodings/zstd/src/array.rs | 7 ++- encodings/zstd/src/test.rs | 4 +- fuzz/src/array/cast.rs | 6 +- fuzz/src/array/compare.rs | 6 +- fuzz/src/array/filter.rs | 5 +- fuzz/src/array/search_sorted.rs | 6 +- fuzz/src/array/slice.rs | 5 +- fuzz/src/array/sort.rs | 6 +- fuzz/src/array/take.rs | 5 +- vortex-array/public-api.lock | 14 ++--- .../src/aggregate_fn/fns/first/mod.rs | 2 +- .../src/aggregate_fn/fns/is_constant/mod.rs | 4 +- .../src/aggregate_fn/fns/is_sorted/bool.rs | 2 +- .../src/aggregate_fn/fns/is_sorted/decimal.rs | 2 +- .../aggregate_fn/fns/is_sorted/primitive.rs | 2 +- vortex-array/src/aggregate_fn/fns/last/mod.rs | 2 +- .../src/aggregate_fn/fns/min_max/bool.rs | 2 +- .../src/aggregate_fn/fns/min_max/decimal.rs | 2 +- .../src/aggregate_fn/fns/min_max/primitive.rs | 2 +- .../aggregate_fn/fns/nan_count/primitive.rs | 2 +- vortex-array/src/aggregate_fn/fns/sum/bool.rs | 2 +- .../src/aggregate_fn/fns/sum/decimal.rs | 2 +- .../src/aggregate_fn/fns/sum/primitive.rs | 2 +- vortex-array/src/arrays/bool/array.rs | 8 +-- vortex-array/src/arrays/bool/compute/take.rs | 2 +- vortex-array/src/arrays/bool/test_harness.rs | 4 +- .../src/arrays/chunked/compute/take.rs | 4 +- .../src/arrays/chunked/vtable/canonical.rs | 10 +--- .../src/arrays/constant/compute/take.rs | 6 +- .../src/arrays/decimal/compute/cast.rs | 2 +- .../src/arrays/decimal/compute/fill_null.rs | 2 +- vortex-array/src/arrays/dict/array.rs | 10 ++-- .../src/arrays/filter/execute/listview.rs | 4 +- vortex-array/src/arrays/filter/execute/mod.rs | 13 ++--- .../src/arrays/filter/execute/struct_.rs | 4 +- .../src/arrays/filter/execute/varbinview.rs | 3 +- .../arrays/fixed_size_list/compute/take.rs | 4 +- vortex-array/src/arrays/list/compute/take.rs | 12 ++-- vortex-array/src/arrays/masked/vtable/mod.rs | 2 +- vortex-array/src/arrays/null/compute/mod.rs | 4 +- .../src/arrays/primitive/array/top_value.rs | 2 +- .../src/arrays/primitive/compute/cast.rs | 2 +- .../src/arrays/primitive/compute/fill_null.rs | 8 +-- .../src/arrays/varbin/compute/compare.rs | 4 +- .../src/arrays/varbin/compute/filter.rs | 2 +- .../src/arrays/varbin/compute/take.rs | 6 +- vortex-array/src/arrays/varbinview/compact.rs | 2 +- .../src/arrays/varbinview/compute/take.rs | 2 +- .../src/arrays/varbinview/compute/zip.rs | 4 +- vortex-array/src/arrow/executor/bool.rs | 2 +- vortex-array/src/arrow/executor/byte_view.rs | 2 +- vortex-array/src/arrow/executor/decimal.rs | 8 +-- vortex-array/src/arrow/executor/primitive.rs | 2 +- vortex-array/src/arrow/executor/temporal.rs | 2 +- vortex-array/src/builders/bool.rs | 2 +- vortex-array/src/builders/decimal.rs | 2 +- vortex-array/src/builders/fixed_size_list.rs | 2 +- vortex-array/src/builders/list.rs | 2 +- vortex-array/src/builders/listview.rs | 2 +- vortex-array/src/builders/primitive.rs | 2 +- vortex-array/src/builders/struct_.rs | 2 +- vortex-array/src/builders/varbinview.rs | 4 +- vortex-array/src/compute/conformance/cast.rs | 4 +- vortex-array/src/mask.rs | 2 +- vortex-array/src/patches.rs | 8 +-- vortex-array/src/test_harness.rs | 2 +- vortex-array/src/validity.rs | 56 +++---------------- vortex-compressor/src/stats/bool.rs | 2 +- vortex-compressor/src/stats/float.rs | 2 +- vortex-compressor/src/stats/integer.rs | 2 +- .../src/kernel/encodings/date_time_parts.rs | 2 +- vortex-duckdb/src/convert/vector.rs | 10 ++-- vortex-duckdb/src/duckdb/vector.rs | 4 +- vortex-duckdb/src/exporter/decimal.rs | 2 +- vortex-duckdb/src/exporter/dict.rs | 4 +- vortex-duckdb/src/exporter/struct_.rs | 2 +- vortex-layout/src/layouts/dict/reader.rs | 2 +- vortex-layout/src/layouts/flat/writer.rs | 2 +- vortex-layout/src/layouts/table.rs | 2 +- 97 files changed, 202 insertions(+), 238 deletions(-) diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index dd11f3eb60a..caba6d60ec7 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -812,7 +812,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask(result_primitive.as_ref().len(), &mut ctx) + .execute_mask(result_primitive.as_ref().len(), &mut ctx) .unwrap() .value(idx); assert_eq!( diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index 392118ad472..2b4546f98de 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -79,7 +79,7 @@ where let validity = values .array() .validity()? - .to_mask(values.array().len(), ctx)?; + .execute_mask(values.array().len(), ctx)?; // exceptional_positions may contain exceptions at invalid positions (which contain garbage // data). We remove null exceptions in order to keep the Patches small. let (valid_exceptional_positions, valid_exceptional_values): (Buffer, Buffer) = diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index 5573c43d694..17742168f83 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -273,7 +273,7 @@ impl VTable for ALPRD { let validity = left_parts .as_ref() .validity()? - .to_mask(left_parts.as_ref().len(), ctx)?; + .execute_mask(left_parts.as_ref().len(), ctx)?; let decoded_array = if ptype == PType::F32 { PrimitiveArray::new( diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index 0314b669a12..cb5ea0358ba 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -12,7 +12,6 @@ use vortex_array::dtype::PType; use vortex_array::extension::datetime::TimeUnit; use vortex_array::extension::datetime::Timestamp; use vortex_array::match_each_integer_ptype; -use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; @@ -96,11 +95,7 @@ pub fn decode_to_temporal( } Ok(TemporalArray::new_timestamp( - PrimitiveArray::new( - values.freeze(), - Validity::copy_from_array(&array.clone().into_array())?, - ) - .into_array(), + PrimitiveArray::new(values.freeze(), array.validity()?).into_array(), options.unit, options.tz.clone(), )) diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 624280ddb75..bfde0671f31 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -206,7 +206,10 @@ pub fn gather_patches( }; let array_len = parray.len(); - let validity_mask = parray.as_ref().validity()?.to_mask(parray.len(), ctx)?; + let validity_mask = parray + .as_ref() + .validity()? + .execute_mask(parray.len(), ctx)?; let patches = if array_len < u8::MAX as usize { match_each_integer_ptype!(parray.ptype(), |T| { @@ -316,7 +319,7 @@ fn bit_width_histogram_typed( let mut bit_widths = vec![0usize; size_of::() * 8 + 1]; match array .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? .bit_buffer() { AllOr::All => { @@ -477,7 +480,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask(compressed.as_ref().len(), &mut ctx) + .execute_mask(compressed.as_ref().len(), &mut ctx) .unwrap() .to_bit_buffer() .set_indices() diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 1bd3e388672..d7eb030ef7a 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -58,7 +58,7 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: We later initialize the the uninitialized range of values with `copy_from_slice`. unsafe { // Append a dense null Mask. - uninit_range.append_mask(array.validity()?.to_mask(array.as_ref().len(), ctx)?); + uninit_range.append_mask(array.validity()?.execute_mask(array.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 070bef28d32..240390f91c9 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -109,7 +109,7 @@ pub(crate) fn fused_decompress< let mut uninit_range = builder.uninit_range(bp.len()); unsafe { // Append a dense null Mask. - uninit_range.append_mask(bp.validity()?.to_mask(bp.as_ref().len(), ctx)?); + uninit_range.append_mask(bp.validity()?.execute_mask(bp.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 743b18ec7f0..186d7f37c3c 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -311,7 +311,7 @@ mod tests { let validity_mask = sliced_array .validity() .unwrap() - .to_mask( + .execute_mask( sliced_array.len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/encodings/fastlanes/src/rle/array/rle_decompress.rs b/encodings/fastlanes/src/rle/array/rle_decompress.rs index 95f2c012a40..7240f3ff71f 100644 --- a/encodings/fastlanes/src/rle/array/rle_decompress.rs +++ b/encodings/fastlanes/src/rle/array/rle_decompress.rs @@ -5,12 +5,10 @@ use fastlanes::RLE; use num_traits::AsPrimitive; use num_traits::NumCast; use vortex_array::ExecutionCtx; -use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_array::match_each_native_ptype; use vortex_array::match_each_unsigned_integer_ptype; -use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -124,6 +122,6 @@ where buffer .freeze() .slice(offset_within_chunk..(offset_within_chunk + array.len())), - Validity::copy_from_array(&array.clone().into_array())?, + array.validity()?, )) } diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 719a13a8507..bafca478804 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -301,7 +301,7 @@ impl VTable for FSST { array .array() .validity()? - .to_mask(array.array().len(), ctx)?, + .execute_mask(array.array().len(), ctx)?, ); Ok(()) } diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index d1ef02e391f..553a3608e75 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -13,7 +13,6 @@ use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::binary::CompareKernel; use vortex_array::scalar_fn::fns::operators::CompareOperator; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; @@ -77,7 +76,8 @@ fn compare_fsst_constant( return Ok(Some( BoolArray::new( buffer, - Validity::copy_from_array(left.array())? + left.array() + .validity()? .union_nullability(right.dtype().nullability()), ) .into_array(), diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index edeb59b4a26..c119cc4ecce 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -261,7 +261,7 @@ fn collect_valid( let mask = parray .array() .validity()? - .to_mask(parray.array().len(), ctx)?; + .execute_mask(parray.array().len(), ctx)?; let result = parray .array() .filter(mask)? diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 294ab45e8e2..b848d04afa0 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -169,7 +169,7 @@ fn test_validity_vtable() { assert_eq!( arr.validity() .unwrap() - .to_mask(arr.len(), &mut ctx) + .execute_mask(arr.len(), &mut ctx) .unwrap(), Mask::from_iter(mask_bools) ); @@ -178,7 +178,7 @@ fn test_validity_vtable() { sliced .validity() .unwrap() - .to_mask(sliced.len(), &mut ctx) + .execute_mask(sliced.len(), &mut ctx) .unwrap(), Mask::from_iter(vec![true, true, false]) ); diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index 60c272b6372..b490076c355 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -189,7 +189,7 @@ pub fn runend_decode_primitive( let validity_mask = values .as_ref() .validity()? - .to_mask(values.as_ref().len(), ctx)?; + .execute_mask(values.as_ref().len(), ctx)?; Ok(match_each_native_ptype!(values.ptype(), |P| { match_each_unsigned_integer_ptype!(ends.ptype(), |E| { runend_decode_typed_primitive( @@ -293,7 +293,7 @@ pub fn runend_decode_varbinview( let validity_mask = values .as_ref() .validity()? - .to_mask(values.as_ref().len(), ctx)?; + .execute_mask(values.as_ref().len(), ctx)?; let views = values.views(); let (decoded_views, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| { diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index c0ccd8c4820..9e03f166e4b 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -41,7 +41,7 @@ pub fn runend_decode_bools( let validity = values .as_ref() .validity()? - .to_mask(values.as_ref().len(), ctx)?; + .execute_mask(values.as_ref().len(), ctx)?; let values_buf = values.to_bit_buffer(); let nullability = values.dtype().nullability(); @@ -390,7 +390,7 @@ mod tests { decoded .as_ref() .validity()? - .to_mask(decoded.as_ref().len(), &mut ctx) + .execute_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(0) ); @@ -400,7 +400,7 @@ mod tests { !decoded .as_ref() .validity()? - .to_mask(decoded.as_ref().len(), &mut ctx) + .execute_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(2000) ); @@ -409,7 +409,7 @@ mod tests { decoded .as_ref() .validity()? - .to_mask(decoded.as_ref().len(), &mut ctx) + .execute_mask(decoded.as_ref().len(), &mut ctx) .unwrap() .value(4000) ); diff --git a/encodings/sequence/src/compute/take.rs b/encodings/sequence/src/compute/take.rs index 7610b1dd038..6e1ebe263df 100644 --- a/encodings/sequence/src/compute/take.rs +++ b/encodings/sequence/src/compute/take.rs @@ -77,7 +77,7 @@ impl TakeExecute for Sequence { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mask = indices.validity()?.to_mask(indices.len(), ctx)?; + let mask = indices.validity()?.execute_mask(indices.len(), ctx)?; let indices = indices.clone().execute::(ctx)?; let result_nullability = array.dtype().nullability() | indices.dtype().nullability(); diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 5786d718a77..598481890cf 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -150,7 +150,7 @@ fn execute_sparse_lists( let validity = { let arr = array.as_array(); - Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + Validity::from_mask(arr.validity()?.execute_mask(arr.len(), ctx)?, nullability) }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { @@ -241,7 +241,7 @@ fn execute_sparse_fixed_size_list( let validity = { let arr = array.as_array(); - Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + Validity::from_mask(arr.validity()?.execute_mask(arr.len(), ctx)?, nullability) }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { @@ -435,7 +435,7 @@ fn execute_sparse_struct( let v = unresolved_patches.values(); v.validity() .vortex_expect("validity_mask") - .to_mask(v.len(), ctx) + .execute_mask(v.len(), ctx) .vortex_expect("Failed to compute validity mask") }, Nullability::Nullable, @@ -507,7 +507,7 @@ fn execute_varbin( let validity = { let arr = array.as_array(); Validity::from_mask( - arr.validity()?.to_mask(arr.len(), ctx)?, + arr.validity()?.execute_mask(arr.len(), ctx)?, dtype.nullability(), ) }; diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index f903f29621c..411bc9c7543 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -412,7 +412,7 @@ impl SparseData { fill_value.dtype() ) } - let mask = array.validity()?.to_mask(array.len(), ctx)?; + let mask = array.validity()?.execute_mask(array.len(), ctx)?; if mask.all_false() { // Array is constant NULL @@ -640,7 +640,7 @@ mod test { sliced .validity() .unwrap() - .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::from_iter(vec![true, false, false, true, false]) ); @@ -666,7 +666,7 @@ mod test { sliced .validity() .unwrap() - .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::from_iter(vec![false, true, true, false, true]) ); @@ -704,7 +704,7 @@ mod test { array .validity() .unwrap() - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .to_bit_buffer() .iter() @@ -722,7 +722,7 @@ mod test { array .validity() .unwrap() - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); @@ -760,7 +760,7 @@ mod test { sparse .validity() .unwrap() - .to_mask(sparse.len(), &mut ctx) + .execute_mask(sparse.len(), &mut ctx) .unwrap(), Mask::from_iter(vec![ true, true, false, true, false, true, false, true, true, false, true, false, @@ -779,7 +779,7 @@ mod test { let actual = array .validity() .unwrap() - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(); let expected = Mask::from_iter([ true, false, true, false, false, false, false, false, true, false, diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index d5f8d626996..4927a3aa7eb 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -372,7 +372,7 @@ fn collect_valid_primitive( let mask = parray .as_ref() .validity()? - .to_mask(parray.as_ref().len(), ctx)?; + .execute_mask(parray.as_ref().len(), ctx)?; let result = parray.filter(mask)?.execute::(ctx)?; Ok(result) } @@ -381,7 +381,10 @@ fn collect_valid_vbv( vbv: &VarBinViewArray, ctx: &mut ExecutionCtx, ) -> VortexResult<(ByteBuffer, Vec)> { - let mask = vbv.as_ref().validity()?.to_mask(vbv.as_ref().len(), ctx)?; + let mask = vbv + .as_ref() + .validity()? + .execute_mask(vbv.as_ref().len(), ctx)?; let buffer_and_value_byte_indices = match mask.bit_buffer() { AllOr::None => (Buffer::empty(), Vec::new()), _ => { diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 88a49aefe07..7ed22886b82 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -151,7 +151,7 @@ fn test_validity_vtable() { assert_eq!( arr.validity() .unwrap() - .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::from_iter(mask_bools) ); @@ -160,7 +160,7 @@ fn test_validity_vtable() { sliced .validity() .unwrap() - .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::from_iter(vec![true, true, false]) ); diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index 26cc9409884..35d7199c4af 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -44,7 +44,7 @@ pub fn cast_canonical_array( .map(|v| *v as Out) .collect::>(), Validity::from_mask( - array.validity()?.to_mask(array.len(), ctx)?, + array.validity()?.execute_mask(array.len(), ctx)?, target.nullability(), ), ) @@ -74,7 +74,7 @@ pub fn cast_canonical_array( .map(|v| *v as f64) .collect::>(), Validity::from_mask( - array.validity()?.to_mask(array.len(), ctx)?, + array.validity()?.execute_mask(array.len(), ctx)?, target.nullability(), ), ) @@ -91,7 +91,7 @@ pub fn cast_canonical_array( .map(|v| *v as f32) .collect::>(), Validity::from_mask( - array.validity()?.to_mask(array.len(), ctx)?, + array.validity()?.execute_mask(array.len(), ctx)?, target.nullability(), ), ) diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index 60a702f2839..848917b3422 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -54,7 +54,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), ctx) + .execute_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), @@ -84,7 +84,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), ctx) + .execute_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), @@ -117,7 +117,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), ctx) + .execute_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 09e10e26e9f..c6ffbdb374e 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -28,7 +28,10 @@ pub fn filter_canonical_array( ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = if array.dtype().is_nullable() { - let validity_buff = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); + let validity_buff = array + .validity()? + .execute_mask(array.len(), ctx)? + .to_bit_buffer(); Validity::from_iter( filter .iter() diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index 30b4d234b18..a35bbc9e4e6 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -72,7 +72,7 @@ pub fn search_sorted_canonical_array( let validity = bool_array .as_ref() .validity()? - .to_mask(bool_array.as_ref().len(), ctx)? + .execute_mask(bool_array.as_ref().len(), ctx)? .to_bit_buffer(); let opt_values = bool_array .to_bit_buffer() @@ -88,7 +88,7 @@ pub fn search_sorted_canonical_array( let validity = primitive_array .as_ref() .validity()? - .to_mask(primitive_array.as_ref().len(), ctx)? + .execute_mask(primitive_array.as_ref().len(), ctx)? .to_bit_buffer(); match_each_native_ptype!(p, |P| { let opt_values = primitive_array @@ -107,7 +107,7 @@ pub fn search_sorted_canonical_array( let validity = decimal_array .as_ref() .validity()? - .to_mask(decimal_array.as_ref().len(), ctx)? + .execute_mask(decimal_array.as_ref().len(), ctx)? .to_bit_buffer(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index ce0cfe062b6..3138d644147 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -29,7 +29,10 @@ pub fn slice_canonical_array( ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = if array.dtype().is_nullable() { - let bool_buff = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); + let bool_buff = array + .validity()? + .execute_mask(array.len(), ctx)? + .to_bit_buffer(); Validity::from(bool_buff.slice(start..stop)) } else { Validity::NonNullable diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index 8ef59930556..a2a70c8b167 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -32,7 +32,7 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR bool_array .as_ref() .validity()? - .to_mask(bool_array.as_ref().len(), ctx)? + .execute_mask(bool_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) @@ -52,7 +52,7 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR primitive_array .as_ref() .validity()? - .to_mask(primitive_array.as_ref().len(), ctx)? + .execute_mask(primitive_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) @@ -74,7 +74,7 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR decimal_array .as_ref() .validity()? - .to_mask(decimal_array.as_ref().len(), ctx)? + .execute_mask(decimal_array.as_ref().len(), ctx)? .to_bit_buffer() .iter(), ) diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index 1a6218eccf7..ea1c8b8071c 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -49,7 +49,10 @@ pub fn take_canonical_array( let nullable: Nullability = indices.contains(&None).into(); let validity = if array.dtype().is_nullable() || nullable == Nullability::Nullable { - let validity_idx = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); + let validity_idx = array + .validity()? + .execute_mask(array.len(), ctx)? + .to_bit_buffer(); Validity::from_iter( indices diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 933d1ea2310..88915460d76 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -1266,28 +1266,28 @@ pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, pub trait vortex_array::arrays::bool::BoolArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::bool::BoolArrayExt::execute_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask + +pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_execute_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub fn vortex_array::arrays::bool::BoolArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::bool::BoolArrayExt::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask - pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn vortex_array::arrays::bool::BoolArrayExt::validity(&self) -> vortex_array::validity::Validity impl> vortex_array::arrays::bool::BoolArrayExt for T -pub fn T::maybe_to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn T::execute_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask + +pub fn T::maybe_execute_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer -pub fn T::to_mask(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask - pub fn T::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn T::validity(&self) -> vortex_array::validity::Validity @@ -19030,8 +19030,6 @@ pub fn vortex_array::validity::Validity::as_array(&self) -> core::option::Option pub fn vortex_array::validity::Validity::cast_nullability(self, nullability: vortex_array::dtype::Nullability, len: usize) -> vortex_error::VortexResult -pub fn vortex_array::validity::Validity::copy_from_array(array: &vortex_array::ArrayRef) -> vortex_error::VortexResult - pub fn vortex_array::validity::Validity::execute_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::validity::Validity::filter(&self, mask: &vortex_mask::Mask) -> vortex_error::VortexResult diff --git a/vortex-array/src/aggregate_fn/fns/first/mod.rs b/vortex-array/src/aggregate_fn/fns/first/mod.rs index f04f31adb8f..22e9ef7e39d 100644 --- a/vortex-array/src/aggregate_fn/fns/first/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/first/mod.rs @@ -99,7 +99,7 @@ impl AggregateFnVTable for First { if partial.value.is_some() { return Ok(true); } - if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.first() { + if let Some(idx) = batch.validity()?.execute_mask(batch.len(), ctx)?.first() { let scalar = batch.execute_scalar(idx, ctx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index 2947ab4a100..078467ad92b 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -59,8 +59,8 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor } // Check validity masks match (null positions must be identical). - let a_mask = a.validity()?.to_mask(a.len(), ctx)?; - let b_mask = b.validity()?.to_mask(b.len(), ctx)?; + let a_mask = a.validity()?.execute_mask(a.len(), ctx)?; + let b_mask = b.validity()?.execute_mask(b.len(), ctx)?; if a_mask != b_mask { return Ok(false); } diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs index 408fa9b7806..afd18f15902 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs @@ -17,7 +17,7 @@ pub(super) fn check_bool_sorted( match array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs index a5e2d597485..1a9fc96aad2 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs @@ -32,7 +32,7 @@ where match array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs index e5433b6b176..da9cc12b7aa 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs @@ -30,7 +30,7 @@ fn compute_is_sorted( match array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { diff --git a/vortex-array/src/aggregate_fn/fns/last/mod.rs b/vortex-array/src/aggregate_fn/fns/last/mod.rs index 5329413b4b4..a0034d0820b 100644 --- a/vortex-array/src/aggregate_fn/fns/last/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/last/mod.rs @@ -97,7 +97,7 @@ impl AggregateFnVTable for Last { batch: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.last() { + if let Some(idx) = batch.validity()?.execute_mask(batch.len(), ctx)?.last() { let scalar = batch.execute_scalar(idx, ctx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs index a4ac8d4f4c0..3d814c693cf 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs @@ -26,7 +26,7 @@ pub(super) fn accumulate_bool( let mask = array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?; + .execute_mask(array.as_ref().len(), ctx)?; let true_non_null = match &mask { Mask::AllTrue(_) => array.to_bit_buffer(), Mask::AllFalse(_) => return Ok(()), diff --git a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs index 8fa6b6b8dcd..019136e2584 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs @@ -39,7 +39,7 @@ where match array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? { Mask::AllTrue(_) => compute_min_max(array.buffer::().iter(), array.decimal_dtype()), Mask::AllFalse(_) => None, diff --git a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs index 7bc46c90a1f..3470ba3728d 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs @@ -39,7 +39,7 @@ where match array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)? + .execute_mask(array.as_ref().len(), ctx)? { Mask::AllTrue(_) => compute_min_max(array.as_slice::().iter()), Mask::AllFalse(_) => None, diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs index 77ad78df1c4..8d8358235cc 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs @@ -17,7 +17,7 @@ pub(super) fn accumulate_primitive( match_each_float_ptype!(p.ptype(), |F| { *count += compute_nan_count_with_validity( p.as_slice::(), - p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?, + p.as_ref().validity()?.execute_mask(p.as_ref().len(), ctx)?, ) as u64; }); Ok(()) diff --git a/vortex-array/src/aggregate_fn/fns/sum/bool.rs b/vortex-array/src/aggregate_fn/fns/sum/bool.rs index f5a02299278..a32f3ba72f7 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/bool.rs @@ -22,7 +22,7 @@ pub(super) fn accumulate_bool( vortex_panic!("expected unsigned sum state for bool input"); }; - let mask = b.as_ref().validity()?.to_mask(b.as_ref().len(), ctx)?; + let mask = b.as_ref().validity()?.execute_mask(b.as_ref().len(), ctx)?; let true_count = match mask.bit_buffer() { AllOr::None => return Ok(false), AllOr::All => b.to_bit_buffer().true_count() as u64, diff --git a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs index f5c801d8528..e37b671eb46 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs @@ -28,7 +28,7 @@ pub(super) fn accumulate_decimal( d: &DecimalArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let mask = d.as_ref().validity()?.to_mask(d.as_ref().len(), ctx)?; + let mask = d.as_ref().validity()?.execute_mask(d.as_ref().len(), ctx)?; let validity = match &mask { Mask::AllTrue(_) => None, Mask::Values(mask_values) => Some(mask_values.bit_buffer()), diff --git a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs index e236c624ad4..19c414b8204 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs @@ -20,7 +20,7 @@ pub(super) fn accumulate_primitive( p: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let mask = p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?; + let mask = p.as_ref().validity()?.execute_mask(p.as_ref().len(), ctx)?; match mask.bit_buffer() { AllOr::None => Ok(false), AllOr::All => accumulate_primitive_all(inner, p), diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 7f2889246a4..6500badc7dc 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -99,7 +99,7 @@ pub trait BoolArrayExt: TypedArrayRef { BitBuffer::new_with_offset(buffer, self.as_ref().len(), self.offset) } - fn maybe_to_mask(&self, ctx: &mut ExecutionCtx) -> VortexResult> { + fn maybe_execute_mask(&self, ctx: &mut ExecutionCtx) -> VortexResult> { let all_valid = match &self.validity() { Validity::NonNullable | Validity::AllValid => true, Validity::AllInvalid => false, @@ -108,8 +108,8 @@ pub trait BoolArrayExt: TypedArrayRef { Ok(all_valid.then(|| Mask::from_buffer(self.to_bit_buffer()))) } - fn to_mask(&self, ctx: &mut ExecutionCtx) -> Mask { - self.maybe_to_mask(ctx) + fn execute_mask(&self, ctx: &mut ExecutionCtx) -> Mask { + self.maybe_execute_mask(ctx) .vortex_expect("failed to check validity") .vortex_expect("cannot convert nullable boolean array to mask") } @@ -117,7 +117,7 @@ pub trait BoolArrayExt: TypedArrayRef { fn to_mask_fill_null_false(&self, ctx: &mut ExecutionCtx) -> Mask { let validity_mask = self .validity() - .to_mask(self.as_ref().len(), ctx) + .execute_mask(self.as_ref().len(), ctx) .vortex_expect("Failed to compute validity mask"); let buffer = match validity_mask { Mask::AllTrue(_) => self.to_bit_buffer(), diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index f7129d08366..1f679445254 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -28,7 +28,7 @@ impl TakeExecute for Bool { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let indices_nulls_zeroed = match indices.validity()?.to_mask(indices.len(), ctx)? { + let indices_nulls_zeroed = match indices.validity()?.execute_mask(indices.len(), ctx)? { Mask::AllTrue(_) => indices.clone(), Mask::AllFalse(_) => { return Ok(Some( diff --git a/vortex-array/src/arrays/bool/test_harness.rs b/vortex-array/src/arrays/bool/test_harness.rs index 056d122b3dc..4e13d62e3ef 100644 --- a/vortex-array/src/arrays/bool/test_harness.rs +++ b/vortex-array/src/arrays/bool/test_harness.rs @@ -13,7 +13,7 @@ impl BoolArray { pub fn opt_bool_vec(&self) -> Vec> { self.validity() .vortex_expect("failed to get validity") - .to_mask( + .execute_mask( self.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) @@ -28,7 +28,7 @@ impl BoolArray { pub fn bool_vec(&self) -> Vec { self.validity() .vortex_expect("failed to get validity") - .to_mask( + .execute_mask( self.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index afa400de612..7024c2b3875 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -34,7 +34,7 @@ fn take_chunked( let indices_mask = indices .as_ref() .validity()? - .to_mask(indices.as_ref().len(), ctx)?; + .execute_mask(indices.as_ref().len(), ctx)?; let indices_values = indices.as_slice::(); let n = indices_values.len(); @@ -103,7 +103,7 @@ fn take_chunked( indices .as_ref() .validity()? - .to_mask(indices.as_ref().len(), ctx)?, + .execute_mask(indices.as_ref().len(), ctx)?, indices.dtype().nullability(), ); flat.take(PrimitiveArray::new(final_take.freeze(), take_validity).into_array()) diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index 1e32f7295bc..0cae05a78d4 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -42,17 +42,13 @@ pub(super) fn _canonicalize( let owned_chunks: Vec = array.iter_chunks().cloned().collect(); Ok(match array.dtype() { DType::Struct(struct_dtype, _) => { - let struct_array = pack_struct_chunks( - &owned_chunks, - Validity::copy_from_array(array.array())?, - struct_dtype, - ctx, - )?; + let struct_array = + pack_struct_chunks(&owned_chunks, array.array().validity()?, struct_dtype, ctx)?; Canonical::Struct(struct_array) } DType::List(elem_dtype, _) => Canonical::List(swizzle_list_chunks( &owned_chunks, - Validity::copy_from_array(array.array())?, + array.array().validity()?, elem_dtype, ctx, )?), diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 15e67a02929..1f05dc9db71 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -23,7 +23,7 @@ impl TakeReduce for Constant { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let result = match indices .validity()? - .to_mask(indices.len(), &mut ctx)? + .execute_mask(indices.len(), &mut ctx)? .bit_buffer() { AllOr::All => { @@ -112,7 +112,7 @@ mod tests { taken .validity() .unwrap() - .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .indices(), AllOr::Some(valid_indices) @@ -138,7 +138,7 @@ mod tests { taken .validity() .unwrap() - .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .indices(), AllOr::All diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 80155954214..75446b17e4e 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -397,7 +397,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( casted.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index 684e5aa61e4..144c90c74f9 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -131,7 +131,7 @@ mod tests { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index f1f6db96f72..e960c3f6764 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -145,7 +145,7 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { let codes = self.codes(); let codes_validity = codes .validity()? - .to_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; + .execute_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; #[expect(deprecated)] let codes_primitive = self.codes().to_primitive(); let values_len = self.values().len(); @@ -302,7 +302,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( dict.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) @@ -328,7 +328,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( dict.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) @@ -358,7 +358,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( dict.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) @@ -384,7 +384,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( dict.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index 1ee018f3168..32f8df02f7e 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -4,11 +4,11 @@ use std::sync::Arc; use vortex_error::VortexExpect; +use vortex_mask::Mask; use vortex_mask::MaskValues; use crate::arrays::ListViewArray; use crate::arrays::filter::execute::filter_validity; -use crate::arrays::filter::execute::values_to_mask; use crate::arrays::listview; use crate::arrays::listview::ListViewArrayExt; use crate::arrays::listview::ListViewRebuildMode; @@ -43,7 +43,7 @@ pub fn filter_listview(array: &ListViewArray, selection_mask: &Arc) ); // Simply filter the offsets and sizes arrays. - let mask_for_filter = values_to_mask(selection_mask); + let mask_for_filter = Mask::Values(Arc::clone(selection_mask)); let new_offsets = offsets .filter(mask_for_filter.clone()) .vortex_expect("ListViewArray offsets are guaranteed to support filter"); diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index fbe840f9bee..943238a7e1f 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -39,15 +39,10 @@ mod slice; mod struct_; mod varbinview; -/// Reconstruct a [`Mask`] from an [`Arc`]. -fn values_to_mask(values: &Arc) -> Mask { - Mask::Values(Arc::clone(values)) -} - /// A helper function that lazily filters a [`Validity`] with selection mask values. fn filter_validity(validity: Validity, mask: &Arc) -> Validity { validity - .filter(&values_to_mask(mask)) + .filter(&Mask::Values(Arc::clone(mask))) .vortex_expect("Somehow unable to wrap filter around a validity array") } @@ -73,7 +68,7 @@ pub(super) fn execute_filter_fast_paths( let child_arr = array.array(); if child_arr .validity()? - .to_mask(child_arr.len(), ctx)? + .execute_mask(child_arr.len(), ctx)? .true_count() == 0 { @@ -101,14 +96,14 @@ pub(super) fn execute_filter(canonical: Canonical, mask: &Arc) -> Ca Canonical::Extension(a) => { let filtered_storage = a .storage_array() - .filter(values_to_mask(mask)) + .filter(Mask::Values(Arc::clone(mask))) .vortex_expect("ExtensionArray storage type somehow could not be filtered"); Canonical::Extension(ExtensionArray::new(a.ext_dtype().clone(), filtered_storage)) } Canonical::Variant(a) => { let filtered_child = a .child() - .filter(values_to_mask(mask)) + .filter(Mask::Values(Arc::clone(mask))) .vortex_expect("VariantArray child could not be filtered"); Canonical::Variant(VariantArray::new(filtered_child)) } diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index c9dac73a8cd..ee209b90fc3 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -4,12 +4,12 @@ use std::sync::Arc; use vortex_error::VortexExpect; +use vortex_mask::Mask; use vortex_mask::MaskValues; use crate::ArrayRef; use crate::arrays::StructArray; use crate::arrays::filter::execute::filter_validity; -use crate::arrays::filter::execute::values_to_mask; use crate::arrays::struct_::StructArrayExt; pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray { @@ -20,7 +20,7 @@ pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray mask, ); - let mask_for_filter = values_to_mask(mask); + let mask_for_filter = Mask::Values(Arc::clone(mask)); let fields: Vec = array .iter_unmasked_fields() .map(|field| { diff --git a/vortex-array/src/arrays/filter/execute/varbinview.rs b/vortex-array/src/arrays/filter/execute/varbinview.rs index b757baa7c77..166e51275d8 100644 --- a/vortex-array/src/arrays/filter/execute/varbinview.rs +++ b/vortex-array/src/arrays/filter/execute/varbinview.rs @@ -12,13 +12,12 @@ use crate::ArrayRef; use crate::IntoArray; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; -use crate::arrays::filter::execute::values_to_mask; use crate::arrow::FromArrowArray; use crate::arrow::IntoArrowArray; pub fn filter_varbinview(array: &VarBinViewArray, mask: &Arc) -> VarBinViewArray { // Delegate to the Arrow implementation of filter over `VarBinView`. - arrow_filter_fn(&array.clone().into_array(), &values_to_mask(mask)) + arrow_filter_fn(&array.clone().into_array(), &Mask::Values(Arc::clone(mask))) .vortex_expect("VarBinViewArray is Arrow-compatible and supports arrow_filter_fn") .as_::() .into_owned() diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index 00c2eb8a198..3f2cf451c00 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -153,7 +153,7 @@ fn take_nullable_fsl( let array_validity = array .fixed_size_list_validity() - .to_mask(array.as_ref().len(), ctx) + .execute_mask(array.as_ref().len(), ctx) .vortex_expect("Failed to compute validity mask"); let indices_len = indices_array.as_ref().len(); let indices_validity = match indices_array @@ -162,7 +162,7 @@ fn take_nullable_fsl( { Validity::NonNullable | Validity::AllValid => Mask::new_true(indices_len), Validity::AllInvalid => Mask::new_false(indices_len), - Validity::Array(a) => a.execute::(ctx)?.to_mask(ctx), + Validity::Array(a) => a.execute::(ctx)?.execute_mask(ctx), }; // We must use placeholder zeros for null lists to maintain the array length without diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 33551227b68..7cc5ccd78cd 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -59,11 +59,13 @@ fn _take( indices_array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let data_validity = array + .list_validity() + .execute_mask(array.as_ref().len(), ctx)?; let indices_validity = indices_array .validity() .vortex_expect("Failed to compute validity mask") - .to_mask(indices_array.as_ref().len(), ctx)?; + .execute_mask(indices_array.as_ref().len(), ctx)?; if !indices_validity.all_true() || !data_validity.all_true() { return _take_nullable::(array, indices_array, ctx); @@ -127,11 +129,13 @@ fn _take_nullable(ctx)?; let offsets: &[O] = offsets_array.as_slice(); let indices: &[I] = indices_array.as_slice(); - let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let data_validity = array + .list_validity() + .execute_mask(array.as_ref().len(), ctx)?; let indices_validity = indices_array .validity() .vortex_expect("Failed to compute validity mask") - .to_mask(indices_array.as_ref().len(), ctx)?; + .execute_mask(indices_array.as_ref().len(), ctx)?; let mut new_offsets = PrimitiveBuilder::::with_capacity( Nullability::NonNullable, diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 9e3083c01e5..d4eb457c434 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -161,7 +161,7 @@ impl VTable for Masked { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let validity_mask = array.masked_validity().to_mask(array.len(), ctx)?; + let validity_mask = array.masked_validity().execute_mask(array.len(), ctx)?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index 502bb8fcc06..df2bc1547ae 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -38,7 +38,7 @@ mod test { sliced_arr .validity() .unwrap() - .to_mask(sliced_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(sliced_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::AllFalse(4) )); @@ -59,7 +59,7 @@ mod test { taken_arr .validity() .unwrap() - .to_mask(taken_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(taken_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::AllFalse(5) )); diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index 8638ea9af97..d3ee5eb5a65 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -33,7 +33,7 @@ impl PrimitiveArray { match_each_native_ptype!(self.ptype(), |P| { let (top, count) = typed_top_value( self.as_slice::

(), - self.as_ref().validity()?.to_mask( + self.as_ref().validity()?.execute_mask( self.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), )?, diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index f1c9d3bcdbc..abd58022ba6 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -243,7 +243,7 @@ mod test { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap(), Mask::from(BitBuffer::from(vec![false, true, true])) ); diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index 5862874a37d..67089bd534e 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -76,7 +76,7 @@ mod test { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); @@ -97,7 +97,7 @@ mod test { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); @@ -120,7 +120,7 @@ mod test { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); @@ -136,7 +136,7 @@ mod test { p.as_ref() .validity() .unwrap() - .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .all_true() ); diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 963e67fdde5..63fe1a622d0 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -184,7 +184,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( result.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx() ) @@ -220,7 +220,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( result.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx() ) diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index 911bc9c5e00..b3e1aa87a4f 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -70,7 +70,7 @@ fn filter_select_var_bin_by_slice( mask_slices, values .varbin_validity() - .to_mask(values.as_ref().len(), ctx) + .execute_mask(values.as_ref().len(), ctx) .vortex_expect("Failed to compute validity mask"), selection_count, ) diff --git a/vortex-array/src/arrays/varbin/compute/take.rs b/vortex-array/src/arrays/varbin/compute/take.rs index 0a6d5624229..7c799bfa54b 100644 --- a/vortex-array/src/arrays/varbin/compute/take.rs +++ b/vortex-array/src/arrays/varbin/compute/take.rs @@ -37,11 +37,13 @@ impl TakeExecute for VarBin { .dtype() .clone() .union_nullability(indices.dtype().nullability()); - let array_validity = array.varbin_validity().to_mask(array.as_ref().len(), ctx)?; + let array_validity = array + .varbin_validity() + .execute_mask(array.as_ref().len(), ctx)?; let indices_validity = indices .as_ref() .validity()? - .to_mask(indices.as_ref().len(), ctx)?; + .execute_mask(indices.as_ref().len(), ctx)?; let array = match_each_integer_ptype!(indices.ptype(), |I| { // On take, offsets get widened to either 32- or 64-bit based on the original type, diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index 9667acbd5dd..6effc7c656a 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -65,7 +65,7 @@ impl VarBinViewArray { where F: FnMut(&Ref), { - match self.as_ref().validity()?.to_mask( + match self.as_ref().validity()?.execute_mask( self.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), )? { diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index ff3d9052b0b..5c5ff08e2d0 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -35,7 +35,7 @@ impl TakeExecute for VarBinView { let indices_mask = indices .as_ref() .validity()? - .to_mask(indices.as_ref().len(), ctx)?; + .execute_mask(indices.as_ref().len(), ctx)?; let views_buffer = match_each_integer_ptype!(indices.ptype(), |I| { take_views(array.views(), indices.as_slice::(), &indices_mask) }); diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index 2be832b500f..f4a5c26a9ce 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -54,8 +54,8 @@ impl ZipKernel for VarBinView { let mut views_builder = BufferMut::::with_capacity(len); let mut validity_builder = LazyBitBufferBuilder::new(len); - let true_validity = if_true.varbinview_validity().to_mask(len, ctx)?; - let false_validity = if_false.varbinview_validity().to_mask(len, ctx)?; + let true_validity = if_true.varbinview_validity().execute_mask(len, ctx)?; + let false_validity = if_false.varbinview_validity().execute_mask(len, ctx)?; let mask = mask.try_to_mask_fill_null_false(ctx)?; let if_false_view = if_false; diff --git a/vortex-array/src/arrow/executor/bool.rs b/vortex-array/src/arrow/executor/bool.rs index 44fb9d4f683..68f2451cd77 100644 --- a/vortex-array/src/arrow/executor/bool.rs +++ b/vortex-array/src/arrow/executor/bool.rs @@ -24,7 +24,7 @@ pub fn canonical_bool_to_arrow( array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ), ))) } diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index 653817a982e..b88b1895d53 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -35,7 +35,7 @@ pub fn canonical_varbinview_to_arrow( array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ); // SAFETY: our own VarBinView array is considered safe. diff --git a/vortex-array/src/arrow/executor/decimal.rs b/vortex-array/src/arrow/executor/decimal.rs index bb4e12ef1fb..9d7caf0ebda 100644 --- a/vortex-array/src/arrow/executor/decimal.rs +++ b/vortex-array/src/arrow/executor/decimal.rs @@ -45,7 +45,7 @@ fn to_arrow_decimal32(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResu array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { @@ -94,7 +94,7 @@ fn to_arrow_decimal64(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResu array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { @@ -138,7 +138,7 @@ fn to_arrow_decimal128(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexRes array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { @@ -177,7 +177,7 @@ fn to_arrow_decimal256(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexRes array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?, + .execute_mask(array.as_ref().len(), ctx)?, ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { diff --git a/vortex-array/src/arrow/executor/primitive.rs b/vortex-array/src/arrow/executor/primitive.rs index 2e7d8a9716c..0d41176972b 100644 --- a/vortex-array/src/arrow/executor/primitive.rs +++ b/vortex-array/src/arrow/executor/primitive.rs @@ -28,7 +28,7 @@ where let validity = array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?; + .execute_mask(array.as_ref().len(), ctx)?; let null_buffer = to_null_buffer(validity); let buffer = array.into_buffer::().into_arrow_scalar_buffer(); Ok(Arc::new(ArrowPrimitiveArray::::new(buffer, null_buffer))) diff --git a/vortex-array/src/arrow/executor/temporal.rs b/vortex-array/src/arrow/executor/temporal.rs index b3941b69916..6fa8e93e704 100644 --- a/vortex-array/src/arrow/executor/temporal.rs +++ b/vortex-array/src/arrow/executor/temporal.rs @@ -160,7 +160,7 @@ where let validity = primitive .as_ref() .validity()? - .to_mask(primitive.as_ref().len(), ctx)?; + .execute_mask(primitive.as_ref().len(), ctx)?; let buffer = primitive.to_buffer::(); let values = buffer.into_arrow_scalar_buffer(); diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index 0e784aca67d..c09d69a5acd 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -125,7 +125,7 @@ impl ArrayBuilder for BoolBuilder { .as_ref() .validity() .vortex_expect("validity_mask") - .to_mask( + .execute_mask( bool_array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index 90c077b083c..c80f97d9ea2 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -211,7 +211,7 @@ impl ArrayBuilder for DecimalBuilder { .as_ref() .validity() .vortex_expect("validity_mask") - .to_mask( + .execute_mask( decimal_array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 36024f04340..5937c37b041 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -249,7 +249,7 @@ impl ArrayBuilder for FixedSizeListBuilder { array .validity() .vortex_expect("validity_mask in extend_from_array_unchecked") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask"), ); } diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 8e2c0e8283b..ae1bb20dfa9 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -229,7 +229,7 @@ impl ArrayBuilder for ListBuilder { array .validity() .vortex_expect("validity_mask in extend_from_array_unchecked") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask"), ); diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index d8217729996..3c7bb8e8885 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -311,7 +311,7 @@ impl ArrayBuilder for ListViewBuilder { array .validity() .vortex_expect("validity_mask in extend_from_array_unchecked") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask"), ); diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index 8d422ac7d2c..59381fc602f 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -194,7 +194,7 @@ impl ArrayBuilder for PrimitiveBuilder { .as_ref() .validity() .vortex_expect("validity_mask") - .to_mask( + .execute_mask( array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index b9832bba2a5..b7c737b3f1c 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -183,7 +183,7 @@ impl ArrayBuilder for StructBuilder { array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask"), ); } diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index c125e5086fe..9150e8e70e1 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -304,7 +304,7 @@ impl ArrayBuilder for VarBinViewBuilder { .as_ref() .validity() .vortex_expect("validity_mask") - .to_mask( + .execute_mask( array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) @@ -330,7 +330,7 @@ impl ArrayBuilder for VarBinViewBuilder { .as_ref() .validity() .vortex_expect("validity_mask") - .to_mask( + .execute_mask( array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), ) diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index 01b7dc67b2a..be5142d8f53 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -288,12 +288,12 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip array .validity() .vortex_expect("validity_mask should succeed in conformance test") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask"), casted .validity() .vortex_expect("validity_mask should succeed in conformance test") - .to_mask(casted.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .execute_mask(casted.len(), &mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("Failed to compute validity mask") ); for i in 0..array.len().min(10) { diff --git a/vortex-array/src/mask.rs b/vortex-array/src/mask.rs index 43fee47c486..ab035670e24 100644 --- a/vortex-array/src/mask.rs +++ b/vortex-array/src/mask.rs @@ -37,7 +37,7 @@ impl Executable for Mask { let mask = bool .as_ref() .validity()? - .to_mask(bool.as_ref().len(), ctx)?; + .execute_mask(bool.as_ref().len(), ctx)?; let bits = bool.into_bit_buffer(); // To handle nullable boolean arrays, we treat nulls as false in the mask. // TODO(ngates): is this correct? Feels like we should just force the caller to diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index 007ffa43a5d..fa532546b50 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -779,7 +779,7 @@ impl Patches { take_indices .as_ref() .validity()? - .to_mask(take_indices.as_ref().len(), ctx)?, + .execute_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { self.search_index_chunked_batch( @@ -797,7 +797,7 @@ impl Patches { take_indices .as_ref() .validity()? - .to_mask(take_indices.as_ref().len(), ctx)?, + .execute_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { let Some(offset) = ::from(self.offset) else { @@ -1183,7 +1183,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( primitive_values.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx() ) @@ -1226,7 +1226,7 @@ mod test { .as_ref() .validity() .unwrap() - .to_mask( + .execute_mask( primitive_values.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx() ) diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index fe1fc88bb15..dabeb8bdde4 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -28,7 +28,7 @@ pub fn check_metadata(name: &str, metadata: &[u8]) { /// Outputs the indices of the true values in a BoolArray pub fn to_int_indices(indices_bits: BoolArray) -> VortexResult> { let buffer = indices_bits.to_bit_buffer(); - let mask = indices_bits.as_ref().validity()?.to_mask( + let mask = indices_bits.as_ref().validity()?.execute_mask( indices_bits.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), )?; diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index 84586754dcf..66a92f66e73 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -12,7 +12,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_mask::AllOr; use vortex_mask::Mask; use vortex_mask::MaskValues; @@ -158,34 +157,11 @@ impl Validity { pub fn take(&self, indices: &ArrayRef) -> VortexResult { match self { - Self::NonNullable => { - let len = indices.len(); - let indices_mask = indices - .validity()? - .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; - match indices_mask.bit_buffer() { - AllOr::All => { - if indices.dtype().is_nullable() { - Ok(Self::AllValid) - } else { - Ok(Self::NonNullable) - } - } - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - } - } - Self::AllValid => { - let len = indices.len(); - let indices_mask = indices - .validity()? - .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; - match indices_mask.bit_buffer() { - AllOr::All => Ok(Self::AllValid), - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - } - } + Self::NonNullable => indices.validity(), + Self::AllValid => Ok(match indices.validity()? { + Self::NonNullable => Self::AllValid, + v => v, + }), Self::AllInvalid => Ok(Self::AllInvalid), Self::Array(is_valid) => { let maybe_is_valid = is_valid.take(indices.clone())?; @@ -227,6 +203,7 @@ impl Validity { /// Converts this validity into a [`Mask`] of the given length. /// /// Valid elements are `true` and invalid elements are `false`. + #[deprecated(note = "Use execute_mask")] pub fn to_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult { match self { Self::NonNullable | Self::AllValid => Ok(Mask::new_true(length)), @@ -392,16 +369,6 @@ impl Validity { } } - /// Create Validity by copying the given array's validity. - #[inline] - pub fn copy_from_array(array: &ArrayRef) -> VortexResult { - let len = array.len(); - let mask = array - .validity()? - .to_mask(len, &mut LEGACY_SESSION.create_execution_ctx())?; - Ok(Validity::from_mask(mask, array.dtype().nullability())) - } - /// Create Validity from boolean array with given nullability of the array. /// /// Note: You want to pass the nullability of parent array and not the nullability of the validity array itself @@ -608,13 +575,7 @@ mod tests { assert!( validity - .patch( - len, - 0, - &indices, - &patches, - &mut LEGACY_SESSION.create_execution_ctx(), - ) + .patch(len, 0, &indices, &patches, &mut ctx,) .unwrap() .mask_eq(&expected, &mut ctx) .unwrap() @@ -624,13 +585,14 @@ mod tests { #[test] #[should_panic] fn out_of_bounds_patch() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); Validity::NonNullable .patch( 2, 0, &buffer![4].into_array(), &Validity::AllInvalid, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut ctx, ) .unwrap(); } diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index 283ad91b5fb..8825ec8a7f6 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -46,7 +46,7 @@ impl BoolStats { let validity = input .as_ref() .validity()? - .to_mask(input.as_ref().len(), ctx)?; + .execute_mask(input.as_ref().len(), ctx)?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 0400bac6cdf..d968e8d368f 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -213,7 +213,7 @@ where let validity = array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?; + .execute_mask(array.as_ref().len(), ctx)?; let mut runs = 1; let head_idx = validity diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index 6c45515f39d..64345ac5f06 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -359,7 +359,7 @@ where let validity = array .as_ref() .validity()? - .to_mask(array.as_ref().len(), ctx)?; + .execute_mask(array.as_ref().len(), ctx)?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index b719ff17aa2..39d1a394a33 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -66,7 +66,7 @@ impl CudaExecute for DateTimePartsExecutor { let time_unit = options.unit; let time_zone = options.tz.clone(); - let validity = Validity::copy_from_array(&array.clone().into_array())?; + let validity = array.validity()?; if output_len == 0 { return Ok(Canonical::empty(array.dtype())); diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index 7f1b59753b9..a9a58c752c4 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -318,7 +318,7 @@ pub fn flat_vector_to_vortex(vector: &VectorRef, len: usize) -> VortexResult { - let validity = vector.validity_ref(len).to_mask(); + let validity = vector.validity_ref(len).execute_mask(); let entries = vector.as_slice_with_len::(len); let (offsets, sizes, child_min_length) = process_duckdb_lists(entries, &validity)?; @@ -531,7 +531,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask(vortex_values.as_ref().len(), &mut ctx) + .execute_mask(vortex_values.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); @@ -653,7 +653,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask(vortex_array.as_ref().len(), &mut ctx) + .execute_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); @@ -830,7 +830,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask(vortex_array.as_ref().len(), &mut ctx) + .execute_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(2, vec![0]) ); @@ -958,7 +958,7 @@ mod tests { .as_ref() .validity() .unwrap() - .to_mask(vortex_array.as_ref().len(), &mut ctx) + .execute_mask(vortex_array.as_ref().len(), &mut ctx) .unwrap(), Mask::from_indices(3, vec![0, 2]) ); diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs index ef2d35b2803..73aca417e49 100644 --- a/vortex-duckdb/src/duckdb/vector.rs +++ b/vortex-duckdb/src/duckdb/vector.rs @@ -328,7 +328,7 @@ impl ValidityRef<'_> { } /// Creates a mask directly from the DuckDB validity mask for optimal performance. - pub fn to_mask(&self) -> Mask { + pub fn execute_mask(&self) -> Mask { let Some(validity) = self.validity else { // All values are valid return Mask::AllTrue(self.len); @@ -341,7 +341,7 @@ impl ValidityRef<'_> { } pub fn to_validity(&self) -> Validity { - Validity::from_mask(self.to_mask(), Nullability::Nullable) + Validity::from_mask(self.execute_mask(), Nullability::Nullable) } } diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 66c96e3a2d0..f6b8d9607e7 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -149,7 +149,7 @@ mod tests { pub(crate) fn new_zero_copy_exporter( array: &DecimalArray, ) -> VortexResult> { - let validity = array.as_ref().validity()?.to_mask( + let validity = array.as_ref().validity()?.execute_mask( array.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx(), )?; diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index 55234fc502d..58873a8d7eb 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -48,13 +48,13 @@ pub(crate) fn new_exporter_with_flatten( if let Some(constant) = values.as_opt::() { return constant::new_exporter_with_mask( ConstantArray::new(constant.scalar().clone(), codes_len), - codes.validity()?.to_mask(codes_len, ctx)?, + codes.validity()?.execute_mask(codes_len, ctx)?, cache, ctx, ); } - let codes_mask = codes.validity()?.to_mask(codes_len, ctx)?; + let codes_mask = codes.validity()?.execute_mask(codes_len, ctx)?; match codes_mask { Mask::AllTrue(_) => {} diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index f786f9ef6bf..f18c5495754 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -56,7 +56,7 @@ pub(crate) fn new_exporter( }) .collect::>>()?; Ok(validity::new_exporter( - validity.to_mask(ctx), + validity.execute_mask(ctx), Box::new(StructExporter { children }), )) } diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 5896c5b466f..dd57926ac63 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -527,7 +527,7 @@ mod tests { let expected = array .validity() .unwrap() - .to_mask(array.len(), &mut ctx_exec) + .execute_mask(array.len(), &mut ctx_exec) .unwrap() .into_array(); let actual_canonical = actual diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 30ec0fe9352..da250414951 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -399,7 +399,7 @@ mod tests { result .validity() .unwrap() - .to_mask(result.len(), &mut ctx_exec) + .execute_mask(result.len(), &mut ctx_exec) .unwrap() .bit_buffer(), AllOr::Some(&validity_boolean_buffer) diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index 36567595835..28b3af52c87 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -244,7 +244,7 @@ impl LayoutStrategy for TableStrategy { sequence_pointer.advance(), chunk .validity()? - .to_mask(chunk.len(), &mut ctx)? + .execute_mask(chunk.len(), &mut ctx)? .into_array(), )); } From 91f1c2f11045d788861e8342bab2fccae9227aae Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 20 Apr 2026 15:36:42 -0400 Subject: [PATCH 147/250] Remove ScalarFnConstantRule (#7575) This reduction rule does execution and prevents any parent reduce rules from triggering --------- Signed-off-by: Robert Kruszewski --- vortex-array/src/arrays/scalar_fn/rules.rs | 28 +------------- vortex-array/src/scalar_fn/fns/between/mod.rs | 37 ++++++++++--------- .../src/scalar_fn/fns/list_contains/mod.rs | 3 -- 3 files changed, 22 insertions(+), 46 deletions(-) diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 4c0ed1acee7..30b46914b8e 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -9,10 +9,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::ArrayRef; -use crate::Canonical; use crate::IntoArray; -use crate::LEGACY_SESSION; -use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; @@ -34,11 +31,8 @@ use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::fns::pack::Pack; use crate::validity::Validity; -pub(super) const RULES: ReduceRuleSet = ReduceRuleSet::new(&[ - &ScalarFnPackToStructRule, - &ScalarFnConstantRule, - &ScalarFnAbstractReduceRule, -]); +pub(super) const RULES: ReduceRuleSet = + ReduceRuleSet::new(&[&ScalarFnPackToStructRule, &ScalarFnAbstractReduceRule]); pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&ScalarFnUnaryFilterPushDownRule), @@ -71,24 +65,6 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { } } -#[derive(Debug)] -struct ScalarFnConstantRule; -impl ArrayReduceRule for ScalarFnConstantRule { - fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { - if !array.children().iter().all(|c| c.is::()) { - return Ok(None); - } - if array.is_empty() { - Ok(Some(Canonical::empty(array.dtype()).into_array())) - } else { - let result = array - .array() - .execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; - Ok(Some(ConstantArray::new(result, array.len()).into_array())) - } - } -} - #[derive(Debug)] struct ScalarFnSliceReduceRule; impl ArrayParentReduceRule for ScalarFnSliceReduceRule { diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index d23c5363366..3384f5faf17 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -334,14 +334,13 @@ impl ScalarFnVTable for Between { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_buffer::buffer; use super::*; use crate::IntoArray; - use crate::LEGACY_SESSION; - #[expect(deprecated)] - use crate::ToCanonical as _; use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::DecimalArray; @@ -356,9 +355,13 @@ mod tests { use crate::expr::root; use crate::scalar::DecimalValue; use crate::scalar::Scalar; + use crate::session::ArraySession; use crate::test_harness::to_int_indices; use crate::validity::Validity; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_display() { let expr = between( @@ -398,7 +401,6 @@ mod tests { let array = buffer![1, 0, 1, 0, 1].into_array(); let upper = buffer![2, 1, 1, 0, 0].into_array(); - #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -407,10 +409,11 @@ mod tests { lower_strict, upper_strict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap() - .to_bool(); + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); let indices = to_int_indices(matches).unwrap(); assert_eq!(indices, expected); @@ -428,7 +431,6 @@ mod tests { ) .into_array(); - #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -437,17 +439,17 @@ mod tests { lower_strict: StrictComparison::NonStrict, upper_strict: StrictComparison::NonStrict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap() - .to_bool(); + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); let indices = to_int_indices(matches).unwrap(); assert!(indices.is_empty()); // upper is a fixed constant let upper = ConstantArray::new(Scalar::from(2), 5).into_array(); - #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -456,17 +458,17 @@ mod tests { lower_strict: StrictComparison::NonStrict, upper_strict: StrictComparison::NonStrict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap() - .to_bool(); + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); let indices = to_int_indices(matches).unwrap(); assert_eq!(indices, vec![0, 1, 3]); // lower is also a constant let lower = ConstantArray::new(Scalar::from(0), 5).into_array(); - #[expect(deprecated)] let matches = between_canonical( &array, &lower, @@ -475,10 +477,11 @@ mod tests { lower_strict: StrictComparison::NonStrict, upper_strict: StrictComparison::NonStrict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap() - .to_bool(); + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); let indices = to_int_indices(matches).unwrap(); assert_eq!(indices, vec![0, 1, 2, 3, 4]); } @@ -517,7 +520,7 @@ mod tests { lower_strict: StrictComparison::Strict, upper_strict: StrictComparison::NonStrict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap(); assert_arrays_eq!( @@ -534,7 +537,7 @@ mod tests { lower_strict: StrictComparison::NonStrict, upper_strict: StrictComparison::Strict, }, - &mut LEGACY_SESSION.create_execution_ctx(), + &mut SESSION.create_execution_ctx(), ) .unwrap(); assert_arrays_eq!( diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 2a79a0c6af2..c1b8687d0b7 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -483,7 +483,6 @@ mod tests { use crate::expr::stats::Stat; use crate::scalar::Scalar; use crate::scalar_fn::fns::list_contains::BoolArray; - use crate::scalar_fn::fns::list_contains::Constant; use crate::scalar_fn::fns::list_contains::ConstantArray; use crate::scalar_fn::fns::list_contains::ListViewArray; use crate::scalar_fn::fns::list_contains::PrimitiveArray; @@ -827,7 +826,6 @@ mod tests { let expr = list_contains(root(), lit(2i32)); let contains = list_array.apply(&expr).unwrap(); - assert!(contains.is::(), "Expected constant result"); let expected = BoolArray::from_iter([true, true]); assert_arrays_eq!(contains, expected); } @@ -845,7 +843,6 @@ mod tests { let expr = list_contains(root(), lit(2i32)); let contains = list_array.apply(&expr).unwrap(); - assert!(contains.is::(), "Expected constant result"); let expected = BoolArray::new( [false, false, false, false, false].into_iter().collect(), From dccfb246d68069059b7bb9125fc714b8d0f3c0d2 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:09:13 -0400 Subject: [PATCH 148/250] Add `tracing` support to the compressor (#7385) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 We have very little observability into the compressor. When we are debugging, we don't really have any idea of what schemes the compressor is trying, how good or how bad estimates are, how reliable sampling is, how the cascading paths look, etc. This change adds structured `tracing` support to `vortex-compressor`. The compressor now emits a top-level `compress` span and decision/debug events on the `vortex_compressor::encode` target, so a normal tracing subscriber can see what the compressor sampled, selected, accepted/rejected, and where nested failures happened. The `scheme.compress_result` event reports `scheme`, `before_nbytes`, `after_nbytes`, `estimated_ratio` when available, `actual_ratio` when available, and `accepted`. Sampling is recorded through `sample.result`; compression failures are recorded through `scheme.compress_failed` / `sample.compress_failed` with `cascade_path` and `cascade_depth`. Zero-byte outputs intentionally omit ratio fields instead of logging infinities. This also adds JSON formatting to the benchmark logging setup via `--log-format json`, which makes `data-gen` / `compress-bench` output usable as JSONL. One useful workflow is to generate TPC-H data with compressor logs enabled and use `jq` to find over-optimistic estimates that were rejected.

Example jq query for rejected over-estimates ```fish set LOG data-gen.jsonl jq -R -s -r ' def rows: split("\n") | map(fromjson? // empty); def r: if type == "number" then ((. * 1000 | round) / 1000) else . end; ([ "estimated_over_actual", "scheme", "estimated_ratio", "actual_ratio", "before_nbytes", "after_nbytes", "extra_bytes", "span_dtype", "span_len" ] | @tsv), ( rows | map(select(.target == "vortex_compressor::encode")) | map(select(.fields.message == "scheme.compress_result")) | map(select(.fields.accepted == false)) | map(select(.fields.estimated_ratio != null and .fields.actual_ratio != null)) | map(select(.fields.estimated_ratio > .fields.actual_ratio)) | map(.fields as $f | { scheme: $f.scheme, estimated_ratio: $f.estimated_ratio, actual_ratio: $f.actual_ratio, estimated_over_actual: ($f.estimated_ratio / $f.actual_ratio), before_nbytes: $f.before_nbytes, after_nbytes: $f.after_nbytes, extra_bytes: ($f.after_nbytes - $f.before_nbytes), span_dtype: (.span.dtype // ""), span_len: (.span.len // "") }) | sort_by(.estimated_over_actual) | reverse | .[:50][] | [ (.estimated_over_actual | r), .scheme, (.estimated_ratio | r), (.actual_ratio | r), .before_nbytes, .after_nbytes, .extra_bytes, .span_dtype, .span_len ] | @tsv ) ' $LOG ``` ``` estimated_over_actual scheme estimated_ratio actual_ratio before_nbytes after_nbytes extra_bytes span_dtype span_len 512 vortex.int.for 1.6 0.003 2 640 638 utf8? 2 512 vortex.int.for 2.667 0.005 2 384 382 utf8? 2 512 vortex.int.for 5.333 0.01 8 768 760 decimal(15,2)? 2 512 vortex.int.for 4.571 0.009 8 896 888 decimal(15,2)? 2 512 vortex.int.for 4 0.008 8 1024 1016 decimal(15,2)? 2 512 vortex.int.for 1.6 0.003 2 640 638 utf8? 2 512 vortex.int.for 8 0.016 2 128 126 utf8? 2 512 vortex.int.for 2.56 0.005 16 3200 3184 i64? 2 512 vortex.int.for 5.818 0.011 16 1408 1392 i64? 2 256 vortex.int.for 2 0.008 4 512 508 utf8 4 256 vortex.int.for 2 0.008 4 512 508 utf8 4 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 256 vortex.int.for 2 0.008 4 512 508 utf8 8192 204.8 vortex.int.for 4 0.02 5 256 251 utf8 5 204.8 vortex.int.for 4 0.02 5 256 251 utf8 8192 204.8 vortex.int.for 4 0.02 5 256 251 utf8 8192 204.8 vortex.int.for 2.667 0.013 5 384 379 utf8 5 204.8 vortex.int.for 2.667 0.013 5 384 379 utf8 5 146.286 vortex.int.for 1.333 0.009 7 768 761 utf8? 19 146.286 vortex.int.for 1.333 0.009 7 768 761 utf8? 19 128 vortex.int.for 2 0.016 8 512 504 utf8? 98 128 vortex.int.for 2 0.016 8 512 504 utf8? 98 113.778 vortex.int.for 1.6 0.014 18 1280 1262 i32? 184 113.778 vortex.int.for 1.6 0.014 18 1280 1262 i32? 184 113.778 vortex.int.for 32 0.281 36 128 92 i32? 184 113.778 vortex.int.for 32 0.281 36 128 92 i32? 184 64 vortex.int.for 2 0.031 16 512 496 utf8? 98 64 vortex.int.for 2 0.031 16 512 496 utf8? 98 64 vortex.int.for 2 0.031 16 512 496 utf8? 98 64 vortex.int.for 2 0.031 16 512 496 utf8? 98 53.895 vortex.int.for 1.6 0.03 19 640 621 utf8? 19 53.895 vortex.int.for 3.2 0.059 76 1280 1204 decimal(15,2)? 19 53.895 vortex.int.for 2.909 0.054 76 1408 1332 decimal(15,2)? 19 53.895 vortex.int.for 1.6 0.03 19 640 621 utf8? 19 53.895 vortex.int.for 1.6 0.03 19 640 621 utf8? 19 53.895 vortex.int.for 32 0.594 76 128 52 i32? 184 53.895 vortex.int.for 32 0.594 76 128 52 i32? 184 53.895 vortex.int.for 4 0.074 76 1024 948 decimal(15,2)? 19 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 51.2 vortex.int.for 2 0.039 20 512 492 utf8? 98 40.96 vortex.int.for 2 0.049 25 512 487 utf8? 25 40.96 vortex.int.for 2 0.049 25 512 487 utf8? 25 ```
## Testing Some basic tracing tests (that was claude-generated). --------- Signed-off-by: Connor Tsui --- Cargo.lock | 1 + benchmarks/compress-bench/src/main.rs | 9 +- vortex-bench/Cargo.toml | 1 + vortex-bench/src/bin/data-gen.rs | 10 +- vortex-bench/src/utils/logging.rs | 79 +++- vortex-btrblocks/src/schemes/integer.rs | 2 - vortex-compressor/Cargo.toml | 3 +- vortex-compressor/src/compressor.rs | 567 +++++++++++++++++++----- vortex-compressor/src/ctx.rs | 38 ++ vortex-compressor/src/estimate.rs | 126 ++++-- vortex-compressor/src/lib.rs | 21 + vortex-compressor/src/scheme.rs | 6 +- vortex-compressor/src/trace.rs | 186 ++++++++ 13 files changed, 890 insertions(+), 159 deletions(-) create mode 100644 vortex-compressor/src/trace.rs diff --git a/Cargo.lock b/Cargo.lock index d67ebdebb97..0499c2ec67f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10458,6 +10458,7 @@ dependencies = [ "rstest", "rustc-hash", "tracing", + "tracing-subscriber", "vortex-array", "vortex-buffer", "vortex-error", diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 26b71f1abe3..4800d3d6772 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -15,6 +15,7 @@ use regex::Regex; use vortex::utils::aliases::hash_map::HashMap; use vortex_bench::Engine; use vortex_bench::Format; +use vortex_bench::LogFormat; use vortex_bench::Target; use vortex_bench::compress::CompressMeasurements; use vortex_bench::compress::CompressOp; @@ -39,7 +40,7 @@ use vortex_bench::public_bi::PBIDataset::CMSprovider; use vortex_bench::public_bi::PBIDataset::Euro2016; use vortex_bench::public_bi::PBIDataset::Food; use vortex_bench::public_bi::PBIDataset::HashTags; -use vortex_bench::setup_logging_and_tracing; +use vortex_bench::setup_logging_and_tracing_with_format; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -69,13 +70,17 @@ struct Args { output_path: Option, #[arg(long)] tracing: bool, + /// Format for the primary stderr log sink. `text` is the default human-readable format; + /// `json` emits one JSON object per event, suitable for piping into `jq`. + #[arg(long, value_enum, default_value_t = LogFormat::Text)] + log_format: LogFormat, } #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); - setup_logging_and_tracing(args.verbose, args.tracing)?; + setup_logging_and_tracing_with_format(args.verbose, args.tracing, args.log_format)?; run_compress( args.iterations, diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index e170a6552a8..2e1e1a9a423 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -61,6 +61,7 @@ tracing = { workspace = true } tracing-perfetto = { workspace = true } tracing-subscriber = { workspace = true, features = [ "env-filter", + "json", "tracing-log", ] } url = { workspace = true } diff --git a/vortex-bench/src/bin/data-gen.rs b/vortex-bench/src/bin/data-gen.rs index d38c977a6c6..35c77d70c48 100644 --- a/vortex-bench/src/bin/data-gen.rs +++ b/vortex-bench/src/bin/data-gen.rs @@ -17,12 +17,13 @@ use vortex_bench::Benchmark; use vortex_bench::BenchmarkArg; use vortex_bench::CompactionStrategy; use vortex_bench::Format; +use vortex_bench::LogFormat; use vortex_bench::Opt; use vortex_bench::Opts; use vortex_bench::conversions::convert_parquet_directory_to_vortex; use vortex_bench::create_benchmark; use vortex_bench::generate_duckdb_registration_sql; -use vortex_bench::setup_logging_and_tracing; +use vortex_bench::setup_logging_and_tracing_with_format; #[derive(Parser)] #[command(name = "bench-data-gen")] @@ -37,6 +38,11 @@ struct Args { #[arg(long)] tracing: bool, + /// Format for the primary stderr log sink. `text` is the default human-readable format; + /// `json` emits one JSON object per event, suitable for piping into `jq`. + #[arg(long, value_enum, default_value_t = LogFormat::Text)] + log_format: LogFormat, + #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))] formats: Vec, @@ -49,7 +55,7 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); let opts = Opts::from(args.options); - setup_logging_and_tracing(args.verbose, args.tracing)?; + setup_logging_and_tracing_with_format(args.verbose, args.tracing, args.log_format)?; let benchmark = create_benchmark(args.benchmark, &opts)?; diff --git a/vortex-bench/src/utils/logging.rs b/vortex-bench/src/utils/logging.rs index cdc99b8fb59..b414184f930 100644 --- a/vortex-bench/src/utils/logging.rs +++ b/vortex-bench/src/utils/logging.rs @@ -4,34 +4,77 @@ use std::fs::File; use std::io::IsTerminal; +use clap::ValueEnum; use tracing::level_filters::LevelFilter; use tracing_perfetto::PerfettoLayer; use tracing_subscriber::EnvFilter; +use tracing_subscriber::Layer; use tracing_subscriber::prelude::*; -/// Initialize logging/tracing for a benchmark -pub fn setup_logging_and_tracing(verbose: bool, tracing: bool) -> anyhow::Result<()> { +/// Format for the primary stderr log sink. +/// +/// `Text` is the default human-readable formatter matching the historical behavior of this crate. +/// `Json` emits one newline-delimited JSON object per event, suitable for piping into `jq` or a log +/// aggregator. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, ValueEnum)] +pub enum LogFormat { + #[default] + Text, + Json, +} + +/// Initialize logging/tracing for a benchmark, hardcoding [`LogFormat::Text`]. +/// +/// See [`setup_logging_and_tracing_with_format`] if you want to select JSON +/// output from a CLI flag. +pub fn setup_logging_and_tracing(verbose: bool, perfetto: bool) -> anyhow::Result<()> { + setup_logging_and_tracing_with_format(verbose, perfetto, LogFormat::Text) +} + +/// Initialize logging/tracing for a benchmark with an explicit stderr format. +/// +/// - `verbose`: when `RUST_LOG` is unset, raises the default filter from `INFO` to `TRACE`. Has no +/// effect when `RUST_LOG` is set (the env var wins). +/// - `perfetto`: when `true`, additionally attaches a [`tracing_perfetto::PerfettoLayer`] that +/// writes span begin/end events to `trace.json` in the current directory. Intended to be loaded +/// into the Perfetto UI for flamegraph visualization. +/// - `format`: controls the primary stderr sink's formatting. See [`LogFormat`]. +pub fn setup_logging_and_tracing_with_format( + verbose: bool, + perfetto: bool, + format: LogFormat, +) -> anyhow::Result<()> { let filter = default_env_filter(verbose); - let fmt_layer = tracing_subscriber::fmt::layer() - .with_writer(std::io::stderr) - .with_level(true) - .with_file(true) - .with_line_number(true) - .with_ansi(std::io::stderr().is_terminal()); + let perfetto_layer = perfetto + .then(|| { + Ok::<_, anyhow::Error>( + PerfettoLayer::new(File::create("trace.json")?).with_debug_annotations(true), + ) + }) + .transpose()?; + + // `fmt::layer()` and `fmt::layer().json()` produce different concrete types, + // so erase each to a `dyn Layer` via `.boxed()` and keep the registry uniform. + let fmt_layer: Box + Send + Sync> = match format { + LogFormat::Text => tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .with_level(true) + .with_file(true) + .with_line_number(true) + .with_ansi(std::io::stderr().is_terminal()) + .boxed(), + LogFormat::Json => tracing_subscriber::fmt::layer() + .json() + .with_writer(std::io::stderr) + .with_current_span(true) + .with_span_list(true) + .boxed(), + }; tracing_subscriber::registry() .with(filter) - .with( - tracing - .then(|| { - Ok::<_, anyhow::Error>( - PerfettoLayer::new(File::create("trace.json")?) - .with_debug_annotations(true), - ) - }) - .transpose()?, - ) + .with(perfetto_layer) .with(fmt_layer) .init(); diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index de428f967ac..d9d21a28b84 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -307,8 +307,6 @@ impl Scheme for ZigZagScheme { let compressed = compressor.compress_child(&encoded.into_array(), &ctx, self.id(), 0)?; - tracing::debug!("zigzag output: {}", compressed.encoding_id()); - Ok(ZigZag::try_new(compressed)?.into_array()) } } diff --git a/vortex-compressor/Cargo.toml b/vortex-compressor/Cargo.toml index da9bd07889c..2d94679157a 100644 --- a/vortex-compressor/Cargo.toml +++ b/vortex-compressor/Cargo.toml @@ -19,7 +19,7 @@ num-traits = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } rustc-hash = { workspace = true } -tracing = { workspace = true } +tracing = { workspace = true, features = ["std", "attributes"] } vortex-array = { workspace = true } vortex-buffer = { workspace = true } vortex-error = { workspace = true } @@ -29,6 +29,7 @@ vortex-utils = { workspace = true } [dev-dependencies] divan = { workspace = true } rstest = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } vortex-array = { workspace = true, features = ["_test-harness"] } [lints] diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 36ed7129d63..bc739637192 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -39,9 +39,11 @@ use crate::builtins::IntDictScheme; use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; use crate::estimate::DeferredEstimate; +use crate::estimate::EstimateScore; use crate::estimate::EstimateVerdict; +use crate::estimate::WinnerEstimate; use crate::estimate::estimate_compression_ratio_with_sampling; -use crate::estimate::is_better_ratio; +use crate::estimate::is_better_score; use crate::scheme::ChildSelection; use crate::scheme::DescendantExclusion; use crate::scheme::Scheme; @@ -49,11 +51,10 @@ use crate::scheme::SchemeExt; use crate::scheme::SchemeId; use crate::stats::ArrayAndStats; use crate::stats::GenerateStatsOptions; +use crate::trace; -/// The implicit root scheme ID for the compressor's own cascading (e.g. list offset compression). -/// -/// This is the **only** [`SchemeId`] that is not auto-provided via [`SchemeExt`]. -const ROOT_SCHEME_ID: SchemeId = SchemeId { +/// Synthetic scheme ID used for the compressor's own root-level cascading. +pub(crate) const ROOT_SCHEME_ID: SchemeId = SchemeId { name: "vortex.compressor.root", }; @@ -65,15 +66,6 @@ mod root_list_children { pub const SIZES: usize = 2; } -/// The winning estimate for a scheme after all deferred work has been resolved. -#[derive(Debug, Clone, Copy, PartialEq)] -enum WinnerEstimate { - /// The scheme must be used immediately. - AlwaysUse, - /// The scheme won by numeric compression ratio. - Ratio(f64), -} - /// The main compressor type implementing cascading adaptive compression. /// /// This compressor applies adaptive compression [`Scheme`]s to arrays based on their data types and @@ -136,22 +128,27 @@ impl CascadingCompressor { /// /// Returns an error if canonicalization or compression fails. pub fn compress(&self, array: &ArrayRef) -> VortexResult { + let before_nbytes = array.nbytes(); + let span = trace::compress_span(array.len(), array.dtype(), before_nbytes); + let _enter = span.enter(); + let canonical = array .clone() .execute::(&mut self.execution_ctx())? .0; - - // Compact it, removing any wasted space before we attempt to compress it. let compact = canonical.compact()?; + let compressed = self.compress_canonical(compact, CompressorContext::new())?; + + trace::record_compress_outcome(&span, before_nbytes, compressed.nbytes()); - self.compress_canonical(compact, CompressorContext::new()) + Ok(compressed) } /// Compresses a child array produced by a cascading scheme. /// - /// If the cascade budget is exhausted, the canonical array is returned as-is. Otherwise, - /// the child context is created by descending and recording the parent scheme + child - /// index, and compression proceeds normally. + /// If the cascade budget is exhausted, the canonical array is returned as-is. Otherwise, the + /// child context is created by descending and recording the parent scheme + child index, and + /// compression proceeds normally. /// /// # Errors /// @@ -164,6 +161,7 @@ impl CascadingCompressor { child_index: usize, ) -> VortexResult { if parent_ctx.finished_cascading() { + trace::cascade_exhausted(parent_id, child_index); return Ok(child.clone()); } @@ -276,17 +274,16 @@ impl CascadingCompressor { /// The main scheme-selection entry point for a single leaf array. /// /// Filters allowed schemes by [`matches`] and exclusion rules, merges their [`stats_options`] - /// into a single [`GenerateStatsOptions`], then delegates to [`choose_scheme`] to pick the - /// winner by estimated compression ratio. + /// into a single [`GenerateStatsOptions`], and picks the winner by estimated compression + /// ratio. /// - /// If a winner is found and its compressed output is actually smaller, that output is returned. - /// Otherwise, the original array is returned unchanged. + /// If a winner is found and its compressed output is actually smaller, that output is + /// returned. Otherwise, the original array is returned unchanged. /// /// Empty and all-null arrays are short-circuited before any scheme evaluation. /// /// [`matches`]: Scheme::matches /// [`stats_options`]: Scheme::stats_options - /// [`choose_scheme`]: Self::choose_scheme fn choose_and_compress( &self, canonical: Canonical, @@ -301,15 +298,10 @@ impl CascadingCompressor { let array: ArrayRef = canonical.into(); - // If there are no schemes that we can compress into, then just return it uncompressed. - if eligible_schemes.is_empty() { + if eligible_schemes.is_empty() || array.is_empty() { return Ok(array); } - // Nothing to compress if empty or all-null. - if array.is_empty() { - return Ok(array); - } if array.all_invalid(&mut self.execution_ctx())? { return Ok( ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()).into_array(), @@ -327,35 +319,51 @@ impl CascadingCompressor { let mut data = ArrayAndStats::new(array, merged_opts); - // TODO(connor): Add tracing support for logging the winner estimate. - if let Some((winner, _winner_estimate)) = + let Some((winner, winner_estimate)) = self.choose_best_scheme(&eligible_schemes, &mut data, ctx.clone())? - { - // Sampling and estimation chose a scheme, so let's compress the whole array with it. - let compressed = winner.compress(self, &mut data, ctx)?; - - // TODO(connor): Add a tracing warning here if compression with the chosen scheme - // failed, since there was likely more we could have done while choosing schemes. - - // Only choose the compressed array if it is smaller than the canonical one. - if compressed.nbytes() < before_nbytes { - // TODO(connor): Add a tracing warning here too. - return Ok(compressed); - } - - // TODO(connor): HACK TO SUPPORT L2 DENORMALIZATION!!! - if compressed.is::() { - return Ok(compressed); + else { + return Ok(data.into_array()); + }; + + // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the + // scheme name and cascade history before propagating. + let error_ctx = trace::enabled_error_context(&ctx); + let compressed = match winner.compress(self, &mut data, ctx) { + Ok(compressed) => compressed, + Err(err) => { + // NB: this is the only way we can tell which scheme panicked / bailed on their + // data, especially for third-party schemes where the error site may not carry any + // compressor context. + trace::scheme_compress_failed(winner.id(), before_nbytes, error_ctx.as_ref(), &err); + return Err(err); } + }; + + let after_nbytes = compressed.nbytes(); + let actual_ratio = (after_nbytes != 0).then(|| before_nbytes as f64 / after_nbytes as f64); + + // TODO(connor): HACK TO SUPPORT L2 DENORMALIZATION!!! + let accepted = after_nbytes < before_nbytes || compressed.is::(); + + trace::scheme_compress_result( + winner.id(), + before_nbytes, + after_nbytes, + winner_estimate.trace_ratio(), + actual_ratio, + accepted, + ); + + if accepted { + Ok(compressed) + } else { + Ok(data.into_array()) } - - // No scheme improved on the original. - Ok(data.into_array()) } - /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme and - /// resolved winner estimate, or `None` if no scheme exceeds 1.0. Ties are broken by - /// registration order (earlier in the list wins). + /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme along + /// with its resolved winner estimate, or `None` if no scheme beats the canonical encoding. + /// Ties are broken by registration order (earlier in the list wins). /// /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio fn choose_best_scheme( @@ -364,65 +372,43 @@ impl CascadingCompressor { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult> { - let mut best: Option<(&'static dyn Scheme, f64)> = None; + let mut best: Option<(&'static dyn Scheme, EstimateScore)> = None; - // TODO(connor): Might want to use an `im` data structure inside of `ctx` if the clones here - // are expensive. + // TODO(connor): Rather than computing the deferred estimates eagerly, it would be better to + // look at all quick estimates and see if it makes sense to sample at all. for &scheme in schemes { - let estimate = scheme.expected_compression_ratio(data, ctx.clone()); - - // TODO(connor): Rather than computing the deferred estimates eagerly, it would be - // better to look at all quick estimates and see if it makes sense to sample at all. - match estimate { - CompressionEstimate::Verdict(verdict) => { - if let Some(winner_estimate) = - Self::check_and_update_estimate_verdict(&mut best, scheme, verdict) - { - return Ok(Some((scheme, winner_estimate))); - } - } + let verdict = match scheme.expected_compression_ratio(data, ctx.clone()) { + CompressionEstimate::Verdict(verdict) => verdict, CompressionEstimate::Deferred(DeferredEstimate::Sample) => { - let sample_ratio = estimate_compression_ratio_with_sampling( + let score = estimate_compression_ratio_with_sampling( scheme, self, data.array(), ctx.clone(), )?; - - if is_better_ratio(sample_ratio, &best) { - best = Some((scheme, sample_ratio)); + if is_better_score(score, &best) { + best = Some((scheme, score)); } + continue; } - CompressionEstimate::Deferred(DeferredEstimate::Callback(estimate_callback)) => { - let verdict = estimate_callback(self, data, ctx.clone())?; - if let Some(winner_estimate) = - Self::check_and_update_estimate_verdict(&mut best, scheme, verdict) - { - return Ok(Some((scheme, winner_estimate))); + CompressionEstimate::Deferred(DeferredEstimate::Callback(callback)) => { + callback(self, data, ctx.clone())? + } + }; + + match verdict { + EstimateVerdict::Skip => {} + EstimateVerdict::AlwaysUse => return Ok(Some((scheme, WinnerEstimate::AlwaysUse))), + EstimateVerdict::Ratio(ratio) => { + let score = EstimateScore::FiniteCompression(ratio); + if is_better_score(score, &best) { + best = Some((scheme, score)); } } } } - Ok(best.map(|(scheme, ratio)| (scheme, WinnerEstimate::Ratio(ratio)))) - } - - /// Updates `best` from a terminal estimate verdict. - fn check_and_update_estimate_verdict( - best: &mut Option<(&'static dyn Scheme, f64)>, - scheme: &'static dyn Scheme, - verdict: EstimateVerdict, - ) -> Option { - match verdict { - EstimateVerdict::Skip => None, - EstimateVerdict::AlwaysUse => Some(WinnerEstimate::AlwaysUse), - EstimateVerdict::Ratio(ratio) => { - if is_better_ratio(ratio, &*best) { - *best = Some((scheme, ratio)); - } - None - } - } + Ok(best.map(|(scheme, score)| (scheme, WinnerEstimate::Score(score)))) } // TODO(connor): Lots of room for optimization here. @@ -544,10 +530,21 @@ impl CascadingCompressor { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::sync::Arc; + + use tracing::Event; + use tracing::Subscriber; + use tracing::field::Field; + use tracing::field::Visit; + use tracing_subscriber::Layer; + use tracing_subscriber::layer::Context; + use tracing_subscriber::prelude::*; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; + use vortex_array::arrays::NullArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::buffer; @@ -559,8 +556,11 @@ mod tests { use crate::ctx::CompressorContext; use crate::estimate::CompressionEstimate; use crate::estimate::DeferredEstimate; + use crate::estimate::EstimateScore; use crate::estimate::EstimateVerdict; + use crate::estimate::WinnerEstimate; use crate::scheme::SchemeExt; + use crate::trace::TARGET_TRACE; fn compressor() -> CascadingCompressor { CascadingCompressor::new(vec![&IntDictScheme, &FloatDictScheme, &StringDictScheme]) @@ -575,6 +575,98 @@ mod tests { matches!(canonical, Canonical::Primitive(primitive) if primitive.ptype().is_int()) } + fn test_integer_array() -> ArrayRef { + PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array() + } + + #[derive(Debug, Clone, PartialEq, Eq)] + struct RecordedEvent { + target: String, + fields: BTreeMap, + } + + #[derive(Default)] + struct EventVisitor { + fields: BTreeMap, + } + + impl Visit for EventVisitor { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.fields + .insert(field.name().to_owned(), format!("{value:?}")); + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.fields + .insert(field.name().to_owned(), value.to_string()); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.fields + .insert(field.name().to_owned(), value.to_string()); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.fields + .insert(field.name().to_owned(), value.to_string()); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.fields + .insert(field.name().to_owned(), value.to_owned()); + } + } + + struct RecordingLayer { + events: Arc>>, + } + + impl RecordingLayer { + fn new(events: Arc>>) -> Self { + Self { events } + } + } + + impl Layer for RecordingLayer + where + S: Subscriber, + { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let mut visitor = EventVisitor::default(); + event.record(&mut visitor); + self.events.lock().push(RecordedEvent { + target: event.metadata().target().to_owned(), + fields: visitor.fields, + }); + } + } + + fn record_events(f: impl FnOnce() -> T) -> (T, Vec) { + let events = Arc::new(Mutex::new(Vec::new())); + let subscriber = + tracing_subscriber::registry().with(RecordingLayer::new(Arc::clone(&events))); + let result = tracing::subscriber::with_default(subscriber, f); + let recorded = events.lock().clone(); + (result, recorded) + } + + fn find_event<'a>( + events: &'a [RecordedEvent], + target: &str, + message: &str, + ) -> &'a RecordedEvent { + events + .iter() + .find(|event| { + event.target == target + && event + .fields + .get("message") + .is_some_and(|value| value == message) + }) + .expect("expected event not found") + } + #[derive(Debug)] struct DirectRatioScheme; @@ -731,6 +823,156 @@ mod tests { } } + #[derive(Debug)] + struct HugeRatioScheme; + + impl Scheme for HugeRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.huge_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::Ratio(100.0)) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct ZeroBytesSamplingScheme; + + impl Scheme for ZeroBytesSamplingScheme { + fn scheme_name(&self) -> &'static str { + "test.zero_bytes_sampling" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Sample) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + Ok(NullArray::new(data.array().len()).into_array()) + } + } + + #[derive(Debug)] + struct NestedFailureParentScheme; + + impl Scheme for NestedFailureParentScheme { + fn scheme_name(&self) -> &'static str { + "test.nested_failure_parent" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) + } + + fn compress( + &self, + compressor: &CascadingCompressor, + data: &mut ArrayAndStats, + ctx: CompressorContext, + ) -> VortexResult { + compressor.compress_child(data.array(), &ctx, self.id(), 1) + } + } + + #[derive(Debug)] + struct NestedFailureLeafScheme; + + impl Scheme for NestedFailureLeafScheme { + fn scheme_name(&self) -> &'static str { + "test.nested_failure_leaf" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + vortex_error::vortex_bail!("nested failure") + } + } + + #[derive(Debug)] + struct SamplingFailureScheme; + + impl Scheme for SamplingFailureScheme { + fn scheme_name(&self) -> &'static str { + "test.sampling_failure" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Sample) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &mut ArrayAndStats, + _ctx: CompressorContext, + ) -> VortexResult { + vortex_error::vortex_bail!("sample failure") + } + } + #[test] fn test_self_exclusion() { let c = compressor(); @@ -827,7 +1069,7 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Ratio(2.0))) + Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(2.0)))) if scheme.id() == DirectRatioScheme.id() )); Ok(()) @@ -844,12 +1086,59 @@ mod tests { assert!(matches!( winner, - Some((scheme, WinnerEstimate::Ratio(3.0))) + Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(3.0)))) if scheme.id() == CallbackRatioScheme.id() )); Ok(()) } + #[test] + fn zero_byte_sample_loses_to_finite_ratio() -> VortexResult<()> { + let compressor = CascadingCompressor::new(vec![&HugeRatioScheme, &ZeroBytesSamplingScheme]); + let schemes: [&'static dyn Scheme; 2] = [&HugeRatioScheme, &ZeroBytesSamplingScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + if scheme.id() == HugeRatioScheme.id() + )); + Ok(()) + } + + #[test] + fn finite_ratio_displaces_zero_byte_sample() -> VortexResult<()> { + let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme, &HugeRatioScheme]); + let schemes: [&'static dyn Scheme; 2] = [&ZeroBytesSamplingScheme, &HugeRatioScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(100.0)))) + if scheme.id() == HugeRatioScheme.id() + )); + Ok(()) + } + + #[test] + fn zero_byte_sample_alone_selects_no_scheme() -> VortexResult<()> { + let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); + let schemes: [&'static dyn Scheme; 1] = [&ZeroBytesSamplingScheme]; + let mut data = estimate_test_data(); + + let winner = + compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + + assert!(winner.is_none()); + Ok(()) + } + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( @@ -890,9 +1179,81 @@ mod tests { // Before the fix this panicked with: // "this must be present since `DictScheme` declared that we need distinct values" - let ratio = + let score = estimate_compression_ratio_with_sampling(&FloatDictScheme, &compressor, &array, ctx)?; - assert!(ratio.is_finite()); + assert!(matches!(score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite())); Ok(()) } + + #[test] + fn compress_failure_event_includes_cascade_path_and_depth() { + let compressor = + CascadingCompressor::new(vec![&NestedFailureParentScheme, &NestedFailureLeafScheme]); + let array = test_integer_array(); + + let (result, events) = record_events(|| compressor.compress(&array)); + + assert!(result.is_err()); + let event = find_event(&events, TARGET_TRACE, "scheme.compress_failed"); + assert_eq!( + event.fields.get("scheme").map(String::as_str), + Some("test.nested_failure_leaf") + ); + assert_eq!( + event.fields.get("cascade_path").map(String::as_str), + Some("test.nested_failure_parent[1]") + ); + assert_eq!( + event.fields.get("cascade_depth").map(String::as_str), + Some("1") + ); + } + + #[test] + fn sample_failure_event_includes_cascade_path_and_depth() { + let compressor = CascadingCompressor::new(vec![&SamplingFailureScheme]); + let array = test_integer_array(); + + let (result, events) = record_events(|| compressor.compress(&array)); + + assert!(result.is_err()); + let event = find_event(&events, TARGET_TRACE, "sample.compress_failed"); + assert_eq!( + event.fields.get("scheme").map(String::as_str), + Some("test.sampling_failure") + ); + assert_eq!( + event.fields.get("cascade_path").map(String::as_str), + Some("root") + ); + assert_eq!( + event.fields.get("cascade_depth").map(String::as_str), + Some("0") + ); + } + + #[test] + fn zero_byte_sample_result_omits_ratio_fields_and_selects_no_scheme() { + let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); + let array = test_integer_array(); + + let (result, events) = record_events(|| compressor.compress(&array)); + + assert!(result.is_ok()); + + let sample_event = find_event(&events, TARGET_TRACE, "sample.result"); + assert_eq!( + sample_event.fields.get("sampled_after").map(String::as_str), + Some("0") + ); + assert!(!sample_event.fields.contains_key("sampled_ratio")); + + assert!(!events.iter().any(|event| { + event.target == TARGET_TRACE + && event + .fields + .get("message") + .is_some_and(|value| value == "scheme.compress_result") + })); + } } diff --git a/vortex-compressor/src/ctx.rs b/vortex-compressor/src/ctx.rs index a488bef17bf..576fc8002a6 100644 --- a/vortex-compressor/src/ctx.rs +++ b/vortex-compressor/src/ctx.rs @@ -3,8 +3,11 @@ //! Compression context for recursive compression. +use std::fmt; + use vortex_error::VortexExpect; +use crate::compressor::ROOT_SCHEME_ID; use crate::scheme::SchemeId; use crate::stats::GenerateStatsOptions; @@ -73,6 +76,16 @@ impl CompressorContext { &self.cascade_history } + /// Returns a display wrapper for the current cascade ancestry. + pub(crate) fn cascade_path(&self) -> impl fmt::Display + '_ { + CascadePath(&self.cascade_history) + } + + /// Returns the current cascade ancestry depth. + pub(crate) fn cascade_depth(&self) -> usize { + self.cascade_history.len() + } + /// Whether cascading is exhausted (no further cascade levels allowed). /// /// This should only be used in the implementation of a [`Scheme`](crate::scheme::Scheme) if the @@ -113,3 +126,28 @@ impl CompressorContext { self } } + +/// Display wrapper for a cascade ancestry path. +struct CascadePath<'a>(&'a [(SchemeId, usize)]); + +impl fmt::Display for CascadePath<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.0.is_empty() { + return f.write_str("root"); + } + + for (index, (scheme_id, child_index)) in self.0.iter().enumerate() { + if index > 0 { + f.write_str(" > ")?; + } + + if *scheme_id == ROOT_SCHEME_ID { + write!(f, "root[{child_index}]")?; + } else { + write!(f, "{scheme_id}[{child_index}]")?; + } + } + + Ok(()) + } +} diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 957b54e57a7..22c6ffb9223 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -16,7 +16,9 @@ use crate::sample::SAMPLE_SIZE; use crate::sample::sample; use crate::sample::sample_count_approx_one_percent; use crate::scheme::Scheme; +use crate::scheme::SchemeExt; use crate::stats::ArrayAndStats; +use crate::trace; /// Closure type for [`DeferredEstimate::Callback`]. /// @@ -84,10 +86,85 @@ pub enum DeferredEstimate { Callback(Box), } -/// Returns `true` if `ratio` is a valid compression ratio (> 1.0, finite, not subnormal) that -/// beats the current best. -pub(super) fn is_better_ratio(ratio: f64, best: &Option<(&'static dyn Scheme, f64)>) -> bool { - ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 && best.is_none_or(|(_, r)| ratio > r) +/// Ranked estimate used for comparing non-terminal compression candidates. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum EstimateScore { + /// A finite compression ratio. Higher means a smaller amount of data, so it is better. + FiniteCompression(f64), + /// Trial compression produced a 0-byte output. + /// + /// This has no finite trace ratio and is not eligible for scheme selection. + /// + /// TODO(connor): A zero-byte sample usually means the sampler happened to hit an all-null + /// sample. Improve this logic so we can distinguish real zero-byte wins from sampling artifacts. + ZeroBytes, +} + +impl EstimateScore { + /// Converts measured sample sizes into a ranked estimate. + pub(super) fn from_sample_sizes(before_nbytes: u64, after_nbytes: u64) -> Self { + if after_nbytes == 0 { + Self::ZeroBytes + } else { + Self::FiniteCompression(before_nbytes as f64 / after_nbytes as f64) + } + } + + /// Returns the traceable numeric ratio, omitting the zero-byte special case. + pub(super) fn trace_ratio(self) -> Option { + match self { + Self::FiniteCompression(ratio) => Some(ratio), + Self::ZeroBytes => None, + } + } + + /// Returns whether this estimate is eligible to compete. + fn is_valid(self) -> bool { + match self { + Self::FiniteCompression(ratio) => { + ratio.is_finite() && !ratio.is_subnormal() && ratio > 1.0 + } + Self::ZeroBytes => false, + } + } + + /// Returns whether this estimate beats another valid estimate. + fn beats(self, other: Self) -> bool { + match (self, other) { + (Self::ZeroBytes, _) => false, + (Self::FiniteCompression(_), Self::ZeroBytes) => true, + (Self::FiniteCompression(ratio), Self::FiniteCompression(best_ratio)) => { + ratio > best_ratio + } + } + } +} + +/// Winner estimate carried from scheme selection into result tracing. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) enum WinnerEstimate { + /// The scheme must be used immediately. + AlwaysUse, + /// The scheme won by a ranked estimate. + Score(EstimateScore), +} + +impl WinnerEstimate { + /// Returns the traceable numeric ratio for the winning estimate. + pub(super) fn trace_ratio(self) -> Option { + match self { + Self::AlwaysUse => None, + Self::Score(score) => score.trace_ratio(), + } + } +} + +/// Returns `true` if `score` beats the current best estimate. +pub(super) fn is_better_score( + score: EstimateScore, + best: &Option<(&'static dyn Scheme, EstimateScore)>, +) -> bool { + score.is_valid() && best.is_none_or(|(_, best_score)| score.beats(best_score)) } /// Estimates compression ratio by compressing a ~1% sample of the data. @@ -103,19 +180,11 @@ pub(super) fn estimate_compression_ratio_with_sampling( compressor: &CascadingCompressor, array: &ArrayRef, ctx: CompressorContext, -) -> VortexResult { +) -> VortexResult { let sample_array = if ctx.is_sample() { array.clone() } else { - let source_len = array.len(); - let sample_count = sample_count_approx_one_percent(source_len); - - tracing::trace!( - "Sampling {} values out of {}", - SAMPLE_SIZE as u64 * sample_count as u64, - source_len - ); - + let sample_count = sample_count_approx_one_percent(array.len()); // `ArrayAndStats` expects a canonical array (so that it can easily compute lazy stats). let canonical: Canonical = sample(array, SAMPLE_SIZE, sample_count).execute(&mut compressor.execution_ctx())?; @@ -123,26 +192,27 @@ pub(super) fn estimate_compression_ratio_with_sampling( }; let mut sample_data = ArrayAndStats::new(sample_array, scheme.stats_options()); + let error_ctx = trace::enabled_error_context(&ctx); let sample_ctx = ctx.with_sampling(); - let after = scheme - .compress(compressor, &mut sample_data, sample_ctx)? - .nbytes(); - let before = sample_data.array().nbytes(); + let compressed = match scheme.compress(compressor, &mut sample_data, sample_ctx) { + Ok(compressed) => compressed, + Err(err) => { + trace::sample_compress_failed(scheme.id(), error_ctx.as_ref(), &err); + return Err(err); + } + }; - // TODO(connor): Issue https://github.com/vortex-data/vortex/issues/7268. - // if after == 0 { - // tracing::warn!( - // scheme = %scheme.id(), - // "sample compressed to 0 bytes, which should only happen for constant arrays", - // ); - // } + let after = compressed.nbytes(); + let before = sample_data.array().nbytes(); - let ratio = before as f64 / after as f64; + let score = EstimateScore::from_sample_sizes(before, after); - tracing::debug!("estimate_compression_ratio_with_sampling(compressor={scheme:#?}) = {ratio}",); + // Single DEBUG event per sampled scheme. Downstream tooling can join this with the eventual + // `scheme.compress_result` on the same scheme to compute sample-vs-full divergence. + trace::sample_result(scheme.id(), before, after, score.trace_ratio()); - Ok(ratio) + Ok(score) } impl fmt::Debug for DeferredEstimate { diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index 65fd3f09c56..9d6c7d7321f 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -15,6 +15,26 @@ //! //! This crate contains no encoding dependencies. Batteries-included compressors are provided by //! downstream crates like `vortex-btrblocks`, which register different encodings to the compressor. +//! +//! # Observability +//! +//! The compressor emits a small set of `tracing` events on a single target so you can see what +//! it's doing without attaching a profiler. +//! +//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see one line per leaf compression +//! decision. The `vortex_compressor::encode` target carries the main decision events +//! (`scheme.compress_result`, `sample.result`, and both `*.compress_failed`) plus the coarse +//! top-level `compress` span and `cascade_exhausted` event. +//! +//! The primary event is `scheme.compress_result`, which carries `scheme`, `before_nbytes`, +//! `after_nbytes`, `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 +//! bytes), `actual_ratio` (absent when the compressed output is 0 bytes), and `accepted`. +//! +//! Failure events additionally carry `cascade_path` and `cascade_depth`, so nested compression +//! errors can be tied back to the ancestor branch that triggered them. +//! +//! From those fields you can derive per-scheme savings, rejection counts, and estimator accuracy +//! with a short `jq` query. pub mod builtins; pub mod ctx; @@ -25,4 +45,5 @@ pub mod stats; mod sample; mod compressor; +mod trace; pub use compressor::CascadingCompressor; diff --git a/vortex-compressor/src/scheme.rs b/vortex-compressor/src/scheme.rs index c3f1760d373..fe1fee7a4c3 100644 --- a/vortex-compressor/src/scheme.rs +++ b/vortex-compressor/src/scheme.rs @@ -20,10 +20,10 @@ use crate::stats::GenerateStatsOptions; /// Unique identifier for a compression scheme. /// -/// The only way to obtain a [`SchemeId`] is through [`SchemeExt::id()`], which is -/// auto-implemented for all [`Scheme`] types. There is no public constructor. +/// The only way to obtain a [`SchemeId`] is through [`SchemeExt::id()`], which is auto-implemented +/// for all [`Scheme`] types. There is no public constructor. /// -/// The only exception to this is for `ROOT_SCHEME_ID` in `compressor.rs`. +/// The only exception to this is for the compressor's synthetic `ROOT_SCHEME_ID`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct SchemeId { /// Only constructable within `vortex-compressor`. diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs new file mode 100644 index 00000000000..d499f18359f --- /dev/null +++ b/vortex-compressor/src/trace.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Internal tracing helpers for compressor observability. + +use std::fmt; + +use crate::ctx::CompressorContext; +use crate::scheme::SchemeId; + +/// Shared tracing target for compressor decisions and coarse cascade structure. +pub(super) const TARGET_TRACE: &str = "vortex_compressor::encode"; + +/// Builds the top-level compression span. +#[inline] +pub(super) fn compress_span( + len: usize, + dtype: &impl fmt::Display, + before_nbytes: u64, +) -> tracing::Span { + tracing::debug_span!( + target: TARGET_TRACE, + "compress", + len, + dtype = %dtype, + before_nbytes, + after_nbytes = tracing::field::Empty, + ratio = tracing::field::Empty, + ) +} + +/// Records the final output size and, when finite, the top-level compression ratio. +#[inline] +pub(super) fn record_compress_outcome(span: &tracing::Span, before_nbytes: u64, after_nbytes: u64) { + span.record("after_nbytes", after_nbytes); + if after_nbytes != 0 { + span.record("ratio", before_nbytes as f64 / after_nbytes as f64); + } +} + +/// Emits the MAX_CASCADE short-circuit event. +#[inline] +pub(super) fn cascade_exhausted(parent: SchemeId, child_index: usize) { + tracing::debug!( + target: TARGET_TRACE, + parent = %parent, + child_index, + "cascade_exhausted", + ); +} + +/// Captures the context needed for error tracing only when ERROR logs are enabled. +#[inline] +pub(super) fn enabled_error_context(ctx: &CompressorContext) -> Option { + tracing::enabled!(target: TARGET_TRACE, tracing::Level::ERROR).then(|| ctx.clone()) +} + +/// Emits a compression-failure event for a winning scheme. +#[inline] +pub(super) fn scheme_compress_failed( + scheme: SchemeId, + before_nbytes: u64, + ctx: Option<&CompressorContext>, + err: &impl fmt::Display, +) { + if let Some(ctx) = ctx { + tracing::error!( + target: TARGET_TRACE, + scheme = %scheme, + before_nbytes, + cascade_path = %ctx.cascade_path(), + cascade_depth = ctx.cascade_depth(), + error = %err, + "scheme.compress_failed", + ); + } +} + +/// Emits the leaf compression result event. +#[inline] +#[allow( + clippy::cognitive_complexity, + reason = "tracing sometimes triggers this" +)] +pub(super) fn scheme_compress_result( + scheme: SchemeId, + before_nbytes: u64, + after_nbytes: u64, + estimated_ratio: Option, + actual_ratio: Option, + accepted: bool, +) { + match (estimated_ratio, actual_ratio) { + (Some(estimated_ratio), Some(actual_ratio)) => { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + before_nbytes, + after_nbytes, + estimated_ratio, + actual_ratio, + accepted, + "scheme.compress_result", + ); + } + (Some(estimated_ratio), None) => { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + before_nbytes, + after_nbytes, + estimated_ratio, + accepted, + "scheme.compress_result", + ); + } + (None, Some(actual_ratio)) => { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + before_nbytes, + after_nbytes, + actual_ratio, + accepted, + "scheme.compress_result", + ); + } + (None, None) => { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + before_nbytes, + after_nbytes, + accepted, + "scheme.compress_result", + ); + } + } +} + +/// Emits a sampling-failure event. +#[inline] +pub(super) fn sample_compress_failed( + scheme: SchemeId, + ctx: Option<&CompressorContext>, + err: &impl fmt::Display, +) { + if let Some(ctx) = ctx { + tracing::error!( + target: TARGET_TRACE, + scheme = %scheme, + cascade_path = %ctx.cascade_path(), + cascade_depth = ctx.cascade_depth(), + error = %err, + "sample.compress_failed", + ); + } +} + +/// Emits the sampling result event. +#[inline] +pub(super) fn sample_result( + scheme: SchemeId, + sampled_before: u64, + sampled_after: u64, + sampled_ratio: Option, +) { + if let Some(sampled_ratio) = sampled_ratio { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + sampled_before, + sampled_after, + sampled_ratio, + "sample.result", + ); + } else { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + sampled_before, + sampled_after, + "sample.result", + ); + } +} From 6d4a7f8ce25a1a7a26f3819c6435021f356c26ad Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 20 Apr 2026 21:16:50 +0100 Subject: [PATCH 149/250] deprecate: into_arrow (#7577) ## deprecated use `execute_arrow` Signed-off-by: Joe Isaacs --- encodings/sparse/src/canonical.rs | 12 +++--- vortex-array/public-api.lock | 14 +++---- .../src/arrays/constant/compute/fill_null.rs | 19 +++++---- .../src/arrays/filter/execute/varbinview.rs | 8 +++- .../src/arrays/varbin/compute/compare.rs | 4 +- vortex-array/src/arrow/datum.rs | 20 ++++++---- vortex-array/src/arrow/executor/decimal.rs | 21 +++++----- vortex-array/src/arrow/executor/list.rs | 24 +++++++++--- vortex-array/src/arrow/executor/list_view.rs | 14 +++++-- vortex-array/src/arrow/executor/struct_.rs | 39 ++++++++++++------- vortex-array/src/arrow/mod.rs | 4 ++ vortex-array/src/canonical.rs | 18 +++++++-- vortex-array/src/expr/exprs.rs | 8 ++-- vortex-array/src/scalar_fn/fns/between/mod.rs | 2 +- .../src/scalar_fn/fns/binary/boolean.rs | 23 ++++++++--- .../src/scalar_fn/fns/binary/compare.rs | 14 ++++--- vortex-array/src/scalar_fn/fns/binary/mod.rs | 26 ++++++------- .../src/scalar_fn/fns/binary/numeric.rs | 9 +++-- vortex-array/src/scalar_fn/fns/like/mod.rs | 9 +++-- vortex-array/src/scalar_fn/fns/zip/mod.rs | 15 ++++--- vortex-cxx/src/read.rs | 6 ++- vortex-jni/src/array.rs | 8 +++- vortex-python/src/arrays/mod.rs | 11 ++++-- vortex-python/src/iter/mod.rs | 7 +++- 24 files changed, 221 insertions(+), 114 deletions(-) diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 598481890cf..359b9403801 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -576,7 +576,7 @@ mod test { use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewArrayExt; - use vortex_array::arrow::IntoArrowArray as _; + use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::DecimalDType; @@ -829,7 +829,7 @@ mod test { Validity::from_mask(Mask::from_excluded_indices(10, vec![8]), Nullable), ) .into_array() - .into_arrow_preferred() + .execute_arrow(None, &mut ctx) .unwrap(); let actual = sparse_struct @@ -838,7 +838,7 @@ mod test { .execute::(&mut ctx) .unwrap() .into_array() - .into_arrow_preferred() + .execute_arrow(None, &mut ctx) .unwrap(); assert_eq!(expected.data_type(), actual.data_type()); @@ -1555,8 +1555,10 @@ mod test { // Note that the preferred arrow list representation is `List` (not `ListView`). let arrow_dtype = expected.dtype().to_arrow_dtype().unwrap(); - let actual = actual.into_arrow(&arrow_dtype).unwrap(); - let expected = expected.into_arrow(&arrow_dtype).unwrap(); + let actual = actual.execute_arrow(Some(&arrow_dtype), &mut ctx).unwrap(); + let expected = expected + .execute_arrow(Some(&arrow_dtype), &mut ctx) + .unwrap(); assert_eq!(actual.data_type(), expected.data_type()); Ok(()) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 88915460d76..1f7b1320b5c 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -6798,11 +6798,11 @@ impl vortex_array::arrow::Datum pub fn vortex_array::arrow::Datum::data_type(&self) -> &arrow_schema::datatype::DataType -pub fn vortex_array::arrow::Datum::try_new(array: &vortex_array::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_array::arrow::Datum::try_new(array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrow::Datum::try_new_array(array: &vortex_array::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_array::arrow::Datum::try_new_array(array: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrow::Datum::try_new_with_target_datatype(array: &vortex_array::ArrayRef, target_datatype: &arrow_schema::datatype::DataType) -> vortex_error::VortexResult +pub fn vortex_array::arrow::Datum::try_new_with_target_datatype(array: &vortex_array::ArrayRef, target_datatype: &arrow_schema::datatype::DataType, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl arrow_array::scalar::Datum for vortex_array::arrow::Datum @@ -15608,7 +15608,7 @@ pub fn vortex_array::scalar_fn::fns::binary::Binary::coerce_args(&self, operator pub fn vortex_array::scalar_fn::fns::binary::Binary::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::scalar_fn::fns::binary::Binary::execute(&self, op: &vortex_array::scalar_fn::fns::operators::Operator, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::scalar_fn::fns::binary::Binary::execute(&self, op: &vortex_array::scalar_fn::fns::operators::Operator, args: &dyn vortex_array::scalar_fn::ExecutionArgs, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::scalar_fn::fns::binary::Binary::fmt_sql(&self, operator: &vortex_array::scalar_fn::fns::operators::Operator, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -16252,7 +16252,7 @@ pub fn vortex_array::scalar_fn::fns::like::Like::coerce_args(&self, options: &Se pub fn vortex_array::scalar_fn::fns::like::Like::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::scalar_fn::fns::like::Like::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::scalar_fn::fns::like::Like::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::scalar_fn::fns::like::Like::fmt_sql(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -17752,7 +17752,7 @@ pub fn vortex_array::scalar_fn::fns::binary::Binary::coerce_args(&self, operator pub fn vortex_array::scalar_fn::fns::binary::Binary::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::scalar_fn::fns::binary::Binary::execute(&self, op: &vortex_array::scalar_fn::fns::operators::Operator, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::scalar_fn::fns::binary::Binary::execute(&self, op: &vortex_array::scalar_fn::fns::operators::Operator, args: &dyn vortex_array::scalar_fn::ExecutionArgs, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::scalar_fn::fns::binary::Binary::fmt_sql(&self, operator: &vortex_array::scalar_fn::fns::operators::Operator, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -18056,7 +18056,7 @@ pub fn vortex_array::scalar_fn::fns::like::Like::coerce_args(&self, options: &Se pub fn vortex_array::scalar_fn::fns::like::Like::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::scalar_fn::fns::like::Like::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::scalar_fn::fns::like::Like::execute(&self, options: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::scalar_fn::fns::like::Like::fmt_sql(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result diff --git a/vortex-array/src/arrays/constant/compute/fill_null.rs b/vortex-array/src/arrays/constant/compute/fill_null.rs index 22fe70c246d..207f908b5cc 100644 --- a/vortex-array/src/arrays/constant/compute/fill_null.rs +++ b/vortex-array/src/arrays/constant/compute/fill_null.rs @@ -22,8 +22,10 @@ impl FillNullReduce for Constant { #[cfg(test)] mod test { use crate::IntoArray as _; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; - use crate::arrow::IntoArrowArray as _; + use crate::arrow::ArrowArrayExecutor; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::Nullability; @@ -32,6 +34,7 @@ mod test { #[test] fn test_null() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let actual = ConstantArray::new(Scalar::null_native::(), 3) .into_array() .fill_null(Scalar::from(1)) @@ -40,8 +43,8 @@ mod test { assert!(!actual.dtype().is_nullable()); - let actual_arrow = actual.clone().into_arrow_preferred().unwrap(); - let expected_arrow = expected.clone().into_arrow_preferred().unwrap(); + let actual_arrow = actual.clone().execute_arrow(None, &mut ctx).unwrap(); + let expected_arrow = expected.clone().execute_arrow(None, &mut ctx).unwrap(); assert_eq!( &actual_arrow, &expected_arrow, @@ -53,6 +56,7 @@ mod test { #[test] fn test_non_null() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let actual = ConstantArray::new(Scalar::from(Some(1)), 3) .into_array() .fill_null(Scalar::from(1)) @@ -61,8 +65,8 @@ mod test { assert!(!actual.dtype().is_nullable()); - let actual_arrow = actual.clone().into_arrow_preferred().unwrap(); - let expected_arrow = expected.clone().into_arrow_preferred().unwrap(); + let actual_arrow = actual.clone().execute_arrow(None, &mut ctx).unwrap(); + let expected_arrow = expected.clone().execute_arrow(None, &mut ctx).unwrap(); assert_eq!( &actual_arrow, &expected_arrow, @@ -74,6 +78,7 @@ mod test { #[test] fn test_non_nullable_with_nullable() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let actual = ConstantArray::new(Scalar::from(1), 3) .into_array() .fill_null(Scalar::new( @@ -87,8 +92,8 @@ mod test { assert!(actual.dtype().is_nullable()); - let actual_arrow = actual.clone().into_arrow_preferred().unwrap(); - let expected_arrow = expected.clone().into_arrow_preferred().unwrap(); + let actual_arrow = actual.clone().execute_arrow(None, &mut ctx).unwrap(); + let expected_arrow = expected.clone().execute_arrow(None, &mut ctx).unwrap(); assert_eq!( &actual_arrow, &expected_arrow, diff --git a/vortex-array/src/arrays/filter/execute/varbinview.rs b/vortex-array/src/arrays/filter/execute/varbinview.rs index 166e51275d8..6c0bb052bc2 100644 --- a/vortex-array/src/arrays/filter/execute/varbinview.rs +++ b/vortex-array/src/arrays/filter/execute/varbinview.rs @@ -10,10 +10,12 @@ use vortex_mask::MaskValues; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; +use crate::arrow::ArrowArrayExecutor; use crate::arrow::FromArrowArray; -use crate::arrow::IntoArrowArray; pub fn filter_varbinview(array: &VarBinViewArray, mask: &Arc) -> VarBinViewArray { // Delegate to the Arrow implementation of filter over `VarBinView`. @@ -29,7 +31,9 @@ fn arrow_filter_fn(array: &ArrayRef, mask: &Mask) -> vortex_error::VortexResult< Mask::AllTrue(_) | Mask::AllFalse(_) => unreachable!("check in filter invoke"), }; - let array_ref = array.clone().into_arrow_preferred()?; + let array_ref = array + .clone() + .execute_arrow(None, &mut LEGACY_SESSION.create_execution_ctx())?; let mask_array = BooleanArray::new(values.bit_buffer().clone().into(), None); let filtered = arrow_select::filter::filter(array_ref.as_ref(), &mask_array)?; diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 63fe1a622d0..511f930d3d0 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -80,10 +80,10 @@ impl CompareKernel for VarBin { )); } - let lhs = Datum::try_new(lhs.array())?; + let lhs = Datum::try_new(lhs.array(), ctx)?; // Use StringViewArray/BinaryViewArray to match the Utf8View/BinaryView types - // produced by Datum::try_new (which uses into_arrow_preferred()) + // produced by Datum::try_new (which uses execute_arrow(None, ctx)) let arrow_rhs: &dyn arrow_array::Datum = match rhs_const.dtype() { DType::Utf8(_) => &rhs_const .as_utf8() diff --git a/vortex-array/src/arrow/datum.rs b/vortex-array/src/arrow/datum.rs index 60a3ecacd42..66c457d3727 100644 --- a/vortex-array/src/arrow/datum.rs +++ b/vortex-array/src/arrow/datum.rs @@ -15,8 +15,9 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrow::ArrowArrayExecutor; use crate::arrow::FromArrowArray; -use crate::arrow::IntoArrowArray; +use crate::executor::ExecutionCtx; /// A wrapper around a generic Arrow array that can be used as a Datum in Arrow compute. #[derive(Debug)] @@ -27,15 +28,15 @@ pub struct Datum { impl Datum { /// Create a new [`Datum`] from an [`ArrayRef`], which can then be passed to Arrow compute. - pub fn try_new(array: &ArrayRef) -> VortexResult { + pub fn try_new(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { if array.is::() { Ok(Self { - array: array.slice(0..1)?.into_arrow_preferred()?, + array: array.slice(0..1)?.execute_arrow(None, ctx)?, is_scalar: true, }) } else { Ok(Self { - array: array.clone().into_arrow_preferred()?, + array: array.clone().execute_arrow(None, ctx)?, is_scalar: false, }) } @@ -43,9 +44,9 @@ impl Datum { /// Create a new [`Datum`] from an `DynArray`, which can then be passed to Arrow compute. /// This not try and convert the array to a scalar if it is constant. - pub fn try_new_array(array: &ArrayRef) -> VortexResult { + pub fn try_new_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { Ok(Self { - array: array.clone().into_arrow_preferred()?, + array: array.clone().execute_arrow(None, ctx)?, is_scalar: false, }) } @@ -53,15 +54,18 @@ impl Datum { pub fn try_new_with_target_datatype( array: &ArrayRef, target_datatype: &DataType, + ctx: &mut ExecutionCtx, ) -> VortexResult { if array.is::() { Ok(Self { - array: array.slice(0..1)?.into_arrow(target_datatype)?, + array: array + .slice(0..1)? + .execute_arrow(Some(target_datatype), ctx)?, is_scalar: true, }) } else { Ok(Self { - array: array.clone().into_arrow(target_datatype)?, + array: array.clone().execute_arrow(Some(target_datatype), ctx)?, is_scalar: false, }) } diff --git a/vortex-array/src/arrow/executor/decimal.rs b/vortex-array/src/arrow/executor/decimal.rs index 9d7caf0ebda..077495354cc 100644 --- a/vortex-array/src/arrow/executor/decimal.rs +++ b/vortex-array/src/arrow/executor/decimal.rs @@ -226,7 +226,6 @@ mod tests { use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrow::ArrowArrayExecutor; - use crate::arrow::IntoArrowArray; use crate::arrow::executor::decimal::DecimalArray; use crate::builders::ArrayBuilder; use crate::builders::DecimalBuilder; @@ -236,16 +235,16 @@ mod tests { #[test] fn decimal_to_arrow() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Make a very simple i128 and i256 array. let decimal_vortex = DecimalArray::new( buffer![1i128, 2i128, 3i128, 4i128, 5i128], DecimalDType::new(19, 2), Validity::NonNullable, ); - let arrow = decimal_vortex.into_array().execute_arrow( - Some(&DataType::Decimal128(19, 2)), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; + let arrow = decimal_vortex + .into_array() + .execute_arrow(Some(&DataType::Decimal128(19, 2)), &mut ctx)?; assert_eq!(arrow.data_type(), &DataType::Decimal128(19, 2)); let decimal_array = arrow.as_any().downcast_ref::().unwrap(); assert_eq!( @@ -265,13 +264,14 @@ mod tests { fn test_to_arrow_decimal128( #[case] _decimal_type: T, ) -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut decimal = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); decimal.append_value(10); decimal.append_value(11); decimal.append_value(12); let decimal = decimal.finish(); - let arrow_array = decimal.into_arrow(&DataType::Decimal128(2, 1))?; + let arrow_array = decimal.execute_arrow(Some(&DataType::Decimal128(2, 1)), &mut ctx)?; let arrow_decimal = arrow_array .as_any() .downcast_ref::() @@ -292,13 +292,14 @@ mod tests { fn test_to_arrow_decimal32(#[case] _decimal_type: T) -> VortexResult<()> { use arrow_array::Decimal32Array; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut decimal = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); decimal.append_value(10); decimal.append_value(11); decimal.append_value(12); let decimal = decimal.finish(); - let arrow_array = decimal.into_arrow(&DataType::Decimal32(2, 1))?; + let arrow_array = decimal.execute_arrow(Some(&DataType::Decimal32(2, 1)), &mut ctx)?; let arrow_decimal = arrow_array .as_any() .downcast_ref::() @@ -319,13 +320,14 @@ mod tests { fn test_to_arrow_decimal64(#[case] _decimal_type: T) -> VortexResult<()> { use arrow_array::Decimal64Array; + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut decimal = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); decimal.append_value(10); decimal.append_value(11); decimal.append_value(12); let decimal = decimal.finish(); - let arrow_array = decimal.into_arrow(&DataType::Decimal64(2, 1))?; + let arrow_array = decimal.execute_arrow(Some(&DataType::Decimal64(2, 1)), &mut ctx)?; let arrow_decimal = arrow_array .as_any() .downcast_ref::() @@ -346,13 +348,14 @@ mod tests { fn test_to_arrow_decimal256( #[case] _decimal_type: T, ) -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut decimal = DecimalBuilder::new::(DecimalDType::new(2, 1), false.into()); decimal.append_value(10); decimal.append_value(11); decimal.append_value(12); let decimal = decimal.finish(); - let arrow_array = decimal.into_arrow(&DataType::Decimal256(2, 1))?; + let arrow_array = decimal.execute_arrow(Some(&DataType::Decimal256(2, 1)), &mut ctx)?; let arrow_decimal = arrow_array .as_any() .downcast_ref::() diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index bb0d4fd65c7..6c7c271e933 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -206,8 +206,10 @@ mod tests { use crate::Canonical; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; - use crate::arrow::IntoArrowArray; + use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::list::ListViewArray; use crate::dtype::DType; use crate::dtype::Nullability::NonNullable; @@ -215,6 +217,7 @@ mod tests { #[test] fn test_to_arrow_list_i32() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a ListViewArray with i32 elements: [[1, 2, 3], [4, 5]] let elements = PrimitiveArray::new(buffer![1i32, 2, 3, 4, 5], Validity::NonNullable); let offsets = PrimitiveArray::new(buffer![0i32, 3], Validity::NonNullable); @@ -233,7 +236,9 @@ mod tests { // Convert to Arrow List with i32 offsets. let field = Field::new("item", DataType::Int32, false); let arrow_dt = DataType::List(field.into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; // Verify the type is correct. assert_eq!(arrow_array.data_type(), &arrow_dt); @@ -267,6 +272,7 @@ mod tests { #[test] fn test_to_arrow_list_i64() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a ListViewArray with i64 offsets: [[10, 20], [30]] let elements = PrimitiveArray::new(buffer![10i64, 20, 30], Validity::NonNullable); let offsets = PrimitiveArray::new(buffer![0i64, 2], Validity::NonNullable); @@ -285,7 +291,9 @@ mod tests { // Convert to Arrow LargeList with i64 offsets. let field = Field::new("item", DataType::Int64, false); let arrow_dt = DataType::LargeList(field.into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; // Verify the type is correct. assert_eq!(arrow_array.data_type(), &arrow_dt); @@ -304,6 +312,7 @@ mod tests { #[test] fn test_to_arrow_list_non_zctl() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Overlapping lists are NOT zero-copy-to-list, so this exercises the rebuild path. // Elements: [1, 2, 3, 4], List 0: [1,2,3], List 1: [2,3,4] (overlap at indices 1-2) let elements = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable); @@ -320,7 +329,9 @@ mod tests { let field = Field::new("item", DataType::Int32, false); let arrow_dt = DataType::List(field.into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; let list = arrow_array .as_any() @@ -343,6 +354,7 @@ mod tests { #[test] fn test_to_arrow_list_empty_zctl() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let dtype = DType::List( Arc::new(DType::Primitive(crate::dtype::PType::I32, NonNullable)), NonNullable, @@ -354,7 +366,9 @@ mod tests { }; let arrow_dt = DataType::List(Field::new("item", DataType::Int32, false).into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; assert_eq!(arrow_array.len(), 0); Ok(()) } diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index bf6d0da0b8b..a6795b1adaa 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -89,13 +89,16 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; - use crate::arrow::IntoArrowArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; + use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::list_view::ListViewArray; use crate::arrow::executor::list_view::PrimitiveArray; use crate::validity::Validity; #[test] fn test_to_arrow_listview_i32() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a ListViewArray with overlapping views: [[1, 2], [2, 3], [3, 4]] let elements = PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable); let offsets = PrimitiveArray::new(buffer![0i32, 1, 2], Validity::NonNullable); @@ -111,7 +114,9 @@ mod tests { // Convert to Arrow ListView with i32 offsets. let field = Field::new("item", DataType::Int32, false); let arrow_dt = DataType::ListView(field.into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; // Verify the type is correct. assert_eq!(arrow_array.data_type(), &arrow_dt); @@ -148,6 +153,7 @@ mod tests { #[test] fn test_to_arrow_listview_i64() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a ListViewArray with nullable elements: [[100], null, [200, 300]] let elements = PrimitiveArray::new(buffer![100i64, 200, 300], Validity::NonNullable); let offsets = PrimitiveArray::new(buffer![0i64, 1, 1], Validity::NonNullable); @@ -167,7 +173,9 @@ mod tests { // Convert to Arrow LargeListView with i64 offsets. let field = Field::new("item", DataType::Int64, false); let arrow_dt = DataType::LargeListView(field.into()); - let arrow_array = list_array.into_array().into_arrow(&arrow_dt)?; + let arrow_array = list_array + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; // Verify the type is correct. assert_eq!(arrow_array.data_type(), &arrow_dt); diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index a85005c0a20..88b62abd4c9 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -190,6 +190,7 @@ fn create_from_fields( mod tests { use std::sync::Arc; + use arrays::varbinview::VarBinViewArray; use arrow_array::ArrayRef; use arrow_array::PrimitiveArray as ArrowPrimitiveArray; use arrow_array::StringViewArray; @@ -210,12 +211,12 @@ mod tests { use crate::arrays::StructArray; use crate::arrow::ArrowArrayExecutor; use crate::arrow::FromArrowArray; - use crate::arrow::IntoArrowArray; use crate::dtype::FieldNames; use crate::validity::Validity; #[test] fn struct_nullable_non_null_to_arrow() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let xs = PrimitiveArray::new(buffer![0i64, 1, 2, 3, 4], Validity::AllValid); let struct_a = StructArray::try_new( @@ -228,12 +229,15 @@ mod tests { let fields = vec![Field::new("xs", DataType::Int64, false)]; let arrow_dt = DataType::Struct(fields.into()); - struct_a.into_array().into_arrow(&arrow_dt)?; + struct_a + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx)?; Ok(()) } #[test] fn struct_nullable_with_nulls_to_arrow() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let xs = PrimitiveArray::from_option_iter(vec![Some(0_i64), Some(1), Some(2), None, Some(3)]); @@ -247,12 +251,18 @@ mod tests { let fields = vec![Field::new("xs", DataType::Int64, false)]; let arrow_dt = DataType::Struct(fields.into()); - assert!(struct_a.into_array().into_arrow(&arrow_dt).is_err()); + assert!( + struct_a + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx) + .is_err() + ); Ok(()) } #[test] fn struct_to_arrow_with_schema_mismatch() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let xs = PrimitiveArray::new(buffer![0i64, 1, 2, 3, 4], Validity::AllValid); let struct_a = StructArray::try_new( @@ -268,7 +278,11 @@ mod tests { ]; let arrow_dt = DataType::Struct(fields.into()); - let err = struct_a.into_array().into_arrow(&arrow_dt).err().unwrap(); + let err = struct_a + .into_array() + .execute_arrow(Some(&arrow_dt), &mut ctx) + .err() + .unwrap(); assert!( err.to_string() .contains("StructArray has 1 fields, but target Arrow type has 2 fields") @@ -278,6 +292,7 @@ mod tests { #[test] fn test_to_arrow() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = StructArray::from_fields( vec![ ( @@ -286,8 +301,7 @@ mod tests { ), ( "b", - arrays::varbinview::VarBinViewArray::from_iter_str(vec!["a", "b", "c"]) - .into_array(), + VarBinViewArray::from_iter_str(vec!["a", "b", "c"]).into_array(), ), ] .as_slice(), @@ -311,10 +325,9 @@ mod tests { let arrow_dtype = array.dtype().to_arrow_dtype()?; assert_eq!( - &array.into_array().execute_arrow( - Some(&arrow_dtype), - &mut LEGACY_SESSION.create_execution_ctx() - )?, + &array + .into_array() + .execute_arrow(Some(&arrow_dtype), &mut ctx)?, &arrow_array ); Ok(()) @@ -322,6 +335,7 @@ mod tests { #[test] fn to_arrow_with_non_nullable_fields() -> VortexResult<()> { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = StructArray::from_fields( vec![ ( @@ -330,14 +344,13 @@ mod tests { ), ( "b", - arrays::varbinview::VarBinViewArray::from_iter_str(vec!["a", "b", "c"]) - .into_array(), + VarBinViewArray::from_iter_str(vec!["a", "b", "c"]).into_array(), ), ] .as_slice(), )?; let orig_dtype = array.dtype().clone(); - let arrow_array = array.into_array().into_arrow_preferred()?; + let arrow_array = array.into_array().execute_arrow(None, &mut ctx)?; let from_arrow = array::ArrayRef::from_arrow(arrow_array.as_ref(), false)?; assert_eq!(&orig_dtype, from_arrow.dtype()); Ok(()) diff --git a/vortex-array/src/arrow/mod.rs b/vortex-array/src/arrow/mod.rs index bcc9e6094a1..efc83aa6af6 100644 --- a/vortex-array/src/arrow/mod.rs +++ b/vortex-array/src/arrow/mod.rs @@ -30,12 +30,16 @@ pub trait FromArrowArray { Self: Sized; } +#[deprecated(note = "Use `execute_arrow(None, ctx)` or `execute_arrow(Some(dt), ctx)` instead")] pub trait IntoArrowArray { + #[deprecated(note = "Use `execute_arrow(None, ctx)` instead")] fn into_arrow_preferred(self) -> VortexResult; + #[deprecated(note = "Use `execute_arrow(Some(data_type), ctx)` instead")] fn into_arrow(self, data_type: &DataType) -> VortexResult; } +#[allow(deprecated)] impl IntoArrowArray for ArrayRef { /// Convert this [`crate::ArrayRef`] into an Arrow [`crate::ArrayRef`] by using the array's /// preferred (cheapest) Arrow [`DataType`]. diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index e8edd0488fc..3190c462691 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -1067,13 +1067,16 @@ mod test { use crate::ArrayRef; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; + use crate::arrow::ArrowArrayExecutor; use crate::arrow::FromArrowArray; - use crate::arrow::IntoArrowArray; use crate::canonical::StructArray; #[test] fn test_canonicalize_nested_struct() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); // Create a struct array with multiple internal components. let nested_struct_array = StructArray::from_fields(&[ ("a", buffer![1u64].into_array()), @@ -1095,7 +1098,7 @@ mod test { let arrow_struct = nested_struct_array .into_array() - .into_arrow_preferred() + .execute_arrow(None, &mut ctx) .unwrap() .as_any() .downcast_ref::() @@ -1130,6 +1133,7 @@ mod test { #[test] fn roundtrip_struct() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let mut nulls = NullBufferBuilder::new(6); nulls.append_n_non_nulls(4); nulls.append_null(); @@ -1165,12 +1169,16 @@ mod test { assert_eq!( &arrow_struct, - vortex_struct.into_arrow_preferred().unwrap().as_struct() + vortex_struct + .execute_arrow(None, &mut ctx) + .unwrap() + .as_struct() ); } #[test] fn roundtrip_list() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let names = Arc::new(StringArray::from_iter(vec![ Some("Joseph"), Some("Angela"), @@ -1187,7 +1195,9 @@ mod test { let vortex_list = ArrayRef::from_arrow(&arrow_list, true).unwrap(); - let rt_arrow_list = vortex_list.into_arrow(list_data_type).unwrap(); + let rt_arrow_list = vortex_list + .execute_arrow(Some(list_data_type), &mut ctx) + .unwrap(); assert_eq!( (Arc::new(arrow_list.clone()) as ArrowArrayRef).as_ref(), diff --git a/vortex-array/src/expr/exprs.rs b/vortex-array/src/expr/exprs.rs index fb8d2f0cb77..13821669034 100644 --- a/vortex-array/src/expr/exprs.rs +++ b/vortex-array/src/expr/exprs.rs @@ -397,17 +397,19 @@ where /// /// ``` /// # use vortex_array::IntoArray; -/// # use vortex_array::arrow::IntoArrowArray as _; +/// # use vortex_array::arrow::ArrowArrayExecutor; +/// # use vortex_array::{LEGACY_SESSION, VortexSessionExecute}; /// # use vortex_buffer::buffer; /// # use vortex_array::expr::{checked_add, lit, root}; /// let xs = buffer![1, 2, 3].into_array(); /// let result = xs.apply(&checked_add(root(), lit(5))).unwrap(); /// +/// let mut ctx = LEGACY_SESSION.create_execution_ctx(); /// assert_eq!( -/// &result.into_arrow_preferred().unwrap(), +/// &result.execute_arrow(None, &mut ctx).unwrap(), /// &buffer![6, 7, 8] /// .into_array() -/// .into_arrow_preferred() +/// .execute_arrow(None, &mut ctx) /// .unwrap() /// ); /// ``` diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index 3384f5faf17..bb44d71c846 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -155,7 +155,7 @@ fn between_canonical( upper.clone(), Operator::from(options.upper_strict.to_compare_operator()), )?; - execute_boolean(&lower_cmp, &upper_cmp, Operator::And) + execute_boolean(&lower_cmp, &upper_cmp, Operator::And, ctx) } /// An optimized scalar expression to compute whether values fall between two bounds. diff --git a/vortex-array/src/scalar_fn/fns/binary/boolean.rs b/vortex-array/src/scalar_fn/fns/binary/boolean.rs index 10d37202f64..68ef0d61156 100644 --- a/vortex-array/src/scalar_fn/fns/binary/boolean.rs +++ b/vortex-array/src/scalar_fn/fns/binary/boolean.rs @@ -10,10 +10,11 @@ use crate::ArrayRef; use crate::IntoArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrow::ArrowArrayExecutor; use crate::arrow::FromArrowArray; -use crate::arrow::IntoArrowArray; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; +use crate::executor::ExecutionCtx; use crate::scalar::Scalar; use crate::scalar_fn::fns::operators::Operator; @@ -37,19 +38,31 @@ pub(crate) fn execute_boolean( lhs: &ArrayRef, rhs: &ArrayRef, op: Operator, + ctx: &mut ExecutionCtx, ) -> VortexResult { if let Some(result) = constant_boolean(lhs, rhs, op)? { return Ok(result); } - arrow_execute_boolean(lhs.clone(), rhs.clone(), op) + arrow_execute_boolean(lhs.clone(), rhs.clone(), op, ctx) } /// Arrow implementation for Kleene boolean operations using [`Operator`]. -fn arrow_execute_boolean(lhs: ArrayRef, rhs: ArrayRef, op: Operator) -> VortexResult { +fn arrow_execute_boolean( + lhs: ArrayRef, + rhs: ArrayRef, + op: Operator, + ctx: &mut ExecutionCtx, +) -> VortexResult { let nullable = lhs.dtype().is_nullable() || rhs.dtype().is_nullable(); - let lhs = lhs.into_arrow(&DataType::Boolean)?.as_boolean().clone(); - let rhs = rhs.into_arrow(&DataType::Boolean)?.as_boolean().clone(); + let lhs = lhs + .execute_arrow(Some(&DataType::Boolean), ctx)? + .as_boolean() + .clone(); + let rhs = rhs + .execute_arrow(Some(&DataType::Boolean), ctx)? + .as_boolean() + .clone(); let array = match op { Operator::And => arrow_arith::boolean::and_kleene(&lhs, &rhs)?, diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 2b9d971006d..4c2a25258b0 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -23,8 +23,8 @@ use crate::arrays::ScalarFnVTable; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; +use crate::arrow::ArrowArrayExecutor; use crate::arrow::Datum; -use crate::arrow::IntoArrowArray; use crate::arrow::from_arrow_array_with_len; use crate::dtype::DType; use crate::dtype::Nullability; @@ -114,6 +114,7 @@ pub(crate) fn execute_compare( lhs: &ArrayRef, rhs: &ArrayRef, op: CompareOperator, + ctx: &mut ExecutionCtx, ) -> VortexResult { let nullable = lhs.dtype().is_nullable() || rhs.dtype().is_nullable(); @@ -136,7 +137,7 @@ pub(crate) fn execute_compare( return Ok(ConstantArray::new(result, lhs.len()).into_array()); } - arrow_compare_arrays(lhs, rhs, op) + arrow_compare_arrays(lhs, rhs, op, ctx) } /// Fall back to Arrow for comparison. @@ -144,6 +145,7 @@ fn arrow_compare_arrays( left: &ArrayRef, right: &ArrayRef, operator: CompareOperator, + ctx: &mut ExecutionCtx, ) -> VortexResult { assert_eq!(left.len(), right.len()); @@ -152,8 +154,8 @@ fn arrow_compare_arrays( // Arrow's vectorized comparison kernels don't support nested types. // For nested types, fall back to `make_comparator` which does element-wise comparison. let arrow_array: BooleanArray = if left.dtype().is_nested() || right.dtype().is_nested() { - let rhs = right.clone().into_arrow_preferred()?; - let lhs = left.clone().into_arrow(rhs.data_type())?; + let rhs = right.clone().execute_arrow(None, ctx)?; + let lhs = left.clone().execute_arrow(Some(rhs.data_type()), ctx)?; assert!( lhs.data_type().equals_datatype(rhs.data_type()), @@ -165,8 +167,8 @@ fn arrow_compare_arrays( compare_nested_arrow_arrays(lhs.as_ref(), rhs.as_ref(), operator)? } else { // Fast path: use vectorized kernels for primitive types. - let lhs = Datum::try_new(left)?; - let rhs = Datum::try_new_with_target_datatype(right, lhs.data_type())?; + let lhs = Datum::try_new(left, ctx)?; + let rhs = Datum::try_new_with_target_datatype(right, lhs.data_type(), ctx)?; match operator { CompareOperator::Eq => cmp::eq(&lhs, &rhs)?, diff --git a/vortex-array/src/scalar_fn/fns/binary/mod.rs b/vortex-array/src/scalar_fn/fns/binary/mod.rs index af81ef0ca28..374d34d1ec9 100644 --- a/vortex-array/src/scalar_fn/fns/binary/mod.rs +++ b/vortex-array/src/scalar_fn/fns/binary/mod.rs @@ -142,24 +142,24 @@ impl ScalarFnVTable for Binary { &self, op: &Operator, args: &dyn ExecutionArgs, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let lhs = args.get(0)?; let rhs = args.get(1)?; match op { - Operator::Eq => execute_compare(&lhs, &rhs, CompareOperator::Eq), - Operator::NotEq => execute_compare(&lhs, &rhs, CompareOperator::NotEq), - Operator::Lt => execute_compare(&lhs, &rhs, CompareOperator::Lt), - Operator::Lte => execute_compare(&lhs, &rhs, CompareOperator::Lte), - Operator::Gt => execute_compare(&lhs, &rhs, CompareOperator::Gt), - Operator::Gte => execute_compare(&lhs, &rhs, CompareOperator::Gte), - Operator::And => execute_boolean(&lhs, &rhs, Operator::And), - Operator::Or => execute_boolean(&lhs, &rhs, Operator::Or), - Operator::Add => execute_numeric(&lhs, &rhs, NumericOperator::Add), - Operator::Sub => execute_numeric(&lhs, &rhs, NumericOperator::Sub), - Operator::Mul => execute_numeric(&lhs, &rhs, NumericOperator::Mul), - Operator::Div => execute_numeric(&lhs, &rhs, NumericOperator::Div), + Operator::Eq => execute_compare(&lhs, &rhs, CompareOperator::Eq, ctx), + Operator::NotEq => execute_compare(&lhs, &rhs, CompareOperator::NotEq, ctx), + Operator::Lt => execute_compare(&lhs, &rhs, CompareOperator::Lt, ctx), + Operator::Lte => execute_compare(&lhs, &rhs, CompareOperator::Lte, ctx), + Operator::Gt => execute_compare(&lhs, &rhs, CompareOperator::Gt, ctx), + Operator::Gte => execute_compare(&lhs, &rhs, CompareOperator::Gte, ctx), + Operator::And => execute_boolean(&lhs, &rhs, Operator::And, ctx), + Operator::Or => execute_boolean(&lhs, &rhs, Operator::Or, ctx), + Operator::Add => execute_numeric(&lhs, &rhs, NumericOperator::Add, ctx), + Operator::Sub => execute_numeric(&lhs, &rhs, NumericOperator::Sub, ctx), + Operator::Mul => execute_numeric(&lhs, &rhs, NumericOperator::Mul, ctx), + Operator::Div => execute_numeric(&lhs, &rhs, NumericOperator::Div, ctx), } } diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric.rs b/vortex-array/src/scalar_fn/fns/binary/numeric.rs index d48b3e230ff..9429a7310d1 100644 --- a/vortex-array/src/scalar_fn/fns/binary/numeric.rs +++ b/vortex-array/src/scalar_fn/fns/binary/numeric.rs @@ -9,6 +9,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrow::Datum; use crate::arrow::from_arrow_array_with_len; +use crate::executor::ExecutionCtx; use crate::scalar::NumericOperator; /// Execute a numeric operation between two arrays. @@ -19,11 +20,12 @@ pub(crate) fn execute_numeric( lhs: &ArrayRef, rhs: &ArrayRef, op: NumericOperator, + ctx: &mut ExecutionCtx, ) -> VortexResult { if let Some(result) = constant_numeric(lhs, rhs, op)? { return Ok(result); } - arrow_numeric(lhs, rhs, op) + arrow_numeric(lhs, rhs, op, ctx) } /// Implementation of numeric operations using the Arrow crate. @@ -31,12 +33,13 @@ pub(crate) fn arrow_numeric( lhs: &ArrayRef, rhs: &ArrayRef, operator: NumericOperator, + ctx: &mut ExecutionCtx, ) -> VortexResult { let nullable = lhs.dtype().is_nullable() || rhs.dtype().is_nullable(); let len = lhs.len(); - let left = Datum::try_new(lhs)?; - let right = Datum::try_new_with_target_datatype(rhs, left.data_type())?; + let left = Datum::try_new(lhs, ctx)?; + let right = Datum::try_new_with_target_datatype(rhs, left.data_type(), ctx)?; let array = match operator { NumericOperator::Add => arrow_arith::numeric::add(&left, &right)?, diff --git a/vortex-array/src/scalar_fn/fns/like/mod.rs b/vortex-array/src/scalar_fn/fns/like/mod.rs index 2288812b940..2e908ca17bd 100644 --- a/vortex-array/src/scalar_fn/fns/like/mod.rs +++ b/vortex-array/src/scalar_fn/fns/like/mod.rs @@ -140,12 +140,12 @@ impl ScalarFnVTable for Like { &self, options: &Self::Options, args: &dyn ExecutionArgs, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { let child = args.get(0)?; let pattern = args.get(1)?; - arrow_like(&child, &pattern, *options) + arrow_like(&child, &pattern, *options, ctx) } fn validity( @@ -206,6 +206,7 @@ pub(crate) fn arrow_like( array: &ArrayRef, pattern: &ArrayRef, options: LikeOptions, + ctx: &mut ExecutionCtx, ) -> VortexResult { let nullable = array.dtype().is_nullable() | pattern.dtype().is_nullable(); let len = array.len(); @@ -217,8 +218,8 @@ pub(crate) fn arrow_like( ); // convert the pattern to the preferred array datatype - let lhs = Datum::try_new(array)?; - let rhs = Datum::try_new_with_target_datatype(pattern, lhs.data_type())?; + let lhs = Datum::try_new(array, ctx)?; + let rhs = Datum::try_new_with_target_datatype(pattern, lhs.data_type(), ctx)?; let result = match (options.negated, options.case_insensitive) { (false, false) => arrow_string::like::like(&lhs, &rhs)?, diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 5277e4bf437..ddb662459a7 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -238,7 +238,7 @@ mod tests { use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::VarBinView; - use crate::arrow::IntoArrowArray; + use crate::arrow::ArrowArrayExecutor; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::BufferGrowthStrategy; @@ -444,17 +444,22 @@ mod tests { let zipped = zipped.as_opt::().unwrap(); assert_eq!(zipped.data_buffers().len(), 2); + let mut arrow_ctx = LEGACY_SESSION.create_execution_ctx(); let expected = arrow_zip( mask.into_array() - .into_arrow_preferred() + .execute_arrow(None, &mut arrow_ctx) .unwrap() .as_boolean(), - &if_true.into_arrow_preferred().unwrap(), - &if_false.into_arrow_preferred().unwrap(), + &if_true.execute_arrow(None, &mut arrow_ctx).unwrap(), + &if_false.execute_arrow(None, &mut arrow_ctx).unwrap(), ) .unwrap(); - let actual = zipped.array().clone().into_arrow_preferred().unwrap(); + let actual = zipped + .array() + .clone() + .execute_arrow(None, &mut arrow_ctx) + .unwrap(); assert_eq!(actual.as_ref(), expected.as_ref()); } } diff --git a/vortex-cxx/src/read.rs b/vortex-cxx/src/read.rs index 1ef4aec2734..ac8ccf03466 100644 --- a/vortex-cxx/src/read.rs +++ b/vortex-cxx/src/read.rs @@ -15,7 +15,9 @@ use arrow_schema::Schema; use arrow_schema::SchemaRef; use futures::stream::TryStreamExt; use vortex::array::ArrayRef; -use vortex::array::arrow::IntoArrowArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowArrayExecutor; use vortex::buffer::Buffer; use vortex::file::OpenOptionsSessionExt; use vortex::io::runtime::BlockingRuntime; @@ -163,7 +165,7 @@ pub(crate) fn scan_builder_into_threadsafe_cloneable_reader( let stream = builder .inner .map(move |b| { - b.into_arrow(&data_type) + b.execute_arrow(Some(&data_type), &mut LEGACY_SESSION.create_execution_ctx()) .map(|struct_array| RecordBatch::from(struct_array.as_struct())) }) .into_stream()? diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index af74bf71ac7..18af64bd2c8 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -28,6 +28,7 @@ use jni::sys::jshort; use jni::sys::jstring; use vortex::array::ArrayRef; use vortex::array::ArrayView; +use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrays::ExtensionArray; use vortex::array::arrays::StructArray; @@ -36,7 +37,7 @@ use vortex::array::arrays::VarBinView; use vortex::array::arrays::extension::ExtensionArrayExt; use vortex::array::arrays::struct_::StructArrayExt; use vortex::array::arrays::varbin::VarBinArrayExt; -use vortex::array::arrow::IntoArrowArray; +use vortex::array::arrow::ArrowArrayExecutor; use vortex::dtype::DType; use vortex::dtype::i256; use vortex::error::VortexError; @@ -117,7 +118,10 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_exportToArrow<'loc let preferred_arrow_type = array_ref.inner.dtype().to_arrow_dtype()?; let viewless_arrow_type = data_type_no_views(preferred_arrow_type); - let arrow_array = array_ref.inner.clone().into_arrow(&viewless_arrow_type)?; + let arrow_array = array_ref.inner.clone().execute_arrow( + Some(&viewless_arrow_type), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let (ffi_array, ffi_schema) = arrow_array::ffi::to_ffi(&arrow_array.to_data()).map_err(VortexError::from)?; diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 28ec1ac2020..3ad3d530ef1 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -31,7 +31,7 @@ use vortex::array::arrays::BoolArray; use vortex::array::arrays::Chunked; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::chunked::ChunkedArrayExt; -use vortex::array::arrow::IntoArrowArray; +use vortex::array::arrow::ArrowArrayExecutor; use vortex::array::builtins::ArrayBuiltins; use vortex::array::match_each_integer_ptype; use vortex::dtype::DType; @@ -337,7 +337,12 @@ impl PyArray { let chunks = chunked_array .iter_chunks() - .map(|chunk| -> PyVortexResult<_> { Ok(chunk.clone().into_arrow(&arrow_dtype)?) }) + .map(|chunk| -> PyVortexResult<_> { + Ok(chunk.clone().execute_arrow( + Some(&arrow_dtype), + &mut LEGACY_SESSION.create_execution_ctx(), + )?) + }) .collect::, _>>()?; let pa_data_type = arrow_dtype.clone().to_pyarrow(py)?; @@ -357,7 +362,7 @@ impl PyArray { )?) } else { Ok(array - .into_arrow_preferred()? + .execute_arrow(None, &mut LEGACY_SESSION.create_execution_ctx())? .into_data() .to_pyarrow(py)? .into_bound(py)) diff --git a/vortex-python/src/iter/mod.rs b/vortex-python/src/iter/mod.rs index c878ae33e93..8e40fdb8259 100644 --- a/vortex-python/src/iter/mod.rs +++ b/vortex-python/src/iter/mod.rs @@ -20,7 +20,9 @@ use pyo3::prelude::*; use pyo3::types::PyIterator; use vortex::array::Canonical; use vortex::array::IntoArray; -use vortex::array::arrow::IntoArrowArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowArrayExecutor; use vortex::array::iter::ArrayIterator; use vortex::array::iter::ArrayIteratorAdapter; use vortex::array::iter::ArrayIteratorExt; @@ -126,7 +128,8 @@ impl PyArrayIterator { Box::new(RecordBatchIterator::new( iter.map(move |chunk| { let data_type = data_type.clone(); - chunk?.into_arrow(&data_type) + chunk? + .execute_arrow(Some(&data_type), &mut LEGACY_SESSION.create_execution_ctx()) }) .map(|chunk| chunk.map_err(|e| ArrowError::ExternalError(Box::new(e)))) .map(|array| array.map(|a| RecordBatch::from(a.as_struct().clone()))), From 9fe20c5d72db3b4fd4c1236da69fa7ed69f913b7 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:37:27 -0400 Subject: [PATCH 150/250] Pass `ExecutionCtx` through the compressor (#7578) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 In order to have a "less breaking change" before, I just put an execution context in an arc'd mutex in the compressor itself. This is really stupid, and the caller should pass an execution context whenever they want to compress something (because we need to look at buffers to canonicalize and do compute in the compressor). ## Testing N/A since this is just moving where things are coming from. --------- Signed-off-by: Connor Tsui --- Cargo.lock | 2 + fuzz/src/array/mod.rs | 9 +- vortex-btrblocks/Cargo.toml | 1 + vortex-btrblocks/benches/compress.rs | 22 +- vortex-btrblocks/benches/compress_listview.rs | 12 +- vortex-btrblocks/public-api.lock | 74 ++-- vortex-btrblocks/src/canonical_compressor.rs | 41 ++- vortex-btrblocks/src/lib.rs | 4 +- vortex-btrblocks/src/schemes/decimal.rs | 15 +- vortex-btrblocks/src/schemes/float.rs | 135 ++++---- vortex-btrblocks/src/schemes/integer.rs | 317 +++++++++-------- vortex-btrblocks/src/schemes/string.rs | 114 +++--- vortex-btrblocks/src/schemes/temporal.rs | 65 ++-- vortex-compressor/Cargo.toml | 1 + vortex-compressor/public-api.lock | 68 ++-- .../src/builtins/constant/bool.rs | 17 +- .../src/builtins/constant/float.rs | 22 +- .../src/builtins/constant/integer.rs | 17 +- .../src/builtins/constant/mod.rs | 17 +- .../src/builtins/constant/string.rs | 21 +- vortex-compressor/src/builtins/dict/float.rs | 34 +- .../src/builtins/dict/integer.rs | 30 +- vortex-compressor/src/builtins/dict/string.rs | 21 +- vortex-compressor/src/compressor.rs | 326 +++++++++++------- vortex-compressor/src/estimate.rs | 16 +- vortex-compressor/src/scheme.rs | 7 +- vortex-layout/public-api.lock | 10 +- vortex-layout/src/layouts/compressed.rs | 24 +- vortex-layout/src/layouts/dict/writer.rs | 4 +- vortex-python/src/compress.rs | 5 +- vortex-tensor/public-api.lock | 8 +- vortex-tensor/src/encodings/l2_denorm.rs | 12 +- .../src/encodings/turboquant/scheme.rs | 12 +- vortex/examples/compression_showcase.rs | 50 ++- vortex/src/lib.rs | 6 +- wasm-test/src/main.rs | 8 +- 36 files changed, 905 insertions(+), 642 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0499c2ec67f..945d35e3e08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10382,6 +10382,7 @@ dependencies = [ "vortex-pco", "vortex-runend", "vortex-sequence", + "vortex-session", "vortex-sparse", "vortex-tensor", "vortex-utils", @@ -10463,6 +10464,7 @@ dependencies = [ "vortex-buffer", "vortex-error", "vortex-mask", + "vortex-session", "vortex-utils", ] diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index d030479ac91..c75eeb309f2 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -237,7 +237,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { }; let compressed = BtrBlocksCompressor::default() - .compress(&indices_array) + .compress(&indices_array, &mut ctx) .vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test"); ( Action::Take(compressed), @@ -541,14 +541,15 @@ fn random_action_from_list( /// Compress an array using the given strategy. #[cfg(feature = "zstd")] pub fn compress_array(array: &ArrayRef, strategy: CompressorStrategy) -> ArrayRef { + let mut ctx = SESSION.create_execution_ctx(); match strategy { CompressorStrategy::Default => BtrBlocksCompressor::default() - .compress(array) + .compress(array, &mut ctx) .vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test"), CompressorStrategy::Compact => BtrBlocksCompressorBuilder::default() .with_compact() .build() - .compress(array) + .compress(array, &mut ctx) .vortex_expect("Compact compress should succeed in fuzz test"), } } @@ -557,7 +558,7 @@ pub fn compress_array(array: &ArrayRef, strategy: CompressorStrategy) -> ArrayRe #[cfg(not(feature = "zstd"))] pub fn compress_array(array: &ArrayRef, _strategy: CompressorStrategy) -> ArrayRef { BtrBlocksCompressor::default() - .compress(array) + .compress(array, &mut SESSION.create_execution_ctx()) .vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test") } diff --git a/vortex-btrblocks/Cargo.toml b/vortex-btrblocks/Cargo.toml index dab4f27fb51..40b0ae52aae 100644 --- a/vortex-btrblocks/Cargo.toml +++ b/vortex-btrblocks/Cargo.toml @@ -44,6 +44,7 @@ divan = { workspace = true } rstest = { workspace = true } test-with = { workspace = true } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-session = { workspace = true } [features] # This feature enabled unstable encodings for which we don't guarantee stability. diff --git a/vortex-btrblocks/benches/compress.rs b/vortex-btrblocks/benches/compress.rs index a2daa33b432..8bc47d16404 100644 --- a/vortex-btrblocks/benches/compress.rs +++ b/vortex-btrblocks/benches/compress.rs @@ -5,6 +5,8 @@ #[cfg(not(codspeed))] mod benchmarks { + use std::sync::LazyLock; + use divan::Bencher; use divan::counter::BytesCount; use divan::counter::ItemsCount; @@ -13,13 +15,17 @@ mod benchmarks { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; + use vortex_array::session::ArraySession; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; + use vortex_session::VortexSession; use vortex_utils::aliases::hash_set::HashSet; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + fn make_clickbench_window_name() -> ArrayRef { // A test that's meant to mirror the WindowName column from ClickBench. let mut values = buffer_mut![-1i32; 65_536]; @@ -41,16 +47,20 @@ mod benchmarks { #[divan::bench] fn btrblocks(bencher: Bencher) { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = SESSION.create_execution_ctx(); let array = make_clickbench_window_name() .execute::(&mut ctx) .unwrap(); let compressor = BtrBlocksCompressor::default(); bencher - .with_inputs(|| &array) - .input_counter(|array| ItemsCount::new(array.len())) - .input_counter(|array| BytesCount::of_many::(array.len())) - .bench_refs(|array| compressor.compress(&array.clone().into_array()).unwrap()); + .with_inputs(|| (&array, SESSION.create_execution_ctx())) + .input_counter(|(array, _)| ItemsCount::new(array.len())) + .input_counter(|(array, _)| BytesCount::of_many::(array.len())) + .bench_refs(|(array, ctx)| { + compressor + .compress(&array.clone().into_array(), ctx) + .unwrap() + }); } } diff --git a/vortex-btrblocks/benches/compress_listview.rs b/vortex-btrblocks/benches/compress_listview.rs index c3d0f8f6aa8..e1b0595ac27 100644 --- a/vortex-btrblocks/benches/compress_listview.rs +++ b/vortex-btrblocks/benches/compress_listview.rs @@ -6,6 +6,8 @@ #[cfg(not(codspeed))] mod benchmarks { + use std::sync::LazyLock; + use divan::Bencher; use divan::counter::BytesCount; use divan::counter::ItemsCount; @@ -14,17 +16,23 @@ mod benchmarks { use rand::prelude::StdRng; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::FieldNames; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; + use vortex_session::VortexSession; const NUM_ROWS: usize = 8192; const SEED: u64 = 42; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + const SHORT_STRINGS: &[&str] = &[ "alpha_one", "bravo_two", @@ -177,10 +185,10 @@ mod benchmarks { let nbytes = array.nbytes(); let compressor = BtrBlocksCompressor::default(); bencher - .with_inputs(|| &array) + .with_inputs(|| (&array, SESSION.create_execution_ctx())) .input_counter(|_| ItemsCount::new(NUM_ROWS)) .input_counter(move |_| BytesCount::new(nbytes as usize)) - .bench_refs(|array| compressor.compress(array).unwrap()); + .bench_refs(|(array, ctx)| compressor.compress(array, ctx).unwrap()); } } diff --git a/vortex-btrblocks/public-api.lock b/vortex-btrblocks/public-api.lock index 179cd45cf5c..725307fcbf2 100644 --- a/vortex-btrblocks/public-api.lock +++ b/vortex-btrblocks/public-api.lock @@ -56,9 +56,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::decimal::D impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::decimal::DecimalScheme -pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -98,9 +98,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::ALP impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::ALPRDScheme -pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::ALPRDScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::ALPRDScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::ALPRDScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -128,9 +128,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::ALP impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::ALPScheme -pub fn vortex_btrblocks::schemes::float::ALPScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::ALPScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::ALPScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::ALPScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::ALPScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -162,11 +162,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::Flo pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -196,11 +196,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::Nul impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::NullDominatedSparseScheme -pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -230,9 +230,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::Pco impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::PcoScheme -pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::PcoScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::PcoScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::PcoScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -270,9 +270,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::B impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::BitPackingScheme -pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -302,9 +302,9 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::F pub fn vortex_btrblocks::schemes::integer::FoRScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::FoRScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -334,11 +334,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::I pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -368,9 +368,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::P impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::PcoScheme -pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::PcoScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::PcoScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::PcoScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -400,11 +400,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::R pub fn vortex_btrblocks::schemes::integer::RunEndScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::RunEndScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::RunEndScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::RunEndScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::RunEndScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::RunEndScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::RunEndScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -436,9 +436,9 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::S pub fn vortex_btrblocks::schemes::integer::SequenceScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::SequenceScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::SequenceScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::SequenceScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -466,11 +466,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::S impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::SparseScheme -pub fn vortex_btrblocks::schemes::integer::SparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::SparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::SparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::SparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::SparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::SparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -504,11 +504,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::Z pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -548,9 +548,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::FS impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::FSSTScheme -pub fn vortex_btrblocks::schemes::string::FSSTScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::FSSTScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::string::FSSTScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::FSSTScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::FSSTScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -580,11 +580,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::Nu impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::NullDominatedSparseScheme -pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -614,9 +614,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::Zs impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::ZstdScheme -pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::string::ZstdScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::ZstdScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::ZstdScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -646,9 +646,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::temporal:: impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::temporal::TemporalScheme -pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -660,7 +660,7 @@ pub struct vortex_btrblocks::BtrBlocksCompressor(pub vortex_compressor::compress impl vortex_btrblocks::BtrBlocksCompressor -pub fn vortex_btrblocks::BtrBlocksCompressor::compress(&self, array: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_btrblocks::BtrBlocksCompressor::compress(&self, array: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl core::clone::Clone for vortex_btrblocks::BtrBlocksCompressor diff --git a/vortex-btrblocks/src/canonical_compressor.rs b/vortex-btrblocks/src/canonical_compressor.rs index 885178dd22d..0be774bfbc5 100644 --- a/vortex-btrblocks/src/canonical_compressor.rs +++ b/vortex-btrblocks/src/canonical_compressor.rs @@ -6,6 +6,7 @@ use std::ops::Deref; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_error::VortexResult; use crate::BtrBlocksCompressorBuilder; @@ -38,8 +39,8 @@ pub struct BtrBlocksCompressor( impl BtrBlocksCompressor { /// Compresses an array using BtrBlocks-inspired compression. - pub fn compress(&self, array: &ArrayRef) -> VortexResult { - self.0.compress(array) + pub fn compress(&self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + self.0.compress(array, ctx) } } @@ -59,21 +60,29 @@ impl Default for BtrBlocksCompressor { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::List; use vortex_array::arrays::ListView; use vortex_array::arrays::ListViewArray; use vortex_array::assert_arrays_eq; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[rstest] #[case::zctl( unsafe { @@ -100,7 +109,8 @@ mod tests { #[case] expect_list: bool, ) -> VortexResult<()> { let array_ref = input.clone().into_array(); - let result = BtrBlocksCompressor::default().compress(&array_ref)?; + let result = BtrBlocksCompressor::default() + .compress(&array_ref, &mut SESSION.create_execution_ctx())?; if expect_list { assert!(result.as_opt::().is_some()); } else { @@ -114,7 +124,10 @@ mod tests { fn test_constant_all_true() -> VortexResult<()> { let array = BoolArray::new(BitBuffer::from(vec![true; 100]), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.clone().into_array())?; + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(compressed.is::()); assert_arrays_eq!(compressed, array); Ok(()) @@ -124,7 +137,10 @@ mod tests { fn test_constant_all_false() -> VortexResult<()> { let array = BoolArray::new(BitBuffer::from(vec![false; 100]), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.clone().into_array())?; + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(compressed.is::()); assert_arrays_eq!(compressed, array); Ok(()) @@ -137,7 +153,10 @@ mod tests { Validity::from(BitBuffer::from(vec![true; 100])), ); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.clone().into_array())?; + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(compressed.is::()); assert_arrays_eq!(compressed, array); Ok(()) @@ -148,7 +167,10 @@ mod tests { let validity = Validity::from(BitBuffer::from_iter((0..100).map(|i| i % 3 != 0))); let array = BoolArray::new(BitBuffer::from(vec![true; 100]), validity); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.clone().into_array())?; + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(!compressed.is::()); assert_arrays_eq!(compressed, array); Ok(()) @@ -161,7 +183,10 @@ mod tests { Validity::NonNullable, ); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.clone().into_array())?; + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(!compressed.is::()); assert_arrays_eq!(compressed, array); Ok(()) diff --git a/vortex-btrblocks/src/lib.rs b/vortex-btrblocks/src/lib.rs index 1f6d5a2d816..39db05246a6 100644 --- a/vortex-btrblocks/src/lib.rs +++ b/vortex-btrblocks/src/lib.rs @@ -22,8 +22,8 @@ //! //! # How It Works //! -//! [`BtrBlocksCompressor::compress()`] takes an `&ArrayRef` and returns an `ArrayRef` that may -//! use a different encoding. It first canonicalizes the input, then dispatches by type. +//! [`BtrBlocksCompressor::compress()`] takes an `&ArrayRef` plus a mutable execution context and +//! returns an `ArrayRef` that may use a different encoding. It first canonicalizes the input, then dispatches by type. //! Primitives and strings go through `choose_and_compress`, which evaluates every enabled //! [`Scheme`] and picks the one with the best compression ratio. Compound types like structs //! and lists recurse into their fields and elements. diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index e68b4b8191f..47d18f80860 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; @@ -45,7 +46,8 @@ impl Scheme for DecimalScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Decimal compression is almost always beneficial (narrowing + primitive compression). CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) @@ -55,14 +57,12 @@ impl Scheme for DecimalScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(joe): add support splitting i128/256 buffers into chunks of primitive values // for compression. 2 for i128 and 4 for i256. - let decimal = data - .array() - .clone() - .execute::(&mut compressor.execution_ctx())?; + let decimal = data.array().clone().execute::(exec_ctx)?; let decimal = narrowed_decimal(decimal); let validity = decimal.validity()?; let prim = match decimal.values_type() { @@ -73,7 +73,8 @@ impl Scheme for DecimalScheme { _ => return Ok(decimal.into_array()), }; - let compressed = compressor.compress_child(&prim.into_array(), &ctx, self.id(), 0)?; + let compressed = + compressor.compress_child(&prim.into_array(), &compress_ctx, self.id(), 0, exec_ctx)?; DecimalByteParts::try_new(compressed, decimal.decimal_dtype()).map(|d| d.into_array()) } diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index a3526f141e6..be3bf69df75 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -12,9 +12,8 @@ use vortex_alp::RDEncoder; use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::Patched; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::use_experimental_patches; @@ -85,11 +84,12 @@ impl Scheme for ALPScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // ALP encodes floats as integers. Without integer compression afterward, the encoded ints // are the same size. - if ctx.finished_cascading() { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } @@ -105,17 +105,19 @@ impl Scheme for ALPScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let alp_encoded = alp_encode( - data.array_as_primitive(), - None, - &mut compressor.execution_ctx(), - )?; + let alp_encoded = alp_encode(data.array_as_primitive(), None, exec_ctx)?; // Compress the ALP ints. - let compressed_alp_ints = - compressor.compress_child(alp_encoded.encoded(), &ctx, self.id(), 0)?; + let compressed_alp_ints = compressor.compress_child( + alp_encoded.encoded(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; let alp_stats = alp_encoded.as_array().statistics().to_owned(); let exponents = alp_encoded.exponents(); @@ -128,18 +130,14 @@ impl Scheme for ALPScheme { match patches { None => Ok(alp_array), - Some(p) => Ok(Patched::from_array_and_patches( - alp_array, - &p, - &mut compressor.execution_ctx(), - )? - .with_stats_set(alp_stats) - .into_array()), + Some(p) => Ok(Patched::from_array_and_patches(alp_array, &p, exec_ctx)? + .with_stats_set(alp_stats) + .into_array()), } } else { let patches = alp_encoded .patches() - .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .map(|p| compress_patches(p, exec_ctx)) .transpose()?; Ok(ALP::new(compressed_alp_ints, exponents, patches).into_array()) @@ -159,7 +157,8 @@ impl Scheme for ALPRDScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // We don't support ALPRD for f16. if data.array_as_primitive().ptype() == PType::F16 { @@ -171,9 +170,10 @@ impl Scheme for ALPRDScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive_array = data.array_as_primitive(); @@ -183,13 +183,13 @@ impl Scheme for ALPRDScheme { ptype => vortex_panic!("cannot ALPRD compress ptype {ptype}"), }; - let alp_rd = encoder.encode(primitive_array, &mut compressor.execution_ctx()); + let alp_rd = encoder.encode(primitive_array, exec_ctx); let dtype = alp_rd.dtype().clone(); let right_bit_width = alp_rd.right_bit_width(); let mut parts = ALPRDArrayOwnedExt::into_data_parts(alp_rd); parts.left_parts_patches = parts .left_parts_patches - .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .map(|p| compress_patches(p, exec_ctx)) .transpose()?; Ok(vortex_alp::ALPRD::try_new( @@ -199,7 +199,7 @@ impl Scheme for ALPRDScheme { parts.right_parts, right_bit_width, parts.left_parts_patches, - &mut compressor.execution_ctx(), + exec_ctx, )? .into_array()) } @@ -230,13 +230,11 @@ impl Scheme for NullDominatedSparseScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { let len = data.array_len() as f64; - // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature - // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. - let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.float_stats(&mut exec_ctx); + let stats = data.float_stats(exec_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -257,20 +255,26 @@ impl Scheme for NullDominatedSparseScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // We pass None as we only run this pathway for NULL-dominated float arrays. - let sparse_encoded = Sparse::encode(data.array(), None, &mut compressor.execution_ctx())?; + let sparse_encoded = Sparse::encode(data.array(), None, exec_ctx)?; if let Some(sparse) = sparse_encoded.as_opt::() { let indices = sparse .patches() .indices() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_indices = - compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; + let compressed_indices = compressor.compress_child( + &indices.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; Sparse::try_new( compressed_indices, @@ -298,22 +302,24 @@ impl Scheme for PcoScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { Ok(vortex_pco::Pco::from_primitive( data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, - &mut compressor.execution_ctx(), + exec_ctx, )? .into_array()) } @@ -344,19 +350,15 @@ impl Scheme for FloatRLEScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // RLE is only useful when we cascade it with another encoding. - if ctx.finished_cascading() { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature - // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. - let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); - if data.float_stats(&mut exec_ctx).average_run_length() - < super::integer::RUN_LENGTH_THRESHOLD - { + if data.float_stats(exec_ctx).average_run_length() < super::integer::RUN_LENGTH_THRESHOLD { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } @@ -367,38 +369,46 @@ impl Scheme for FloatRLEScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - super::integer::rle_compress(self, compressor, data, ctx) + super::integer::rle_compress(self, compressor, data, compress_ctx, exec_ctx) } } #[cfg(test)] mod tests { use std::iter; + use std::sync::LazyLock; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::display::DisplayOptions; use vortex_array::dtype::Nullability; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer_mut; use vortex_compressor::CascadingCompressor; use vortex_error::VortexResult; use vortex_fastlanes::RLE; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; use crate::schemes::float::FloatRLEScheme; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_empty() -> VortexResult<()> { let btr = BtrBlocksCompressor::default(); let array = PrimitiveArray::new(Buffer::::empty(), Validity::NonNullable).into_array(); - let result = btr.compress(&array)?; + let result = btr.compress(&array, &mut SESSION.create_execution_ctx())?; assert!(result.is_empty()); Ok(()) @@ -413,7 +423,7 @@ mod tests { let array = values.into_array(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array)?; + let compressed = btr.compress(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 1024); let display = compressed @@ -435,7 +445,8 @@ mod tests { let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let compressor = CascadingCompressor::new(vec![&FloatRLEScheme]); - let compressed = compressor.compress(&array.into_array())?; + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); let expected = Buffer::copy_from(&values).into_array(); @@ -456,7 +467,7 @@ mod tests { let array = array.finish_into_primitive().into_array(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array)?; + let compressed = btr.compress(&array, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 96); let display = compressed @@ -472,26 +483,34 @@ mod tests { /// Tests to verify that each float compression scheme produces the expected encoding. #[cfg(test)] mod scheme_selection_tests { + use std::sync::LazyLock; + use vortex_alp::ALP; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::dtype::Nullability; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_constant_compressed() -> VortexResult<()> { let values: Vec = vec![42.5; 100]; let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -501,7 +520,7 @@ mod scheme_selection_tests { let values: Vec = (0..1000).map(|i| (i as f64) * 0.01).collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -514,7 +533,7 @@ mod scheme_selection_tests { .collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); assert!(compressed.children()[0].is::()); Ok(()) @@ -529,7 +548,7 @@ mod scheme_selection_tests { builder.append_nulls(95); let array = builder.finish_into_primitive(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; // Verify the compressed array preserves values. assert_eq!(compressed.len(), 100); Ok(()) diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index d9d21a28b84..69dee5be17c 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -5,9 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Patched; use vortex_array::arrays::PrimitiveArray; @@ -131,17 +130,15 @@ impl Scheme for FoRScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // FoR only subtracts the min. Without further compression (e.g. BitPacking), the output is // the same size. - if ctx.finished_cascading() { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut local_ctx); + let stats = data.integer_stats(exec_ctx); // Only apply when the min is not already zero. if stats.erased().min_is_zero() { @@ -186,25 +183,25 @@ impl Scheme for FoRScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let primitive = data - .array() - .clone() - .execute::(&mut compressor.execution_ctx())?; + let primitive = data.array().clone().execute::(exec_ctx)?; let for_array = FoR::encode(primitive)?; let biased = for_array .encoded() .clone() - .execute::(&mut compressor.execution_ctx())?; + .execute::(exec_ctx)?; // Immediately bitpack. If any other scheme was preferable, it would be chosen instead // of bitpacking. // NOTE: we could delegate in the future if we had another downstream codec that performs // as well. - let leaf_ctx = ctx.clone().as_leaf(); - let mut biased_data = ArrayAndStats::new(biased.into_array(), ctx.merged_stats_options()); - let compressed = BitPackingScheme.compress(compressor, &mut biased_data, leaf_ctx)?; + let leaf_ctx = compress_ctx.clone().as_leaf(); + let mut biased_data = + ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); + let compressed = + BitPackingScheme.compress(compressor, &mut biased_data, leaf_ctx, exec_ctx)?; // TODO(connor): This should really be `new_unchecked`. let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; @@ -272,17 +269,15 @@ impl Scheme for ZigZagScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // ZigZag only transforms negative values to positive. Without further compression, // the output is the same size. - if ctx.finished_cascading() { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut local_ctx); + let stats = data.integer_stats(exec_ctx); // ZigZag is only useful when there are negative values. if !stats.erased().min_is_negative() { @@ -296,16 +291,20 @@ impl Scheme for ZigZagScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // Zigzag encode the values, then recursively compress the inner values. let zag = zigzag_encode(data.array_as_primitive())?; - let encoded = zag - .encoded() - .clone() - .execute::(&mut compressor.execution_ctx())?; - - let compressed = compressor.compress_child(&encoded.into_array(), &ctx, self.id(), 0)?; + let encoded = zag.encoded().clone().execute::(exec_ctx)?; + + let compressed = compressor.compress_child( + &encoded.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; Ok(ZigZag::try_new(compressed)?.into_array()) } @@ -323,11 +322,10 @@ impl Scheme for BitPackingScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut local_ctx); + let stats = data.integer_stats(exec_ctx); // BitPacking only works for non-negative values. if stats.erased().min_is_negative() { @@ -339,13 +337,14 @@ impl Scheme for BitPackingScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let primitive_array = data.array_as_primitive(); - let histogram = bit_width_histogram(primitive_array, &mut compressor.execution_ctx())?; + let histogram = bit_width_histogram(primitive_array, exec_ctx)?; let bw = find_best_bit_width(primitive_array.ptype(), &histogram)?; // If best bw is determined to be the current bit-width, return the original array. @@ -355,12 +354,7 @@ impl Scheme for BitPackingScheme { // Otherwise we can bitpack the array. let primitive_array = primitive_array.into_owned(); - let packed = bitpack_encode( - &primitive_array, - bw, - Some(&histogram), - &mut compressor.execution_ctx(), - )?; + let packed = bitpack_encode(&primitive_array, bw, Some(&histogram), exec_ctx)?; let packed_stats = packed.statistics().to_owned(); let ptype = packed.dtype().as_ptype(); @@ -382,18 +376,16 @@ impl Scheme for BitPackingScheme { match patches { None => array, - Some(p) => { - Patched::from_array_and_patches(array, &p, &mut compressor.execution_ctx())? - .with_stats_set(packed_stats) - .into_array() - } + Some(p) => Patched::from_array_and_patches(array, &p, exec_ctx)? + .with_stats_set(packed_stats) + .into_array(), } } else { // Compress patches and place back into BitPackedArray. let patches = parts .patches .take() - .map(|p| compress_patches(p, &mut compressor.execution_ctx())) + .map(|p| compress_patches(p, exec_ctx)) .transpose()?; parts.patches = patches; BitPacked::try_new( @@ -459,12 +451,11 @@ impl Scheme for SparseScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { let len = data.array_len() as f64; - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut local_ctx); + let stats = data.integer_stats(exec_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -506,11 +497,12 @@ impl Scheme for SparseScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let len = data.array_len(); // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats(&mut compressor.execution_ctx()).clone(); + let stats = data.integer_stats(exec_ctx).clone(); let array = data.array(); let (most_frequent_value, most_frequent_count) = stats @@ -540,7 +532,7 @@ impl Scheme for SparseScheme { most_frequent_value.ptype(), array.dtype().nullability(), )), - &mut compressor.execution_ctx(), + exec_ctx, )?; if let Some(sparse) = sparse_encoded.as_opt::() { @@ -548,23 +540,29 @@ impl Scheme for SparseScheme { .patches() .values() .clone() - .execute::(&mut compressor.execution_ctx())?; + .execute::(exec_ctx)?; let compressed_values = compressor.compress_child( &sparse_values_primitive.into_array(), - &ctx, + &compress_ctx, self.id(), 0, + exec_ctx, )?; let indices = sparse .patches() .indices() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_indices = - compressor.compress_child(&indices.into_array(), &ctx, self.id(), 1)?; + let compressed_indices = compressor.compress_child( + &indices.into_array(), + &compress_ctx, + self.id(), + 1, + exec_ctx, + )?; Sparse::try_new( compressed_indices, @@ -638,12 +636,11 @@ impl Scheme for RunEndScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // If the run length is below the threshold, drop it. - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - if data.integer_stats(&mut local_ctx).average_run_length() < RUN_END_THRESHOLD { + if data.integer_stats(exec_ctx).average_run_length() < RUN_END_THRESHOLD { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } @@ -654,17 +651,23 @@ impl Scheme for RunEndScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // Run-end encode the ends. - let (ends, values) = - runend_encode(data.array_as_primitive(), &mut compressor.execution_ctx()); - - let values_primitive = values.execute::(&mut compressor.execution_ctx())?; - let compressed_values = - compressor.compress_child(&values_primitive.into_array(), &ctx, self.id(), 0)?; + let (ends, values) = runend_encode(data.array_as_primitive(), exec_ctx); + + let values_primitive = values.execute::(exec_ctx)?; + let compressed_values = compressor.compress_child( + &values_primitive.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; - let compressed_ends = compressor.compress_child(&ends.into_array(), &ctx, self.id(), 1)?; + let compressed_ends = + compressor.compress_child(&ends.into_array(), &compress_ctx, self.id(), 1, exec_ctx)?; // SAFETY: compression doesn't affect invariants. Ok(unsafe { @@ -673,7 +676,7 @@ impl Scheme for RunEndScheme { compressed_values, 0, data.array_len(), - &mut compressor.execution_ctx(), + exec_ctx, ) .into_array() }) @@ -712,17 +715,15 @@ impl Scheme for SequenceScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // It is pointless checking if a sample is a sequence since it will not correspond to the // entire array. - if ctx.is_sample() { + if compress_ctx.is_sample() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut local_ctx); + let stats = data.integer_stats(exec_ctx); // `SequenceArray` does not support nulls. if stats.null_count() > 0 { @@ -742,10 +743,8 @@ impl Scheme for SequenceScheme { // why do we divide the ratio by 2??? CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |compressor, data, _ctx| { - let Some(encoded) = - sequence_encode(data.array_as_primitive(), &mut compressor.execution_ctx())? - else { + |_compressor, data, _ctx, exec_ctx| { + let Some(encoded) = sequence_encode(data.array_as_primitive(), exec_ctx)? else { // If we are unable to sequence encode this array, make sure we skip. return Ok(EstimateVerdict::Skip); }; @@ -760,16 +759,17 @@ impl Scheme for SequenceScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let stats = data.integer_stats(&mut compressor.execution_ctx()); + let stats = data.integer_stats(exec_ctx); if stats.null_count() > 0 { vortex_bail!("sequence encoding does not support nulls"); } - sequence_encode(data.array_as_primitive(), &mut compressor.execution_ctx())? + sequence_encode(data.array_as_primitive(), exec_ctx)? .ok_or_else(|| vortex_err!("cannot sequence encode array")) } } @@ -787,7 +787,8 @@ impl Scheme for PcoScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { use vortex_array::dtype::PType; @@ -801,15 +802,16 @@ impl Scheme for PcoScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { Ok(vortex_pco::Pco::from_primitive( data.array_as_primitive(), pco::DEFAULT_COMPRESSION_LEVEL, 8192, - &mut compressor.execution_ctx(), + exec_ctx, )? .into_array()) } @@ -820,16 +822,22 @@ pub(crate) fn rle_compress( scheme: &dyn Scheme, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let rle_array = RLE::encode(data.array_as_primitive(), &mut compressor.execution_ctx())?; + let rle_array = RLE::encode(data.array_as_primitive(), exec_ctx)?; let rle_values_primitive = rle_array .values() .clone() - .execute::(&mut compressor.execution_ctx())?; - let compressed_values = - compressor.compress_child(&rle_values_primitive.into_array(), &ctx, scheme.id(), 0)?; + .execute::(exec_ctx)?; + let compressed_values = compressor.compress_child( + &rle_values_primitive.into_array(), + &compress_ctx, + scheme.id(), + 0, + exec_ctx, + )?; // Delta is an unstable encoding, once we deem it stable we can switch over to this always. #[cfg(feature = "unstable_encodings")] @@ -837,14 +845,15 @@ pub(crate) fn rle_compress( let rle_indices_primitive = rle_array .indices() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; try_compress_delta( compressor, &rle_indices_primitive.into_array(), - &ctx, + &compress_ctx, scheme.id(), 1, + exec_ctx, )? }; @@ -853,18 +862,29 @@ pub(crate) fn rle_compress( let rle_indices_primitive = rle_array .indices() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - compressor.compress_child(&rle_indices_primitive.into_array(), &ctx, scheme.id(), 1)? + compressor.compress_child( + &rle_indices_primitive.into_array(), + &compress_ctx, + scheme.id(), + 1, + exec_ctx, + )? }; let rle_offsets_primitive = rle_array .values_idx_offsets() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_offsets = - compressor.compress_child(&rle_offsets_primitive.into_array(), &ctx, scheme.id(), 2)?; + let compressed_offsets = compressor.compress_child( + &rle_offsets_primitive.into_array(), + &compress_ctx, + scheme.id(), + 2, + exec_ctx, + )?; // SAFETY: Recursive compression doesn't affect the invariants. unsafe { @@ -886,17 +906,25 @@ fn try_compress_delta( parent_ctx: &CompressorContext, parent_id: SchemeId, child_index: usize, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let child_primitive = child - .clone() - .execute::(&mut compressor.execution_ctx())?; - let (bases, deltas) = - vortex_fastlanes::delta_compress(&child_primitive, &mut compressor.execution_ctx())?; - - let compressed_bases = - compressor.compress_child(&bases.into_array(), parent_ctx, parent_id, child_index)?; - let compressed_deltas = - compressor.compress_child(&deltas.into_array(), parent_ctx, parent_id, child_index)?; + let child_primitive = child.clone().execute::(exec_ctx)?; + let (bases, deltas) = vortex_fastlanes::delta_compress(&child_primitive, exec_ctx)?; + + let compressed_bases = compressor.compress_child( + &bases.into_array(), + parent_ctx, + parent_id, + child_index, + exec_ctx, + )?; + let compressed_deltas = compressor.compress_child( + &deltas.into_array(), + parent_ctx, + parent_id, + child_index, + exec_ctx, + )?; Delta::try_new(compressed_bases, compressed_deltas, 0, child.len()).map(IntoArray::into_array) } @@ -926,16 +954,14 @@ impl Scheme for IntRLEScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // RLE is only useful when we cascade it with another encoding. - if ctx.finished_cascading() { + if compress_ctx.finished_cascading() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - if data.integer_stats(&mut local_ctx).average_run_length() < RUN_LENGTH_THRESHOLD { + if data.integer_stats(exec_ctx).average_run_length() < RUN_LENGTH_THRESHOLD { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } @@ -946,26 +972,30 @@ impl Scheme for IntRLEScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - rle_compress(self, compressor, data, ctx) + rle_compress(self, compressor, data, compress_ctx, exec_ctx) } } #[cfg(test)] mod tests { use std::iter; + use std::sync::LazyLock; use itertools::Itertools; use rand::Rng; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::Masked; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -974,16 +1004,20 @@ mod tests { use vortex_error::VortexResult; use vortex_fastlanes::RLE; use vortex_sequence::Sequence; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; use crate::schemes::integer::IntRLEScheme; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_empty() -> VortexResult<()> { // Make sure empty array compression does not fail. let btr = BtrBlocksCompressor::default(); let array = PrimitiveArray::new(Buffer::::empty(), Validity::NonNullable); - let result = btr.compress(&array.into_array())?; + let result = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(result.is_empty()); Ok(()) @@ -1010,7 +1044,10 @@ mod tests { } let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&codes.freeze().into_array())?; + let compressed = btr.compress( + &codes.freeze().into_array(), + &mut SESSION.create_execution_ctx(), + )?; assert!(compressed.is::()); Ok(()) } @@ -1026,7 +1063,7 @@ mod tests { let validity = array.validity()?; let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); assert!(compressed.children()[0].is::()); @@ -1044,7 +1081,7 @@ mod tests { let array = PrimitiveArray::from_option_iter(values.clone().into_iter().map(Some)); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); let decoded = compressed; @@ -1062,7 +1099,8 @@ mod tests { let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let compressor = CascadingCompressor::new(vec![&IntRLEScheme]); - let compressed = compressor.compress(&array.into_array())?; + let compressed = + compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); let expected = Buffer::copy_from(&values).into_array(); @@ -1084,7 +1122,7 @@ mod tests { .into_array(); let btr = BtrBlocksCompressor::default(); - btr.compress(&prim)?; + btr.compress(&prim, &mut SESSION.create_execution_ctx())?; Ok(()) } @@ -1094,17 +1132,20 @@ mod tests { #[cfg(test)] mod scheme_selection_tests { use std::iter; + use std::sync::LazyLock; use rand::Rng; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -1112,16 +1153,20 @@ mod scheme_selection_tests { use vortex_fastlanes::FoR; use vortex_runend::RunEnd; use vortex_sequence::Sequence; + use vortex_session::VortexSession; use vortex_sparse::Sparse; use crate::BtrBlocksCompressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_constant_compressed() -> VortexResult<()> { let values: Vec = iter::repeat_n(42, 100).collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1131,7 +1176,7 @@ mod scheme_selection_tests { let values: Vec = (0..1000).map(|i| 1_000_000 + ((i * 37) % 100)).collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1141,7 +1186,7 @@ mod scheme_selection_tests { let values: Vec = (0..1000).map(|i| i % 16).collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); assert_eq!( compressed.statistics().get_as::(Stat::NullCount), @@ -1170,7 +1215,7 @@ mod scheme_selection_tests { } let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1194,7 +1239,7 @@ mod scheme_selection_tests { let array = PrimitiveArray::new(Buffer::copy_from(&codes), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1207,7 +1252,7 @@ mod scheme_selection_tests { } let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1217,7 +1262,7 @@ mod scheme_selection_tests { let values: Vec = (0..1000).map(|i| i * 7).collect(); let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -1230,7 +1275,7 @@ mod scheme_selection_tests { } let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array.into_array())?; + let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?; eprintln!("{}", compressed.display_tree()); assert!(compressed.is::()); Ok(()) diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 6aee7e8d634..037d8979d59 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -5,9 +5,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -75,7 +74,8 @@ impl Scheme for FSSTScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } @@ -84,38 +84,39 @@ impl Scheme for FSSTScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let utf8 = data.array_as_utf8().into_owned(); let compressor_fsst = fsst_train_compressor(&utf8); - let fsst = fsst_compress( - &utf8, - utf8.len(), - utf8.dtype(), - &compressor_fsst, - &mut compressor.execution_ctx(), - ); + let fsst = fsst_compress(&utf8, utf8.len(), utf8.dtype(), &compressor_fsst, exec_ctx); let uncompressed_lengths_primitive = fsst .uncompressed_lengths() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; let compressed_original_lengths = compressor.compress_child( &uncompressed_lengths_primitive.into_array(), - &ctx, + &compress_ctx, self.id(), 0, + exec_ctx, )?; let codes_offsets_primitive = fsst .codes() .offsets() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_codes_offsets = - compressor.compress_child(&codes_offsets_primitive.into_array(), &ctx, self.id(), 1)?; + let compressed_codes_offsets = compressor.compress_child( + &codes_offsets_primitive.into_array(), + &compress_ctx, + self.id(), + 1, + exec_ctx, + )?; let compressed_codes = VarBinArray::try_new( compressed_codes_offsets, fsst.codes().bytes().clone(), @@ -129,7 +130,7 @@ impl Scheme for FSSTScheme { fsst.symbol_lengths().clone(), compressed_codes, compressed_original_lengths, - &mut compressor.execution_ctx(), + exec_ctx, )?; Ok(fsst.into_array()) @@ -167,12 +168,11 @@ impl Scheme for NullDominatedSparseScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { let len = data.array_len() as f64; - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.string_stats(&mut local_ctx); + let stats = data.string_stats(exec_ctx); let value_count = stats.value_count(); // All-null arrays should be compressed as constant instead anyways. @@ -193,10 +193,11 @@ impl Scheme for NullDominatedSparseScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // We pass None as we only run this pathway for NULL-dominated string arrays. - let sparse_encoded = Sparse::encode(data.array(), None, &mut compressor.execution_ctx())?; + let sparse_encoded = Sparse::encode(data.array(), None, exec_ctx)?; if let Some(sparse) = sparse_encoded.as_opt::() { // Compress the indices only (not the values for strings). @@ -204,10 +205,15 @@ impl Scheme for NullDominatedSparseScheme { .patches() .indices() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_indices = - compressor.compress_child(&indices.into_array(), &ctx, self.id(), 0)?; + let compressed_indices = compressor.compress_child( + &indices.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; Sparse::try_new( compressed_indices, @@ -235,25 +241,24 @@ impl Scheme for ZstdScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let compacted = data.array_as_utf8().into_owned().compact_buffers()?; - Ok(vortex_zstd::Zstd::from_var_bin_view_without_dict( - &compacted, - 3, - 8192, - &mut compressor.execution_ctx(), - )? - .into_array()) + Ok( + vortex_zstd::Zstd::from_var_bin_view_without_dict(&compacted, 3, 8192, exec_ctx)? + .into_array(), + ) } } @@ -270,7 +275,8 @@ impl Scheme for ZstdBuffersScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } @@ -279,25 +285,34 @@ impl Scheme for ZstdBuffersScheme { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - Ok(vortex_zstd::ZstdBuffers::compress(data.array(), 3, &LEGACY_SESSION)?.into_array()) + Ok(vortex_zstd::ZstdBuffers::compress(data.array(), 3, exec_ctx.session())?.into_array()) } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::VarBinViewArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; use vortex_array::display::DisplayOptions; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; + use vortex_array::session::ArraySession; use vortex_error::VortexResult; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_strings() -> VortexResult<()> { let mut strings = Vec::new(); @@ -311,7 +326,7 @@ mod tests { let array_ref = strings.into_array(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array_ref)?; + let compressed = btr.compress(&array_ref, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 2048); let display = compressed @@ -334,7 +349,7 @@ mod tests { let array_ref = strings.into_array(); let btr = BtrBlocksCompressor::default(); - let compressed = btr.compress(&array_ref)?; + let compressed = btr.compress(&array_ref, &mut SESSION.create_execution_ctx())?; assert_eq!(compressed.len(), 100); let display = compressed @@ -350,23 +365,32 @@ mod tests { /// Tests to verify that each string compression scheme produces the expected encoding. #[cfg(test)] mod scheme_selection_tests { + use std::sync::LazyLock; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; + use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_fsst::FSST; + use vortex_session::VortexSession; use crate::BtrBlocksCompressor; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + #[test] fn test_constant_compressed() -> VortexResult<()> { let strings: Vec> = vec![Some("constant_value"); 100]; let array = VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable)); let array_ref = array.into_array(); - let compressed = BtrBlocksCompressor::default().compress(&array_ref)?; + let compressed = BtrBlocksCompressor::default() + .compress(&array_ref, &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -380,7 +404,8 @@ mod scheme_selection_tests { } let array = VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable)); let array_ref = array.into_array(); - let compressed = BtrBlocksCompressor::default().compress(&array_ref)?; + let compressed = BtrBlocksCompressor::default() + .compress(&array_ref, &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } @@ -395,7 +420,8 @@ mod scheme_selection_tests { } let array = VarBinViewArray::from_iter(strings, DType::Utf8(Nullability::NonNullable)); let array_ref = array.into_array(); - let compressed = BtrBlocksCompressor::default().compress(&array_ref)?; + let compressed = BtrBlocksCompressor::default() + .compress(&array_ref, &mut SESSION.create_execution_ctx())?; assert!(compressed.is::()); Ok(()) } diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index a2eeeb6774a..6405baeff40 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_array::arrays::ConstantArray; @@ -61,7 +62,8 @@ impl Scheme for TemporalScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Temporal compression (splitting into parts) is almost always beneficial. CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) @@ -71,24 +73,21 @@ impl Scheme for TemporalScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let array = data.array().clone(); - let ext_array = array.execute::(&mut compressor.execution_ctx())?; + let ext_array = array.execute::(exec_ctx)?; let temporal_array = TemporalArray::try_from(ext_array.clone().into_array())?; // Check for constant array and return early if so. - let is_constant = is_constant( - &ext_array.clone().into_array(), - &mut compressor.execution_ctx(), - )?; + let is_constant = is_constant(&ext_array.clone().into_array(), exec_ctx)?; if is_constant { - return Ok(ConstantArray::new( - ext_array.execute_scalar(0, &mut compressor.execution_ctx())?, - ext_array.len(), - ) - .into_array()); + return Ok( + ConstantArray::new(ext_array.execute_scalar(0, exec_ctx)?, ext_array.len()) + .into_array(), + ); } let dtype = temporal_array.dtype().clone(); @@ -96,22 +95,32 @@ impl Scheme for TemporalScheme { days, seconds, subseconds, - } = split_temporal(temporal_array, &mut compressor.execution_ctx())?; - - let days_primitive = days - .execute::(&mut compressor.execution_ctx())? - .narrow()?; - let days = compressor.compress_child(&days_primitive.into_array(), &ctx, self.id(), 0)?; - let seconds_primitive = seconds - .execute::(&mut compressor.execution_ctx())? - .narrow()?; - let seconds = - compressor.compress_child(&seconds_primitive.into_array(), &ctx, self.id(), 1)?; - let subseconds_primitive = subseconds - .execute::(&mut compressor.execution_ctx())? - .narrow()?; - let subseconds = - compressor.compress_child(&subseconds_primitive.into_array(), &ctx, self.id(), 2)?; + } = split_temporal(temporal_array, exec_ctx)?; + + let days_primitive = days.execute::(exec_ctx)?.narrow()?; + let days = compressor.compress_child( + &days_primitive.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; + let seconds_primitive = seconds.execute::(exec_ctx)?.narrow()?; + let seconds = compressor.compress_child( + &seconds_primitive.into_array(), + &compress_ctx, + self.id(), + 1, + exec_ctx, + )?; + let subseconds_primitive = subseconds.execute::(exec_ctx)?.narrow()?; + let subseconds = compressor.compress_child( + &subseconds_primitive.into_array(), + &compress_ctx, + self.id(), + 2, + exec_ctx, + )?; Ok(DateTimeParts::try_new(dtype, days, seconds, subseconds)?.into_array()) } diff --git a/vortex-compressor/Cargo.toml b/vortex-compressor/Cargo.toml index 2d94679157a..2d28d86a5e6 100644 --- a/vortex-compressor/Cargo.toml +++ b/vortex-compressor/Cargo.toml @@ -31,6 +31,7 @@ divan = { workspace = true } rstest = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } vortex-array = { workspace = true, features = ["_test-harness"] } +vortex-session = { workspace = true } [lints] workspace = true diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 29dc7b7163d..116cfba31f2 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -26,11 +26,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::BoolCons pub fn vortex_compressor::builtins::BoolConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::BoolConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::BoolConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -64,11 +64,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatCon pub fn vortex_compressor::builtins::FloatConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -102,11 +102,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatDic pub fn vortex_compressor::builtins::FloatDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -140,11 +140,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntConst pub fn vortex_compressor::builtins::IntConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -178,11 +178,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntDictS pub fn vortex_compressor::builtins::IntDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -216,11 +216,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringCo pub fn vortex_compressor::builtins::StringConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -254,11 +254,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringDi pub fn vortex_compressor::builtins::StringDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -338,7 +338,7 @@ impl core::fmt::Debug for vortex_compressor::estimate::EstimateVerdict pub fn vortex_compressor::estimate::EstimateVerdict::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &mut vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) +pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &mut vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) pub mod vortex_compressor::scheme @@ -428,11 +428,11 @@ pub trait vortex_compressor::scheme::Scheme: core::fmt::Debug + core::marker::Se pub fn vortex_compressor::scheme::Scheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::scheme::Scheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::scheme::Scheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::scheme::Scheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::scheme::Scheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::scheme::Scheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::scheme::Scheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -446,11 +446,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::BoolCons pub fn vortex_compressor::builtins::BoolConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::BoolConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::BoolConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -464,11 +464,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatCon pub fn vortex_compressor::builtins::FloatConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -482,11 +482,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatDic pub fn vortex_compressor::builtins::FloatDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -500,11 +500,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntConst pub fn vortex_compressor::builtins::IntConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -518,11 +518,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntDictS pub fn vortex_compressor::builtins::IntDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -536,11 +536,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringCo pub fn vortex_compressor::builtins::StringConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -554,11 +554,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringDi pub fn vortex_compressor::builtins::StringDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -932,11 +932,9 @@ pub struct vortex_compressor::CascadingCompressor impl vortex_compressor::CascadingCompressor -pub fn vortex_compressor::CascadingCompressor::compress(&self, array: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_compressor::CascadingCompressor::compress(&self, array: &vortex_array::array::erased::ArrayRef, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_compressor::CascadingCompressor::compress_child(&self, child: &vortex_array::array::erased::ArrayRef, parent_ctx: &vortex_compressor::ctx::CompressorContext, parent_id: vortex_compressor::scheme::SchemeId, child_index: usize) -> vortex_error::VortexResult - -pub fn vortex_compressor::CascadingCompressor::execution_ctx(&self) -> parking_lot::mutex::MutexGuard<'_, vortex_array::executor::ExecutionCtx> +pub fn vortex_compressor::CascadingCompressor::compress_child(&self, child: &vortex_array::array::erased::ArrayRef, parent_ctx: &vortex_compressor::ctx::CompressorContext, parent_id: vortex_compressor::scheme::SchemeId, child_index: usize, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::CascadingCompressor::new(schemes: alloc::vec::Vec<&'static dyn vortex_compressor::scheme::Scheme>) -> Self diff --git a/vortex-compressor/src/builtins/constant/bool.rs b/vortex-compressor/src/builtins/constant/bool.rs index 9ed23911f2d..e99edfbf315 100644 --- a/vortex-compressor/src/builtins/constant/bool.rs +++ b/vortex-compressor/src/builtins/constant/bool.rs @@ -5,8 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_error::VortexResult; use crate::CascadingCompressor; @@ -30,18 +29,17 @@ impl Scheme for BoolConstantScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. - if ctx.is_sample() { + if compress_ctx.is_sample() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.bool_stats(&mut ctx); + let stats = data.bool_stats(exec_ctx); // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { @@ -60,8 +58,9 @@ impl Scheme for BoolConstantScheme { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - compress_constant_array_with_validity(data.array()) + compress_constant_array_with_validity(data.array(), exec_ctx) } } diff --git a/vortex-compressor/src/builtins/constant/float.rs b/vortex-compressor/src/builtins/constant/float.rs index 3c9214f3c37..70e8525855a 100644 --- a/vortex-compressor/src/builtins/constant/float.rs +++ b/vortex-compressor/src/builtins/constant/float.rs @@ -5,8 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_error::VortexResult; @@ -33,19 +32,17 @@ impl Scheme for FloatConstantScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. - if ctx.is_sample() { + if compress_ctx.is_sample() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); - // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature - // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. - let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.float_stats(&mut exec_ctx); + let stats = data.float_stats(exec_ctx); // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { @@ -69,8 +66,8 @@ impl Scheme for FloatConstantScheme { // This is an expensive check, but in practice the distinct count is known because we often // include dictionary encoding in our set of schemes, so we rarely call this. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |compressor, data, _ctx| { - if is_constant(data.array(), &mut compressor.execution_ctx())? { + |_compressor, data, _ctx, exec_ctx| { + if is_constant(data.array(), exec_ctx)? { Ok(EstimateVerdict::AlwaysUse) } else { Ok(EstimateVerdict::Skip) @@ -83,8 +80,9 @@ impl Scheme for FloatConstantScheme { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - compress_constant_array_with_validity(data.array()) + compress_constant_array_with_validity(data.array(), exec_ctx) } } diff --git a/vortex-compressor/src/builtins/constant/integer.rs b/vortex-compressor/src/builtins/constant/integer.rs index e98fc343731..298c9de7e84 100644 --- a/vortex-compressor/src/builtins/constant/integer.rs +++ b/vortex-compressor/src/builtins/constant/integer.rs @@ -5,8 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_error::VortexResult; use super::is_integer_primitive; @@ -31,18 +30,17 @@ impl Scheme for IntConstantScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. - if ctx.is_sample() { + if compress_ctx.is_sample() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut ctx); + let stats = data.integer_stats(exec_ctx); // Note that we only compute distinct counts if other schemes have requested it. if let Some(distinct_count) = stats.distinct_count() { @@ -72,8 +70,9 @@ impl Scheme for IntConstantScheme { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - compress_constant_array_with_validity(data.array()) + compress_constant_array_with_validity(data.array(), exec_ctx) } } diff --git a/vortex-compressor/src/builtins/constant/mod.rs b/vortex-compressor/src/builtins/constant/mod.rs index 6f51c76a646..139cb8d2790 100644 --- a/vortex-compressor/src/builtins/constant/mod.rs +++ b/vortex-compressor/src/builtins/constant/mod.rs @@ -4,9 +4,8 @@ //! Constant encoding schemes for bool, float, integer, and string arrays. use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::MaskedArray; use vortex_array::scalar::Scalar; @@ -44,22 +43,24 @@ mod string; /// Assumes that the source array has constant valid scalars. /// /// If the array has any nulls, returns a [`MaskedArray`] with a [`ConstantArray`] child.` -fn compress_constant_array_with_validity(source: &ArrayRef) -> VortexResult { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - if source.all_invalid(&mut ctx)? { +fn compress_constant_array_with_validity( + source: &ArrayRef, + ctx: &mut ExecutionCtx, +) -> VortexResult { + if source.all_invalid(ctx)? { return Ok( ConstantArray::new(Scalar::null(source.dtype().clone()), source.len()).into_array(), ); } let scalar_idx = (0..source.len()) - .position(|idx| source.is_valid(idx, &mut ctx).unwrap_or(false)) + .position(|idx| source.is_valid(idx, ctx).unwrap_or(false)) .vortex_expect("We checked that there exists a scalar that is not invalid"); - let scalar = source.execute_scalar(scalar_idx, &mut ctx)?; + let scalar = source.execute_scalar(scalar_idx, ctx)?; let const_arr = ConstantArray::new(scalar, source.len()).into_array(); - if !source.all_valid(&mut ctx)? { + if !source.all_valid(ctx)? { Ok(MaskedArray::try_new(const_arr, source.validity()?)?.into_array()) } else { Ok(const_arr) diff --git a/vortex-compressor/src/builtins/constant/string.rs b/vortex-compressor/src/builtins/constant/string.rs index 77bbb5d49c7..e723336fc3c 100644 --- a/vortex-compressor/src/builtins/constant/string.rs +++ b/vortex-compressor/src/builtins/constant/string.rs @@ -5,8 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_error::VortexResult; @@ -33,18 +32,17 @@ impl Scheme for StringConstantScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { // Constant detection on a sample is a false positive, since the sample being constant does // not mean the full array is constant. - if ctx.is_sample() { + if compress_ctx.is_sample() { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } let array_len = data.array().len(); - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.string_stats(&mut local_ctx); + let stats = data.string_stats(exec_ctx); // We want to use `Constant` if there are only nulls in the array. if stats.value_count() == 0 { @@ -62,8 +60,8 @@ impl Scheme for StringConstantScheme { // This is an expensive check, but the alternative of not compressing a constant array is // far less preferable. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |compressor, data, _ctx| { - if is_constant(data.array(), &mut compressor.execution_ctx())? { + |_compressor, data, _ctx, exec_ctx| { + if is_constant(data.array(), exec_ctx)? { Ok(EstimateVerdict::AlwaysUse) } else { Ok(EstimateVerdict::Skip) @@ -76,8 +74,9 @@ impl Scheme for StringConstantScheme { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - compress_constant_array_with_validity(data.array()) + compress_constant_array_with_validity(data.array(), exec_ctx) } } diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index e800c2c50e7..45b3097a0b0 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -9,9 +9,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -85,12 +84,10 @@ impl Scheme for FloatDictScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - // TRAIT-IMPL BOUNDARY: `Scheme::expected_compression_ratio` has a fixed signature - // and does not receive an `ExecutionCtx`, so we fall back to the legacy session here. - let mut exec_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.float_stats(&mut exec_ctx); + let stats = data.float_stats(exec_ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); @@ -113,28 +110,28 @@ impl Scheme for FloatDictScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = { - let mut exec_ctx = compressor.execution_ctx(); - data.float_stats(&mut exec_ctx).clone() - }; + let stats = data.float_stats(exec_ctx).clone(); let dict = dictionary_encode(data.array_as_primitive(), &stats)?; let has_all_values_referenced = dict.has_all_values_referenced(); // Values = child 0. - let compressed_values = compressor.compress_child(dict.values(), &ctx, self.id(), 0)?; + let compressed_values = + compressor.compress_child(dict.values(), &compress_ctx, self.id(), 0, exec_ctx)?; // Codes = child 1. let narrowed_codes = dict .codes() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()? .into_array(); - let compressed_codes = compressor.compress_child(&narrowed_codes, &ctx, self.id(), 1)?; + let compressed_codes = + compressor.compress_child(&narrowed_codes, &compress_ctx, self.id(), 1, exec_ctx)?; // SAFETY: compressing codes or values does not alter the invariants. unsafe { @@ -250,15 +247,16 @@ impl_encode!(f64, u64); #[cfg(test)] mod tests { use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_session::VortexSession; use super::dictionary_encode; use crate::stats::FloatStats; @@ -266,7 +264,9 @@ mod tests { #[test] fn test_float_dict_encode() -> VortexResult<()> { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = VortexSession::empty() + .with::() + .create_execution_ctx(); let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 13e3559e989..8f83d2840cb 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -9,9 +9,8 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -59,12 +58,11 @@ impl Scheme for IntDictScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { let bit_width = data.array_as_primitive().ptype().bit_width(); - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.integer_stats(&mut ctx); + let stats = data.integer_stats(exec_ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); @@ -106,23 +104,26 @@ impl Scheme for IntDictScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats(&mut compressor.execution_ctx()).clone(); + let stats = data.integer_stats(exec_ctx).clone(); let dict = dictionary_encode(data.array_as_primitive(), &stats)?; // Values = child 0. - let compressed_values = compressor.compress_child(dict.values(), &ctx, self.id(), 0)?; + let compressed_values = + compressor.compress_child(dict.values(), &compress_ctx, self.id(), 0, exec_ctx)?; // Codes = child 1. let narrowed_codes = dict .codes() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()? .into_array(); - let compressed_codes = compressor.compress_child(&narrowed_codes, &ctx, self.id(), 1)?; + let compressed_codes = + compressor.compress_child(&narrowed_codes, &compress_ctx, self.id(), 1, exec_ctx)?; // SAFETY: compressing codes does not change their values. unsafe { @@ -253,22 +254,25 @@ impl_encode!(i64); #[cfg(test)] mod tests { use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_session::VortexSession; use super::dictionary_encode; use crate::stats::IntegerStats; #[test] fn test_dict_encode_integer_stats() -> VortexResult<()> { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let mut ctx = VortexSession::empty() + .with::() + .create_execution_ctx(); let data = buffer![100i32, 200, 100, 0, 100]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index 82a30b2774b..935bf6c0312 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -8,9 +8,8 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArrayExt; @@ -70,11 +69,10 @@ impl Scheme for StringDictScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { - // TODO(ctx): trait fixes - Scheme::expected_compression_ratio has a fixed signature. - let mut local_ctx = LEGACY_SESSION.create_execution_ctx(); - let stats = data.string_stats(&mut local_ctx); + let stats = data.string_stats(exec_ctx); if stats.value_count() == 0 { return CompressionEstimate::Verdict(EstimateVerdict::Skip); @@ -97,21 +95,24 @@ impl Scheme for StringDictScheme { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let dict = dict_encode(data.array())?; // Values = child 0. - let compressed_values = compressor.compress_child(dict.values(), &ctx, self.id(), 0)?; + let compressed_values = + compressor.compress_child(dict.values(), &compress_ctx, self.id(), 0, exec_ctx)?; // Codes = child 1. let narrowed_codes = dict .codes() .clone() - .execute::(&mut compressor.execution_ctx())? + .execute::(exec_ctx)? .narrow()? .into_array(); - let compressed_codes = compressor.compress_child(&narrowed_codes, &ctx, self.id(), 1)?; + let compressed_codes = + compressor.compress_child(&narrowed_codes, &compress_ctx, self.id(), 1, exec_ctx)?; // SAFETY: compressing codes or values does not alter the invariants. unsafe { diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index bc739637192..da8d5ad56fa 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -3,17 +3,11 @@ //! Cascading array compression implementation. -use std::sync::Arc; - -use parking_lot::Mutex; -use parking_lot::MutexGuard; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::CanonicalValidity; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; @@ -88,11 +82,6 @@ pub struct CascadingCompressor { /// Descendant exclusion rules for the compressor's own cascading (e.g. excluding Dict from /// list offsets). root_exclusions: Vec, - - /// Shared execution context for array operations during compression. - /// - /// This should have low contention as we only execute arrays one at a time during compression. - ctx: Arc>, } impl CascadingCompressor { @@ -110,16 +99,9 @@ impl CascadingCompressor { Self { schemes, root_exclusions, - // TODO(connor): The caller should probably pass this in. - ctx: Arc::new(Mutex::new(LEGACY_SESSION.create_execution_ctx())), } } - /// Returns a mutable borrow of the execution context. - pub fn execution_ctx(&self) -> MutexGuard<'_, ExecutionCtx> { - self.ctx.lock() - } - /// Compresses an array using cascading adaptive compression. /// /// First canonicalizes and compacts the array, then applies optimal compression schemes. @@ -127,17 +109,18 @@ impl CascadingCompressor { /// # Errors /// /// Returns an error if canonicalization or compression fails. - pub fn compress(&self, array: &ArrayRef) -> VortexResult { + pub fn compress( + &self, + array: &ArrayRef, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { let before_nbytes = array.nbytes(); let span = trace::compress_span(array.len(), array.dtype(), before_nbytes); let _enter = span.enter(); - let canonical = array - .clone() - .execute::(&mut self.execution_ctx())? - .0; + let canonical = array.clone().execute::(exec_ctx)?.0; let compact = canonical.compact()?; - let compressed = self.compress_canonical(compact, CompressorContext::new())?; + let compressed = self.compress_canonical(compact, CompressorContext::new(), exec_ctx)?; trace::record_compress_outcome(&span, before_nbytes, compressed.nbytes()); @@ -159,22 +142,20 @@ impl CascadingCompressor { parent_ctx: &CompressorContext, parent_id: SchemeId, child_index: usize, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { if parent_ctx.finished_cascading() { trace::cascade_exhausted(parent_id, child_index); return Ok(child.clone()); } - let canonical = child - .clone() - .execute::(&mut self.execution_ctx())? - .0; + let canonical = child.clone().execute::(exec_ctx)?.0; let compact = canonical.compact()?; let child_ctx = parent_ctx .clone() .descend_with_scheme(parent_id, child_index); - self.compress_canonical(compact, child_ctx) + self.compress_canonical(compact, child_ctx, exec_ctx) } /// Compresses a canonical array by dispatching to type-specific logic. @@ -185,23 +166,24 @@ impl CascadingCompressor { fn compress_canonical( &self, array: Canonical, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { match array { Canonical::Null(null_array) => Ok(null_array.into_array()), Canonical::Bool(bool_array) => { - self.choose_and_compress(Canonical::Bool(bool_array), ctx) + self.choose_and_compress(Canonical::Bool(bool_array), compress_ctx, exec_ctx) } Canonical::Primitive(primitive) => { - self.choose_and_compress(Canonical::Primitive(primitive), ctx) + self.choose_and_compress(Canonical::Primitive(primitive), compress_ctx, exec_ctx) } Canonical::Decimal(decimal) => { - self.choose_and_compress(Canonical::Decimal(decimal), ctx) + self.choose_and_compress(Canonical::Decimal(decimal), compress_ctx, exec_ctx) } Canonical::Struct(struct_array) => { let fields = struct_array .iter_unmasked_fields() - .map(|field| self.compress(field)) + .map(|field| self.compress(field, exec_ctx)) .collect::, _>>()?; Ok(StructArray::try_new( @@ -215,13 +197,13 @@ impl CascadingCompressor { Canonical::List(list_view_array) => { if list_view_array.is_zero_copy_to_list() || list_view_array.elements().is_empty() { let list_array = list_from_list_view(list_view_array)?; - self.compress_list_array(list_array, ctx) + self.compress_list_array(list_array, compress_ctx, exec_ctx) } else { - self.compress_list_view_array(list_view_array, ctx) + self.compress_list_view_array(list_view_array, compress_ctx, exec_ctx) } } Canonical::FixedSizeList(fsl_array) => { - let compressed_elems = self.compress(fsl_array.elements())?; + let compressed_elems = self.compress(fsl_array.elements(), exec_ctx)?; Ok(FixedSizeListArray::try_new( compressed_elems, @@ -236,7 +218,7 @@ impl CascadingCompressor { .dtype() .eq_ignore_nullability(&DType::Utf8(Nullability::NonNullable)) { - self.choose_and_compress(Canonical::VarBinView(strings), ctx) + self.choose_and_compress(Canonical::VarBinView(strings), compress_ctx, exec_ctx) } else { // We do not compress binary arrays. Ok(strings.into_array()) @@ -246,8 +228,11 @@ impl CascadingCompressor { let before_nbytes = ext_array.as_ref().nbytes(); // Try scheme-based compression first. - let result = - self.choose_and_compress(Canonical::Extension(ext_array.clone()), ctx)?; + let result = self.choose_and_compress( + Canonical::Extension(ext_array.clone()), + compress_ctx, + exec_ctx, + )?; if result.nbytes() < before_nbytes { return Ok(result); } @@ -258,7 +243,7 @@ impl CascadingCompressor { } // Otherwise, fall back to compressing the underlying storage array. - let compressed_storage = self.compress(ext_array.storage_array())?; + let compressed_storage = self.compress(ext_array.storage_array(), exec_ctx)?; Ok( ExtensionArray::new(ext_array.ext_dtype().clone(), compressed_storage) @@ -287,13 +272,14 @@ impl CascadingCompressor { fn choose_and_compress( &self, canonical: Canonical, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let eligible_schemes: Vec<&'static dyn Scheme> = self .schemes .iter() .copied() - .filter(|s| s.matches(&canonical) && !self.is_excluded(*s, &ctx)) + .filter(|s| s.matches(&canonical) && !self.is_excluded(*s, &compress_ctx)) .collect(); let array: ArrayRef = canonical.into(); @@ -302,7 +288,7 @@ impl CascadingCompressor { return Ok(array); } - if array.all_invalid(&mut self.execution_ctx())? { + if array.all_invalid(exec_ctx)? { return Ok( ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()).into_array(), ); @@ -315,20 +301,20 @@ impl CascadingCompressor { .fold(GenerateStatsOptions::default(), |acc, s| { acc.merge(s.stats_options()) }); - let ctx = ctx.with_merged_stats_options(merged_opts); + let compress_ctx = compress_ctx.with_merged_stats_options(merged_opts); let mut data = ArrayAndStats::new(array, merged_opts); let Some((winner, winner_estimate)) = - self.choose_best_scheme(&eligible_schemes, &mut data, ctx.clone())? + self.choose_best_scheme(&eligible_schemes, &mut data, compress_ctx.clone(), exec_ctx)? else { return Ok(data.into_array()); }; // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the // scheme name and cascade history before propagating. - let error_ctx = trace::enabled_error_context(&ctx); - let compressed = match winner.compress(self, &mut data, ctx) { + let error_ctx = trace::enabled_error_context(&compress_ctx); + let compressed = match winner.compress(self, &mut data, compress_ctx, exec_ctx) { Ok(compressed) => compressed, Err(err) => { // NB: this is the only way we can tell which scheme panicked / bailed on their @@ -370,31 +356,34 @@ impl CascadingCompressor { &self, schemes: &[&'static dyn Scheme], data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut best: Option<(&'static dyn Scheme, EstimateScore)> = None; // TODO(connor): Rather than computing the deferred estimates eagerly, it would be better to // look at all quick estimates and see if it makes sense to sample at all. for &scheme in schemes { - let verdict = match scheme.expected_compression_ratio(data, ctx.clone()) { - CompressionEstimate::Verdict(verdict) => verdict, - CompressionEstimate::Deferred(DeferredEstimate::Sample) => { - let score = estimate_compression_ratio_with_sampling( - scheme, - self, - data.array(), - ctx.clone(), - )?; - if is_better_score(score, &best) { - best = Some((scheme, score)); + let verdict = + match scheme.expected_compression_ratio(data, compress_ctx.clone(), exec_ctx) { + CompressionEstimate::Verdict(verdict) => verdict, + CompressionEstimate::Deferred(DeferredEstimate::Sample) => { + let score = estimate_compression_ratio_with_sampling( + scheme, + self, + data.array(), + compress_ctx.clone(), + exec_ctx, + )?; + if is_better_score(score, &best) { + best = Some((scheme, score)); + } + continue; } - continue; - } - CompressionEstimate::Deferred(DeferredEstimate::Callback(callback)) => { - callback(self, data, ctx.clone())? - } - }; + CompressionEstimate::Deferred(DeferredEstimate::Callback(callback)) => { + callback(self, data, compress_ctx.clone(), exec_ctx)? + } + }; match verdict { EstimateVerdict::Skip => {} @@ -465,21 +454,26 @@ impl CascadingCompressor { fn compress_list_array( &self, list_array: ListArray, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let list_array = list_array.reset_offsets(true)?; - let compressed_elems = self.compress(list_array.elements())?; + let compressed_elems = self.compress(list_array.elements(), exec_ctx)?; // Record the root scheme with the offsets child index so root exclusion rules apply. - let offset_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); + let offset_ctx = + compress_ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); let list_offsets_primitive = list_array .offsets() .clone() - .execute::(&mut self.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_offsets = - self.compress_canonical(Canonical::Primitive(list_offsets_primitive), offset_ctx)?; + let compressed_offsets = self.compress_canonical( + Canonical::Primitive(list_offsets_primitive), + offset_ctx, + exec_ctx, + )?; Ok( ListArray::try_new(compressed_elems, compressed_offsets, list_array.validity()?)? @@ -492,31 +486,36 @@ impl CascadingCompressor { fn compress_list_view_array( &self, list_view: ListViewArray, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let compressed_elems = self.compress(list_view.elements())?; + let compressed_elems = self.compress(list_view.elements(), exec_ctx)?; - let offset_ctx = ctx + let offset_ctx = compress_ctx .clone() .descend_with_scheme(ROOT_SCHEME_ID, root_list_children::OFFSETS); let list_view_offsets_primitive = list_view .offsets() .clone() - .execute::(&mut self.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; let compressed_offsets = self.compress_canonical( Canonical::Primitive(list_view_offsets_primitive), offset_ctx, + exec_ctx, )?; - let sizes_ctx = ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::SIZES); + let sizes_ctx = compress_ctx.descend_with_scheme(ROOT_SCHEME_ID, root_list_children::SIZES); let list_view_sizes_primitive = list_view .sizes() .clone() - .execute::(&mut self.execution_ctx())? + .execute::(exec_ctx)? .narrow()?; - let compressed_sizes = - self.compress_canonical(Canonical::Primitive(list_view_sizes_primitive), sizes_ctx)?; + let compressed_sizes = self.compress_canonical( + Canonical::Primitive(list_view_sizes_primitive), + sizes_ctx, + exec_ctx, + )?; Ok(ListViewArray::try_new( compressed_elems, @@ -532,7 +531,9 @@ impl CascadingCompressor { mod tests { use std::collections::BTreeMap; use std::sync::Arc; + use std::sync::LazyLock; + use parking_lot::Mutex; use tracing::Event; use tracing::Subscriber; use tracing::field::Field; @@ -542,12 +543,15 @@ mod tests { use tracing_subscriber::prelude::*; use vortex_array::ArrayRef; use vortex_array::Canonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Constant; use vortex_array::arrays::NullArray; use vortex_array::arrays::PrimitiveArray; + use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; + use vortex_session::VortexSession; use super::*; use crate::builtins::FloatDictScheme; @@ -562,6 +566,9 @@ mod tests { use crate::scheme::SchemeExt; use crate::trace::TARGET_TRACE; + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + fn compressor() -> CascadingCompressor { CascadingCompressor::new(vec![&IntDictScheme, &FloatDictScheme, &StringDictScheme]) } @@ -682,7 +689,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::Ratio(2.0)) } @@ -691,7 +699,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -712,7 +721,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } @@ -721,7 +731,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -742,10 +753,11 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx| Ok(EstimateVerdict::AlwaysUse), + |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::AlwaysUse), ))) } @@ -753,7 +765,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -774,10 +787,11 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx| Ok(EstimateVerdict::Skip), + |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::Skip), ))) } @@ -785,7 +799,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -806,10 +821,11 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx| Ok(EstimateVerdict::Ratio(3.0)), + |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(3.0)), ))) } @@ -817,7 +833,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -838,7 +855,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::Ratio(100.0)) } @@ -847,7 +865,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { unreachable!("test helper should never be selected for compression") } @@ -868,7 +887,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } @@ -877,7 +897,8 @@ mod tests { &self, _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { Ok(NullArray::new(data.array().len()).into_array()) } @@ -898,7 +919,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } @@ -907,9 +929,10 @@ mod tests { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - compressor.compress_child(data.array(), &ctx, self.id(), 1) + compressor.compress_child(data.array(), &compress_ctx, self.id(), 1, exec_ctx) } } @@ -928,7 +951,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } @@ -937,7 +961,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { vortex_error::vortex_bail!("nested failure") } @@ -958,7 +983,8 @@ mod tests { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Sample) } @@ -967,7 +993,8 @@ mod tests { &self, _compressor: &CascadingCompressor, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { vortex_error::vortex_bail!("sample failure") } @@ -1028,9 +1055,14 @@ mod tests { CascadingCompressor::new(vec![&DirectRatioScheme, &ImmediateAlwaysUseScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &ImmediateAlwaysUseScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1046,9 +1078,14 @@ mod tests { CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackAlwaysUseScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackAlwaysUseScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1063,9 +1100,14 @@ mod tests { let compressor = CascadingCompressor::new(vec![&CallbackSkipScheme, &DirectRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&CallbackSkipScheme, &DirectRatioScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1080,9 +1122,14 @@ mod tests { let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackRatioScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1097,9 +1144,14 @@ mod tests { let compressor = CascadingCompressor::new(vec![&HugeRatioScheme, &ZeroBytesSamplingScheme]); let schemes: [&'static dyn Scheme; 2] = [&HugeRatioScheme, &ZeroBytesSamplingScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1114,9 +1166,14 @@ mod tests { let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme, &HugeRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&ZeroBytesSamplingScheme, &HugeRatioScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(matches!( winner, @@ -1131,9 +1188,14 @@ mod tests { let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); let schemes: [&'static dyn Scheme; 1] = [&ZeroBytesSamplingScheme]; let mut data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); - let winner = - compressor.choose_best_scheme(&schemes, &mut data, CompressorContext::new())?; + let winner = compressor.choose_best_scheme( + &schemes, + &mut data, + CompressorContext::new(), + &mut exec_ctx, + )?; assert!(winner.is_none()); Ok(()) @@ -1150,7 +1212,8 @@ mod tests { // The compressor should produce a `ConstantArray` for an all-null array regardless of // which schemes are registered. let compressor = CascadingCompressor::new(vec![&IntDictScheme]); - let compressed = compressor.compress(&array)?; + let mut exec_ctx = SESSION.create_execution_ctx(); + let compressed = compressor.compress(&array, &mut exec_ctx)?; assert!(compressed.is::()); Ok(()) } @@ -1179,8 +1242,14 @@ mod tests { // Before the fix this panicked with: // "this must be present since `DictScheme` declared that we need distinct values" - let score = - estimate_compression_ratio_with_sampling(&FloatDictScheme, &compressor, &array, ctx)?; + let mut exec_ctx = SESSION.create_execution_ctx(); + let score = estimate_compression_ratio_with_sampling( + &FloatDictScheme, + &compressor, + &array, + ctx, + &mut exec_ctx, + )?; assert!(matches!(score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite())); Ok(()) } @@ -1191,7 +1260,10 @@ mod tests { CascadingCompressor::new(vec![&NestedFailureParentScheme, &NestedFailureLeafScheme]); let array = test_integer_array(); - let (result, events) = record_events(|| compressor.compress(&array)); + let (result, events) = record_events(|| { + let mut exec_ctx = SESSION.create_execution_ctx(); + compressor.compress(&array, &mut exec_ctx) + }); assert!(result.is_err()); let event = find_event(&events, TARGET_TRACE, "scheme.compress_failed"); @@ -1214,7 +1286,10 @@ mod tests { let compressor = CascadingCompressor::new(vec![&SamplingFailureScheme]); let array = test_integer_array(); - let (result, events) = record_events(|| compressor.compress(&array)); + let (result, events) = record_events(|| { + let mut exec_ctx = SESSION.create_execution_ctx(); + compressor.compress(&array, &mut exec_ctx) + }); assert!(result.is_err()); let event = find_event(&events, TARGET_TRACE, "sample.compress_failed"); @@ -1237,7 +1312,10 @@ mod tests { let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); let array = test_integer_array(); - let (result, events) = record_events(|| compressor.compress(&array)); + let (result, events) = record_events(|| { + let mut exec_ctx = SESSION.create_execution_ctx(); + compressor.compress(&array, &mut exec_ctx) + }); assert!(result.is_ok()); diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 22c6ffb9223..4fed4cae323 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -7,6 +7,7 @@ use std::fmt; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_error::VortexResult; @@ -29,6 +30,7 @@ pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &mut ArrayAndStats, CompressorContext, + &mut ExecutionCtx, ) -> VortexResult + Send + Sync; @@ -179,23 +181,23 @@ pub(super) fn estimate_compression_ratio_with_sampling( scheme: &S, compressor: &CascadingCompressor, array: &ArrayRef, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let sample_array = if ctx.is_sample() { + let sample_array = if compress_ctx.is_sample() { array.clone() } else { let sample_count = sample_count_approx_one_percent(array.len()); // `ArrayAndStats` expects a canonical array (so that it can easily compute lazy stats). - let canonical: Canonical = - sample(array, SAMPLE_SIZE, sample_count).execute(&mut compressor.execution_ctx())?; + let canonical: Canonical = sample(array, SAMPLE_SIZE, sample_count).execute(exec_ctx)?; canonical.into_array() }; let mut sample_data = ArrayAndStats::new(sample_array, scheme.stats_options()); - let error_ctx = trace::enabled_error_context(&ctx); - let sample_ctx = ctx.with_sampling(); + let error_ctx = trace::enabled_error_context(&compress_ctx); + let sample_ctx = compress_ctx.with_sampling(); - let compressed = match scheme.compress(compressor, &mut sample_data, sample_ctx) { + let compressed = match scheme.compress(compressor, &mut sample_data, sample_ctx, exec_ctx) { Ok(compressed) => compressed, Err(err) => { trace::sample_compress_failed(scheme.id(), error_ctx.as_ref(), &err); diff --git a/vortex-compressor/src/scheme.rs b/vortex-compressor/src/scheme.rs index fe1fee7a4c3..24582b1bc2e 100644 --- a/vortex-compressor/src/scheme.rs +++ b/vortex-compressor/src/scheme.rs @@ -10,6 +10,7 @@ use std::hash::Hasher; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_error::VortexResult; use crate::CascadingCompressor; @@ -211,7 +212,8 @@ pub trait Scheme: Debug + Send + Sync { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate; /// Compress the array using this scheme. @@ -223,7 +225,8 @@ pub trait Scheme: Debug + Send + Sync { &self, compressor: &CascadingCompressor, data: &mut ArrayAndStats, - ctx: CompressorContext, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult; } diff --git a/vortex-layout/public-api.lock b/vortex-layout/public-api.lock index c98f4c6e04f..6b029d9c50d 100644 --- a/vortex-layout/public-api.lock +++ b/vortex-layout/public-api.lock @@ -186,19 +186,19 @@ pub fn vortex_layout::layouts::compressed::CompressingStrategy::write_stream<'li pub trait vortex_layout::layouts::compressed::CompressorPlugin: core::marker::Send + core::marker::Sync + 'static -pub fn vortex_layout::layouts::compressed::CompressorPlugin::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_layout::layouts::compressed::CompressorPlugin::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_layout::layouts::compressed::CompressorPlugin for alloc::sync::Arc -pub fn alloc::sync::Arc::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn alloc::sync::Arc::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult impl vortex_layout::layouts::compressed::CompressorPlugin for vortex_btrblocks::canonical_compressor::BtrBlocksCompressor -pub fn vortex_btrblocks::canonical_compressor::BtrBlocksCompressor::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn vortex_btrblocks::canonical_compressor::BtrBlocksCompressor::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_layout::layouts::compressed::CompressorPlugin for F where F: core::ops::function::Fn(&vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync + 'static +impl vortex_layout::layouts::compressed::CompressorPlugin for F where F: core::ops::function::Fn(&vortex_array::array::erased::ArrayRef, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync + 'static -pub fn F::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult +pub fn F::compress_chunk(&self, chunk: &vortex_array::array::erased::ArrayRef, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_layout::layouts::dict diff --git a/vortex-layout/src/layouts/compressed.rs b/vortex-layout/src/layouts/compressed.rs index fe75f66c946..8ed69e756cb 100644 --- a/vortex-layout/src/layouts/compressed.rs +++ b/vortex-layout/src/layouts/compressed.rs @@ -7,6 +7,7 @@ use async_trait::async_trait; use futures::StreamExt as _; use vortex_array::ArrayContext; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::VortexSessionExecute; use vortex_array::expr::stats::Stat; use vortex_btrblocks::BtrBlocksCompressor; @@ -26,27 +27,27 @@ use crate::sequence::SequentialStreamExt; /// /// API consumers are free to implement this trait to provide new plugin compressors. pub trait CompressorPlugin: Send + Sync + 'static { - fn compress_chunk(&self, chunk: &ArrayRef) -> VortexResult; + fn compress_chunk(&self, chunk: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult; } impl CompressorPlugin for Arc { - fn compress_chunk(&self, chunk: &ArrayRef) -> VortexResult { - self.as_ref().compress_chunk(chunk) + fn compress_chunk(&self, chunk: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + self.as_ref().compress_chunk(chunk, ctx) } } impl CompressorPlugin for F where - F: Fn(&ArrayRef) -> VortexResult + Send + Sync + 'static, + F: Fn(&ArrayRef, &mut ExecutionCtx) -> VortexResult + Send + Sync + 'static, { - fn compress_chunk(&self, chunk: &ArrayRef) -> VortexResult { - self(chunk) + fn compress_chunk(&self, chunk: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + self(chunk, ctx) } } impl CompressorPlugin for BtrBlocksCompressor { - fn compress_chunk(&self, chunk: &ArrayRef) -> VortexResult { - self.compress(chunk) + fn compress_chunk(&self, chunk: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + self.compress(chunk, ctx) } } @@ -109,11 +110,10 @@ impl LayoutStrategy for CompressingStrategy { let session = compute_session.clone(); handle.spawn_cpu(move || { let (sequence_id, chunk) = chunk?; + let mut ctx = session.create_execution_ctx(); // Compute the stats for the chunk prior to compression - chunk - .statistics() - .compute_all(&stats, &mut session.create_execution_ctx())?; - Ok((sequence_id, compressor.compress_chunk(&chunk)?)) + chunk.statistics().compute_all(&stats, &mut ctx)?; + Ok((sequence_id, compressor.compress_chunk(&chunk, &mut ctx)?)) }) }) .buffered(self.concurrency); diff --git a/vortex-layout/src/layouts/dict/writer.rs b/vortex-layout/src/layouts/dict/writer.rs index a1ca9a2b9df..7013becd4df 100644 --- a/vortex-layout/src/layouts/dict/writer.rs +++ b/vortex-layout/src/layouts/dict/writer.rs @@ -20,6 +20,7 @@ use futures::stream::once; use futures::try_join; use vortex_array::ArrayContext; use vortex_array::ArrayRef; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Dict; use vortex_array::builders::dict::DictConstraints; use vortex_array::builders::dict::DictEncoder; @@ -151,7 +152,8 @@ impl LayoutStrategy for DictStrategy { let should_fallback = match first_chunk { None => true, // empty stream Some(chunk) => { - let compressed = BtrBlocksCompressor::default().compress(&chunk)?; + let mut exec_ctx = session.create_execution_ctx(); + let compressed = BtrBlocksCompressor::default().compress(&chunk, &mut exec_ctx)?; !compressed.is::() } }; diff --git a/vortex-python/src/compress.rs b/vortex-python/src/compress.rs index ef63426acae..fa77cbb1800 100644 --- a/vortex-python/src/compress.rs +++ b/vortex-python/src/compress.rs @@ -2,8 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use pyo3::prelude::*; +use vortex::array::VortexSessionExecute; use vortex::compressor::BtrBlocksCompressor; +use crate::SESSION; use crate::arrays::PyArrayRef; use crate::error::PyVortexResult; use crate::install_module; @@ -51,6 +53,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { /// 'vortex.alp(f64?, len=1000)' #[pyfunction] pub fn compress(array: PyArrayRef) -> PyVortexResult { - let compressed = BtrBlocksCompressor::default().compress(array.inner())?; + let compressed = BtrBlocksCompressor::default() + .compress(array.inner(), &mut SESSION.create_execution_ctx())?; Ok(PyArrayRef::from(compressed)) } diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index d233322a3bf..49c21c193e9 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -12,9 +12,9 @@ pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::fmt(&self, f: &mut c impl vortex_compressor::scheme::Scheme for vortex_tensor::encodings::l2_denorm::L2DenormScheme -pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -64,9 +64,9 @@ impl core::marker::StructuralPartialEq for vortex_tensor::encodings::turboquant: impl vortex_compressor::scheme::Scheme for vortex_tensor::encodings::turboquant::TurboQuantScheme -pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _ctx: vortex_compressor::ctx::CompressorContext) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index d29b2e94daf..b21595db003 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -3,6 +3,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; @@ -33,19 +34,20 @@ impl Scheme for L2DenormScheme { fn expected_compression_ratio( &self, _data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) } fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let l2_denorm = - normalize_as_l2_denorm(data.array().clone(), &mut compressor.execution_ctx())?; + let l2_denorm = normalize_as_l2_denorm(data.array().clone(), exec_ctx)?; Ok(l2_denorm.into_array()) } } diff --git a/vortex-tensor/src/encodings/turboquant/scheme.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs index 81fca209a8d..c2ea8f91e1f 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -21,6 +21,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; use vortex_compressor::estimate::CompressionEstimate; @@ -71,7 +72,8 @@ impl Scheme for TurboQuantScheme { fn expected_compression_ratio( &self, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { let len = data.array().len(); let dtype = data.array().dtype(); @@ -94,12 +96,12 @@ impl Scheme for TurboQuantScheme { fn compress( &self, - compressor: &CascadingCompressor, + _compressor: &CascadingCompressor, data: &mut ArrayAndStats, - _ctx: CompressorContext, + _compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - let mut ctx = compressor.execution_ctx(); - turboquant_encode(data.array().clone(), &TurboQuantConfig::default(), &mut ctx) + turboquant_encode(data.array().clone(), &TurboQuantConfig::default(), exec_ctx) } } diff --git a/vortex/examples/compression_showcase.rs b/vortex/examples/compression_showcase.rs index e7f80fff657..6aa9cf08218 100644 --- a/vortex/examples/compression_showcase.rs +++ b/vortex/examples/compression_showcase.rs @@ -8,8 +8,10 @@ //! //! Run with: cargo run --example compression_showcase +use vortex::VortexSessionDefault; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinArray; @@ -17,6 +19,7 @@ use vortex::array::validity::Validity; use vortex::compressor::BtrBlocksCompressor; use vortex::dtype::DType; use vortex::dtype::Nullability; +use vortex::session::VortexSession; use vortex_buffer::Buffer; fn main() -> Result<(), Box> { @@ -25,35 +28,37 @@ fn main() -> Result<(), Box> { println!("This example demonstrates how Vortex automatically selects"); println!("optimal compression strategies for different data patterns.\n"); + let session = VortexSession::default(); + // 1. Compress sequential/monotonic data println!("1. Sequential Data Compression:"); - compress_sequential_data()?; + compress_sequential_data(&session)?; // 2. Compress repetitive data println!("\n2. Repetitive Data Compression:"); - compress_repetitive_data()?; + compress_repetitive_data(&session)?; // 3. Compress string data println!("\n3. String Data Compression:"); - compress_string_data()?; + compress_string_data(&session)?; // 4. Compress floating-point data println!("\n4. Floating-Point Data Compression:"); - compress_float_data()?; + compress_float_data(&session)?; // 5. Compress sparse data println!("\n5. Sparse Data Compression:"); - compress_sparse_data()?; + compress_sparse_data(&session)?; // 6. Compress structured data println!("\n6. Structured Data Compression:"); - compress_structured_data()?; + compress_structured_data(&session)?; println!("\n=== Compression showcase completed! ==="); Ok(()) } -fn compress_sequential_data() -> Result<(), Box> { +fn compress_sequential_data(session: &VortexSession) -> Result<(), Box> { // Create sequential data (e.g., timestamps, IDs) let sequential: PrimitiveArray = (1000..11000).map(|i| i as u64).collect(); @@ -63,7 +68,10 @@ fn compress_sequential_data() -> Result<(), Box> { // Compress using default strategy let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&sequential.into_array())?; + let compressed = compressor.compress( + &sequential.into_array(), + &mut session.create_execution_ctx(), + )?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; @@ -76,7 +84,7 @@ fn compress_sequential_data() -> Result<(), Box> { Ok(()) } -fn compress_repetitive_data() -> Result<(), Box> { +fn compress_repetitive_data(session: &VortexSession) -> Result<(), Box> { // Create highly repetitive data (run-length encoding opportunity) let mut repetitive = Vec::new(); for i in 0..100 { @@ -91,7 +99,8 @@ fn compress_repetitive_data() -> Result<(), Box> { println!(" Uncompressed size: ~{} bytes", uncompressed_size); let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&array.into_array())?; + let compressed = + compressor.compress(&array.into_array(), &mut session.create_execution_ctx())?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; @@ -104,7 +113,7 @@ fn compress_repetitive_data() -> Result<(), Box> { Ok(()) } -fn compress_string_data() -> Result<(), Box> { +fn compress_string_data(session: &VortexSession) -> Result<(), Box> { // Create string data with patterns let categories = vec!["Electronics", "Clothing", "Food", "Books"]; let mut strings = Vec::new(); @@ -123,7 +132,8 @@ fn compress_string_data() -> Result<(), Box> { println!(" Uncompressed size: ~{} bytes", uncompressed_size); let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&array.into_array())?; + let compressed = + compressor.compress(&array.into_array(), &mut session.create_execution_ctx())?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; @@ -136,7 +146,7 @@ fn compress_string_data() -> Result<(), Box> { Ok(()) } -fn compress_float_data() -> Result<(), Box> { +fn compress_float_data(session: &VortexSession) -> Result<(), Box> { // Create floating-point data with patterns let floats: Buffer = (0..10000).map(|i| (i as f64) * 0.1 + 100.0).collect(); let array = floats.into_array(); @@ -146,7 +156,7 @@ fn compress_float_data() -> Result<(), Box> { println!(" Uncompressed size: ~{} bytes", uncompressed_size); let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&array)?; + let compressed = compressor.compress(&array, &mut session.create_execution_ctx())?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; @@ -159,7 +169,7 @@ fn compress_float_data() -> Result<(), Box> { Ok(()) } -fn compress_sparse_data() -> Result<(), Box> { +fn compress_sparse_data(session: &VortexSession) -> Result<(), Box> { // Create sparse data (mostly zeros with few non-zero values) let mut sparse = vec![0i64; 10000]; for i in (0..10000).step_by(100) { @@ -172,7 +182,8 @@ fn compress_sparse_data() -> Result<(), Box> { println!(" Uncompressed size: ~{} bytes", uncompressed_size); let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&array.into_array())?; + let compressed = + compressor.compress(&array.into_array(), &mut session.create_execution_ctx())?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; @@ -185,7 +196,7 @@ fn compress_sparse_data() -> Result<(), Box> { Ok(()) } -fn compress_structured_data() -> Result<(), Box> { +fn compress_structured_data(session: &VortexSession) -> Result<(), Box> { // Create a struct array with multiple columns let size = 5000; @@ -222,7 +233,10 @@ fn compress_structured_data() -> Result<(), Box> { println!(" Uncompressed size: ~{} bytes", uncompressed_size); let compressor = BtrBlocksCompressor::default(); - let compressed = compressor.compress(&struct_array.into_array())?; + let compressed = compressor.compress( + &struct_array.into_array(), + &mut session.create_execution_ctx(), + )?; let compressed_size = compressed.nbytes(); let ratio = uncompressed_size as f64 / compressed_size as f64; diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 9053dbc7505..12b71709b72 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -246,7 +246,11 @@ mod test { let array = PrimitiveArray::new(buffer![42u64; 100_000], Validity::NonNullable); // You can compress an array in-memory with the BtrBlocks compressor - let compressed = BtrBlocksCompressor::default().compress(&array.clone().into_array())?; + let session = VortexSession::default(); + let compressed = BtrBlocksCompressor::default().compress( + &array.clone().into_array(), + &mut session.create_execution_ctx(), + )?; println!( "BtrBlocks size: {} / {}", compressed.nbytes(), diff --git a/wasm-test/src/main.rs b/wasm-test/src/main.rs index 67617c37955..964d3a36c9a 100644 --- a/wasm-test/src/main.rs +++ b/wasm-test/src/main.rs @@ -2,10 +2,13 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex::array::IntoArray; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::validity::Validity; use vortex::buffer::buffer; use vortex::compressor::BtrBlocksCompressor; +use vortex::session::VortexSession; +use vortex::VortexSessionDefault; //use wasm_bindgen::prelude::*; @@ -13,7 +16,10 @@ pub fn main() { // Extremely simple test of compression/decompression and a few compute functions. let array = PrimitiveArray::new(buffer![1i32; 1024], Validity::AllValid).into_array(); - let compressed = BtrBlocksCompressor::default().compress(&array).unwrap(); + let session = VortexSession::default(); + let compressed = BtrBlocksCompressor::default() + .compress(&array, &mut session.create_execution_ctx()) + .unwrap(); println!("Compressed size: {}", compressed.len()); println!("Tree view: {}", compressed.display_tree()); } From b66f299be438aa5e7879522bd8bae4f69801deef Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Tue, 21 Apr 2026 10:38:24 -0400 Subject: [PATCH 151/250] alp CUDA bench (#7579) ## Summary Adds a benchmark for CUDA execution of ALP arrays, similar to our existing one for bit unpacking. --------- Signed-off-by: Andrew Duffy --- vortex-cuda/Cargo.toml | 4 + vortex-cuda/benches/alp_cuda.rs | 146 ++++++++++++++++++++ vortex-cuda/benches/bitpacked_cuda.rs | 4 +- vortex-cuda/benches/common/mod.rs | 7 + vortex-cuda/benches/date_time_parts_cuda.rs | 2 +- vortex-cuda/benches/dict_cuda.rs | 2 +- vortex-cuda/benches/for_cuda.rs | 4 +- vortex-cuda/benches/runend_cuda.rs | 2 +- 8 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 vortex-cuda/benches/alp_cuda.rs diff --git a/vortex-cuda/Cargo.toml b/vortex-cuda/Cargo.toml index edf91b00def..a58c1c379ce 100644 --- a/vortex-cuda/Cargo.toml +++ b/vortex-cuda/Cargo.toml @@ -83,6 +83,10 @@ harness = false name = "bitpacked_cuda" harness = false +[[bench]] +name = "alp_cuda" +harness = false + [[bench]] name = "dynamic_dispatch_cuda" harness = false diff --git a/vortex-cuda/benches/alp_cuda.rs b/vortex-cuda/benches/alp_cuda.rs new file mode 100644 index 00000000000..11e9b894dd6 --- /dev/null +++ b/vortex-cuda/benches/alp_cuda.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! CUDA benchmarks for ALP decompression. + +#![expect(clippy::unwrap_used)] +#![expect(clippy::cast_possible_truncation)] + +mod common; + +use std::mem::size_of; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use criterion::BenchmarkId; +use criterion::Criterion; +use criterion::Throughput; +use cudarc::driver::DeviceRepr; +use futures::executor::block_on; +use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; +use vortex::array::arrays::PrimitiveArray; +use vortex::array::validity::Validity::NonNullable; +use vortex::buffer::Buffer; +use vortex::dtype::NativePType; +use vortex::encodings::alp::ALPArray; +use vortex::encodings::alp::ALPArrayExt; +use vortex::encodings::alp::ALPFloat; +use vortex::encodings::alp::alp_encode; +use vortex::error::VortexExpect; +use vortex::session::VortexSession; +use vortex_cuda::CudaSession; +use vortex_cuda::executor::CudaArrayExt; +use vortex_cuda_macros::cuda_available; +use vortex_cuda_macros::cuda_not_available; + +use crate::common::TimedLaunchStrategy; + +const N_ROWS: usize = 100_000_000; + +/// Patch frequencies to benchmark (as fractions). +const PATCH_FREQUENCIES: &[(f64, &str)] = &[(0.0, "0%"), (0.01, "1%"), (0.10, "10%")]; + +/// Create an ALP-encoded array of `len` floats with the requested patch frequency. +/// +/// Base values are integer-valued floats which encode cleanly. When `patch_frequency > 0`, +/// PI is placed at regular intervals; PI cannot round-trip through ALP at the chosen +/// exponents, so each PI becomes a patch. +fn make_alp_array(len: usize, patch_frequency: f64) -> ALPArray +where + T: ALPFloat + NativePType, +{ + let patch_interval = if patch_frequency > 0.0 { + (1.0 / patch_frequency) as usize + } else { + usize::MAX + }; + let outlier = T::from(std::f64::consts::PI).unwrap(); + + let values: Buffer = (0..len) + .map(|i| { + if patch_interval != usize::MAX && i % patch_interval == 0 { + outlier + } else { + T::from((i % 256) as u32).unwrap() + } + }) + .collect(); + + let primitive_array = PrimitiveArray::new(values, NonNullable); + let encoded = alp_encode( + primitive_array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("failed to ALP-encode array"); + + if patch_frequency > 0.0 { + assert!( + encoded.patches().is_some(), + "expected patches for patch_frequency={patch_frequency}", + ); + } + + encoded +} + +fn benchmark_alp_decode_typed(c: &mut Criterion, type_name: &str) +where + T: ALPFloat + NativePType + DeviceRepr, +{ + let mut group = c.benchmark_group(format!("alp_cuda_{}", type_name)); + + let nbytes = N_ROWS * size_of::(); + group.throughput(Throughput::Bytes(nbytes as u64)); + + for &(patch_freq, patch_label) in PATCH_FREQUENCIES { + let array = make_alp_array::(N_ROWS, patch_freq); + + group.bench_with_input( + BenchmarkId::new("alp_decode", patch_label), + &array, + |b, array| { + b.iter_custom(|iters| { + let timed = TimedLaunchStrategy::default(); + let timer = timed.timer(); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); + + for _ in 0..iters { + block_on(array.clone().into_array().execute_cuda(&mut cuda_ctx)).unwrap(); + } + + Duration::from_nanos(timer.load(Ordering::Relaxed)) + }); + }, + ); + } + + group.finish(); +} + +fn benchmark_alp_decode(c: &mut Criterion) { + benchmark_alp_decode_typed::(c, "f32"); + benchmark_alp_decode_typed::(c, "f64"); +} + +criterion::criterion_group! { + name = benches; + config = Criterion::default().without_plots() + .sample_size(10) + .warm_up_time(Duration::from_nanos(1)) + .measurement_time(Duration::from_nanos(1)) + .nresamples(10); + targets = benchmark_alp_decode +} + +#[cuda_available] +criterion::criterion_main!(benches); + +#[cuda_not_available] +fn main() {} diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index 88015a1cbfb..449061e6978 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -123,7 +123,7 @@ where |b, array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") @@ -168,7 +168,7 @@ where |b, array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") diff --git a/vortex-cuda/benches/common/mod.rs b/vortex-cuda/benches/common/mod.rs index 71e21c85c34..d17c5a41b68 100644 --- a/vortex-cuda/benches/common/mod.rs +++ b/vortex-cuda/benches/common/mod.rs @@ -16,6 +16,13 @@ pub struct TimedLaunchStrategy { pub total_time_ns: Arc, } +impl TimedLaunchStrategy { + /// Returns a shared handle to the accumulated kernel time, for reading after launches complete. + pub fn timer(&self) -> Arc { + Arc::clone(&self.total_time_ns) + } +} + impl LaunchStrategy for TimedLaunchStrategy { fn event_flags(&self) -> CUevent_flags { // using blocking_sync to make sure all events flush before we complete. diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index 8a1e9a91641..da696681418 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -64,7 +64,7 @@ fn benchmark_datetimeparts(c: &mut Criterion) { |b, dtp_array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") diff --git a/vortex-cuda/benches/dict_cuda.rs b/vortex-cuda/benches/dict_cuda.rs index fd6a1206773..1e1a51d3a8a 100644 --- a/vortex-cuda/benches/dict_cuda.rs +++ b/vortex-cuda/benches/dict_cuda.rs @@ -92,7 +92,7 @@ where |b, dict_array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index 16f54013176..29402737a52 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -87,7 +87,7 @@ where |b, for_array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") @@ -125,7 +125,7 @@ where |b, for_array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") diff --git a/vortex-cuda/benches/runend_cuda.rs b/vortex-cuda/benches/runend_cuda.rs index 1606c6a4dcd..31c59037b0d 100644 --- a/vortex-cuda/benches/runend_cuda.rs +++ b/vortex-cuda/benches/runend_cuda.rs @@ -82,7 +82,7 @@ where |b, runend_array| { b.iter_custom(|iters| { let timed = TimedLaunchStrategy::default(); - let timer = Arc::clone(&timed.total_time_ns); + let timer = timed.timer(); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) From 3e00b5ac63228ba1bbf788c192944b143c319a1f Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:05:51 -0400 Subject: [PATCH 152/250] Make `StatsCache` and `ArrayAndStats` have interior mutability (#7583) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 It was unfortunate that `ArrayAndStats` had to be taken by `&mut`, so this change makes updates use interior mutability. ## API Changes `Scheme` takes `ArrayAndStats` by shared reference instead of by exclusive reference. ## Testing N/A since there isn't any semantic change here, just API changes. Signed-off-by: Connor Tsui --- vortex-btrblocks/public-api.lock | 72 +++++++++--------- vortex-btrblocks/src/schemes/decimal.rs | 4 +- vortex-btrblocks/src/schemes/float.rs | 20 ++--- vortex-btrblocks/src/schemes/integer.rs | 42 +++++----- vortex-btrblocks/src/schemes/string.rs | 16 ++-- vortex-btrblocks/src/schemes/temporal.rs | 4 +- vortex-compressor/public-api.lock | 72 +++++++++--------- .../src/builtins/constant/bool.rs | 4 +- .../src/builtins/constant/float.rs | 4 +- .../src/builtins/constant/integer.rs | 4 +- .../src/builtins/constant/string.rs | 4 +- vortex-compressor/src/builtins/dict/float.rs | 7 +- .../src/builtins/dict/integer.rs | 7 +- vortex-compressor/src/builtins/dict/string.rs | 4 +- vortex-compressor/src/compressor.rs | 76 +++++++++---------- vortex-compressor/src/estimate.rs | 6 +- vortex-compressor/src/scheme.rs | 4 +- vortex-compressor/src/stats/cache.rs | 63 ++++++++------- vortex-tensor/public-api.lock | 8 +- vortex-tensor/src/encodings/l2_denorm.rs | 4 +- .../src/encodings/turboquant/scheme.rs | 4 +- 21 files changed, 215 insertions(+), 214 deletions(-) diff --git a/vortex-btrblocks/public-api.lock b/vortex-btrblocks/public-api.lock index 725307fcbf2..64ae058adc7 100644 --- a/vortex-btrblocks/public-api.lock +++ b/vortex-btrblocks/public-api.lock @@ -56,9 +56,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::decimal::D impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::decimal::DecimalScheme -pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::decimal::DecimalScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -98,9 +98,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::ALP impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::ALPRDScheme -pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::ALPRDScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::ALPRDScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::ALPRDScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::ALPRDScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -128,9 +128,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::ALP impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::ALPScheme -pub fn vortex_btrblocks::schemes::float::ALPScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::ALPScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::ALPScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::ALPScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::ALPScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -162,11 +162,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::Flo pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::FloatRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -196,11 +196,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::Nul impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::NullDominatedSparseScheme -pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::NullDominatedSparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -230,9 +230,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::float::Pco impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::float::PcoScheme -pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::float::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::float::PcoScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::float::PcoScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::float::PcoScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -270,9 +270,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::B impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::BitPackingScheme -pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::BitPackingScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -302,9 +302,9 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::F pub fn vortex_btrblocks::schemes::integer::FoRScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::FoRScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -334,11 +334,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::I pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::IntRLEScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -368,9 +368,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::P impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::PcoScheme -pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::PcoScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::PcoScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::PcoScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::PcoScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -400,11 +400,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::R pub fn vortex_btrblocks::schemes::integer::RunEndScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::RunEndScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::RunEndScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::RunEndScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::RunEndScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::RunEndScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::RunEndScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -436,9 +436,9 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::S pub fn vortex_btrblocks::schemes::integer::SequenceScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::SequenceScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::integer::SequenceScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::SequenceScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::SequenceScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -466,11 +466,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::S impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::SparseScheme -pub fn vortex_btrblocks::schemes::integer::SparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::SparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::SparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::SparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::SparseScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::SparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -504,11 +504,11 @@ impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::Z pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -548,9 +548,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::FS impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::FSSTScheme -pub fn vortex_btrblocks::schemes::string::FSSTScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::FSSTScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::string::FSSTScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::FSSTScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::FSSTScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -580,11 +580,11 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::Nu impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::NullDominatedSparseScheme -pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::NullDominatedSparseScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -614,9 +614,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::string::Zs impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::string::ZstdScheme -pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::string::ZstdScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::string::ZstdScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::string::ZstdScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::string::ZstdScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -646,9 +646,9 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::temporal:: impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::temporal::TemporalScheme -pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_btrblocks::schemes::temporal::TemporalScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool diff --git a/vortex-btrblocks/src/schemes/decimal.rs b/vortex-btrblocks/src/schemes/decimal.rs index 47d18f80860..d60e5c6b175 100644 --- a/vortex-btrblocks/src/schemes/decimal.rs +++ b/vortex-btrblocks/src/schemes/decimal.rs @@ -45,7 +45,7 @@ impl Scheme for DecimalScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -56,7 +56,7 @@ impl Scheme for DecimalScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index be3bf69df75..8c55b38d345 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -83,7 +83,7 @@ impl Scheme for ALPScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -104,7 +104,7 @@ impl Scheme for ALPScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -156,7 +156,7 @@ impl Scheme for ALPRDScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -171,7 +171,7 @@ impl Scheme for ALPRDScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -229,7 +229,7 @@ impl Scheme for NullDominatedSparseScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -254,7 +254,7 @@ impl Scheme for NullDominatedSparseScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -301,7 +301,7 @@ impl Scheme for PcoScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -311,7 +311,7 @@ impl Scheme for PcoScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -349,7 +349,7 @@ impl Scheme for FloatRLEScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -368,7 +368,7 @@ impl Scheme for FloatRLEScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 69dee5be17c..7e6c3503239 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -129,7 +129,7 @@ impl Scheme for FoRScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -182,7 +182,7 @@ impl Scheme for FoRScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -198,10 +198,9 @@ impl Scheme for FoRScheme { // NOTE: we could delegate in the future if we had another downstream codec that performs // as well. let leaf_ctx = compress_ctx.clone().as_leaf(); - let mut biased_data = + let biased_data = ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options()); - let compressed = - BitPackingScheme.compress(compressor, &mut biased_data, leaf_ctx, exec_ctx)?; + let compressed = BitPackingScheme.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?; // TODO(connor): This should really be `new_unchecked`. let for_compressed = FoR::try_new(compressed, for_array.reference_scalar().clone())?; @@ -268,7 +267,7 @@ impl Scheme for ZigZagScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -290,7 +289,7 @@ impl Scheme for ZigZagScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -321,7 +320,7 @@ impl Scheme for BitPackingScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -338,7 +337,7 @@ impl Scheme for BitPackingScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -450,7 +449,7 @@ impl Scheme for SparseScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -496,13 +495,12 @@ impl Scheme for SparseScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { let len = data.array_len(); - // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats(exec_ctx).clone(); + let stats = data.integer_stats(exec_ctx); let array = data.array(); let (most_frequent_value, most_frequent_count) = stats @@ -635,7 +633,7 @@ impl Scheme for RunEndScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -650,7 +648,7 @@ impl Scheme for RunEndScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -714,7 +712,7 @@ impl Scheme for SequenceScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -760,7 +758,7 @@ impl Scheme for SequenceScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -786,7 +784,7 @@ impl Scheme for PcoScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -803,7 +801,7 @@ impl Scheme for PcoScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -821,7 +819,7 @@ impl Scheme for PcoScheme { pub(crate) fn rle_compress( scheme: &dyn Scheme, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -953,7 +951,7 @@ impl Scheme for IntRLEScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -971,7 +969,7 @@ impl Scheme for IntRLEScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index 037d8979d59..0df5a268157 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -73,7 +73,7 @@ impl Scheme for FSSTScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -83,7 +83,7 @@ impl Scheme for FSSTScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -167,7 +167,7 @@ impl Scheme for NullDominatedSparseScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -192,7 +192,7 @@ impl Scheme for NullDominatedSparseScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -240,7 +240,7 @@ impl Scheme for ZstdScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -250,7 +250,7 @@ impl Scheme for ZstdScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -274,7 +274,7 @@ impl Scheme for ZstdBuffersScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -284,7 +284,7 @@ impl Scheme for ZstdBuffersScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 6405baeff40..73671da0ff3 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -61,7 +61,7 @@ impl Scheme for TemporalScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -72,7 +72,7 @@ impl Scheme for TemporalScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 116cfba31f2..05cbad5ad6f 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -26,11 +26,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::BoolCons pub fn vortex_compressor::builtins::BoolConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::BoolConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::BoolConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -64,11 +64,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatCon pub fn vortex_compressor::builtins::FloatConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -102,11 +102,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatDic pub fn vortex_compressor::builtins::FloatDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -140,11 +140,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntConst pub fn vortex_compressor::builtins::IntConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -178,11 +178,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntDictS pub fn vortex_compressor::builtins::IntDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -216,11 +216,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringCo pub fn vortex_compressor::builtins::StringConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -254,11 +254,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringDi pub fn vortex_compressor::builtins::StringDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -338,7 +338,7 @@ impl core::fmt::Debug for vortex_compressor::estimate::EstimateVerdict pub fn vortex_compressor::estimate::EstimateVerdict::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &mut vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) +pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) pub mod vortex_compressor::scheme @@ -428,11 +428,11 @@ pub trait vortex_compressor::scheme::Scheme: core::fmt::Debug + core::marker::Se pub fn vortex_compressor::scheme::Scheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::scheme::Scheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::scheme::Scheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::scheme::Scheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::scheme::Scheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::scheme::Scheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::scheme::Scheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -446,11 +446,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::BoolCons pub fn vortex_compressor::builtins::BoolConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::BoolConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::BoolConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::BoolConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::BoolConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -464,11 +464,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatCon pub fn vortex_compressor::builtins::FloatConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -482,11 +482,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::FloatDic pub fn vortex_compressor::builtins::FloatDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::FloatDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::FloatDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::FloatDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::FloatDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -500,11 +500,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntConst pub fn vortex_compressor::builtins::IntConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -518,11 +518,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::IntDictS pub fn vortex_compressor::builtins::IntDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::IntDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::IntDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::IntDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::IntDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -536,11 +536,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringCo pub fn vortex_compressor::builtins::StringConstantScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringConstantScheme::compress(&self, _compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringConstantScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringConstantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringConstantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -554,11 +554,11 @@ impl vortex_compressor::scheme::Scheme for vortex_compressor::builtins::StringDi pub fn vortex_compressor::builtins::StringDictScheme::ancestor_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &mut vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_compressor::builtins::StringDictScheme::compress(&self, compressor: &vortex_compressor::CascadingCompressor, data: &vortex_compressor::stats::ArrayAndStats, compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_compressor::builtins::StringDictScheme::descendant_exclusions(&self) -> alloc::vec::Vec -pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_compressor::builtins::StringDictScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_compressor::builtins::StringDictScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -690,19 +690,19 @@ pub fn vortex_compressor::stats::ArrayAndStats::array_as_utf8(&self) -> vortex_a pub fn vortex_compressor::stats::ArrayAndStats::array_len(&self) -> usize -pub fn vortex_compressor::stats::ArrayAndStats::bool_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::BoolStats +pub fn vortex_compressor::stats::ArrayAndStats::bool_stats(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> alloc::sync::Arc -pub fn vortex_compressor::stats::ArrayAndStats::float_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::FloatStats +pub fn vortex_compressor::stats::ArrayAndStats::float_stats(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> alloc::sync::Arc -pub fn vortex_compressor::stats::ArrayAndStats::get_or_insert_with(&mut self, f: impl core::ops::function::FnOnce() -> T) -> &T +pub fn vortex_compressor::stats::ArrayAndStats::get_or_insert_with(&self, f: impl core::ops::function::FnOnce() -> T) -> alloc::sync::Arc -pub fn vortex_compressor::stats::ArrayAndStats::integer_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::IntegerStats +pub fn vortex_compressor::stats::ArrayAndStats::integer_stats(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> alloc::sync::Arc pub fn vortex_compressor::stats::ArrayAndStats::into_array(self) -> vortex_array::array::erased::ArrayRef pub fn vortex_compressor::stats::ArrayAndStats::new(array: vortex_array::array::erased::ArrayRef, opts: vortex_compressor::stats::GenerateStatsOptions) -> Self -pub fn vortex_compressor::stats::ArrayAndStats::string_stats(&mut self, ctx: &mut vortex_array::executor::ExecutionCtx) -> &vortex_compressor::stats::StringStats +pub fn vortex_compressor::stats::ArrayAndStats::string_stats(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> alloc::sync::Arc pub struct vortex_compressor::stats::BoolStats diff --git a/vortex-compressor/src/builtins/constant/bool.rs b/vortex-compressor/src/builtins/constant/bool.rs index e99edfbf315..3716e85e9ea 100644 --- a/vortex-compressor/src/builtins/constant/bool.rs +++ b/vortex-compressor/src/builtins/constant/bool.rs @@ -28,7 +28,7 @@ impl Scheme for BoolConstantScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -57,7 +57,7 @@ impl Scheme for BoolConstantScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/src/builtins/constant/float.rs b/vortex-compressor/src/builtins/constant/float.rs index 70e8525855a..1445a6cca2d 100644 --- a/vortex-compressor/src/builtins/constant/float.rs +++ b/vortex-compressor/src/builtins/constant/float.rs @@ -31,7 +31,7 @@ impl Scheme for FloatConstantScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -79,7 +79,7 @@ impl Scheme for FloatConstantScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/src/builtins/constant/integer.rs b/vortex-compressor/src/builtins/constant/integer.rs index 298c9de7e84..a10481ad100 100644 --- a/vortex-compressor/src/builtins/constant/integer.rs +++ b/vortex-compressor/src/builtins/constant/integer.rs @@ -29,7 +29,7 @@ impl Scheme for IntConstantScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -69,7 +69,7 @@ impl Scheme for IntConstantScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/src/builtins/constant/string.rs b/vortex-compressor/src/builtins/constant/string.rs index e723336fc3c..68ccfe14e8a 100644 --- a/vortex-compressor/src/builtins/constant/string.rs +++ b/vortex-compressor/src/builtins/constant/string.rs @@ -31,7 +31,7 @@ impl Scheme for StringConstantScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -73,7 +73,7 @@ impl Scheme for StringConstantScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index 45b3097a0b0..51d553b591f 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -83,7 +83,7 @@ impl Scheme for FloatDictScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -109,12 +109,11 @@ impl Scheme for FloatDictScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.float_stats(exec_ctx).clone(); + let stats = data.float_stats(exec_ctx); let dict = dictionary_encode(data.array_as_primitive(), &stats)?; let has_all_values_referenced = dict.has_all_values_referenced(); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 8f83d2840cb..140afdcebf1 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -57,7 +57,7 @@ impl Scheme for IntDictScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -103,12 +103,11 @@ impl Scheme for IntDictScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { - // TODO(connor): Fight the borrow checker (needs interior mutability)! - let stats = data.integer_stats(exec_ctx).clone(); + let stats = data.integer_stats(exec_ctx); let dict = dictionary_encode(data.array_as_primitive(), &stats)?; // Values = child 0. diff --git a/vortex-compressor/src/builtins/dict/string.rs b/vortex-compressor/src/builtins/dict/string.rs index 935bf6c0312..ac6affdf854 100644 --- a/vortex-compressor/src/builtins/dict/string.rs +++ b/vortex-compressor/src/builtins/dict/string.rs @@ -68,7 +68,7 @@ impl Scheme for StringDictScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -94,7 +94,7 @@ impl Scheme for StringDictScheme { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index da8d5ad56fa..0f67619e3cb 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -303,10 +303,10 @@ impl CascadingCompressor { }); let compress_ctx = compress_ctx.with_merged_stats_options(merged_opts); - let mut data = ArrayAndStats::new(array, merged_opts); + let data = ArrayAndStats::new(array, merged_opts); let Some((winner, winner_estimate)) = - self.choose_best_scheme(&eligible_schemes, &mut data, compress_ctx.clone(), exec_ctx)? + self.choose_best_scheme(&eligible_schemes, &data, compress_ctx.clone(), exec_ctx)? else { return Ok(data.into_array()); }; @@ -314,7 +314,7 @@ impl CascadingCompressor { // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the // scheme name and cascade history before propagating. let error_ctx = trace::enabled_error_context(&compress_ctx); - let compressed = match winner.compress(self, &mut data, compress_ctx, exec_ctx) { + let compressed = match winner.compress(self, &data, compress_ctx, exec_ctx) { Ok(compressed) => compressed, Err(err) => { // NB: this is the only way we can tell which scheme panicked / bailed on their @@ -355,7 +355,7 @@ impl CascadingCompressor { fn choose_best_scheme( &self, schemes: &[&'static dyn Scheme], - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { @@ -688,7 +688,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -698,7 +698,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -720,7 +720,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -730,7 +730,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -752,7 +752,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -764,7 +764,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -786,7 +786,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -798,7 +798,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -820,7 +820,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -832,7 +832,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -854,7 +854,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -864,7 +864,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -886,7 +886,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -896,7 +896,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -918,7 +918,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -928,7 +928,7 @@ mod tests { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -950,7 +950,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -960,7 +960,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -982,7 +982,7 @@ mod tests { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -992,7 +992,7 @@ mod tests { fn compress( &self, _compressor: &CascadingCompressor, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -1054,12 +1054,12 @@ mod tests { let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &ImmediateAlwaysUseScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &ImmediateAlwaysUseScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1077,12 +1077,12 @@ mod tests { let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackAlwaysUseScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackAlwaysUseScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1099,12 +1099,12 @@ mod tests { fn callback_skip_is_ignored() -> VortexResult<()> { let compressor = CascadingCompressor::new(vec![&CallbackSkipScheme, &DirectRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&CallbackSkipScheme, &DirectRatioScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1121,12 +1121,12 @@ mod tests { fn callback_ratio_competes_numerically() -> VortexResult<()> { let compressor = CascadingCompressor::new(vec![&DirectRatioScheme, &CallbackRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &CallbackRatioScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1143,12 +1143,12 @@ mod tests { fn zero_byte_sample_loses_to_finite_ratio() -> VortexResult<()> { let compressor = CascadingCompressor::new(vec![&HugeRatioScheme, &ZeroBytesSamplingScheme]); let schemes: [&'static dyn Scheme; 2] = [&HugeRatioScheme, &ZeroBytesSamplingScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1165,12 +1165,12 @@ mod tests { fn finite_ratio_displaces_zero_byte_sample() -> VortexResult<()> { let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme, &HugeRatioScheme]); let schemes: [&'static dyn Scheme; 2] = [&ZeroBytesSamplingScheme, &HugeRatioScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; @@ -1187,12 +1187,12 @@ mod tests { fn zero_byte_sample_alone_selects_no_scheme() -> VortexResult<()> { let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); let schemes: [&'static dyn Scheme; 1] = [&ZeroBytesSamplingScheme]; - let mut data = estimate_test_data(); + let data = estimate_test_data(); let mut exec_ctx = SESSION.create_execution_ctx(); let winner = compressor.choose_best_scheme( &schemes, - &mut data, + &data, CompressorContext::new(), &mut exec_ctx, )?; diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 4fed4cae323..22a99fbccdc 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -28,7 +28,7 @@ use crate::trace; #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, - &mut ArrayAndStats, + &ArrayAndStats, CompressorContext, &mut ExecutionCtx, ) -> VortexResult @@ -193,11 +193,11 @@ pub(super) fn estimate_compression_ratio_with_sampling( canonical.into_array() }; - let mut sample_data = ArrayAndStats::new(sample_array, scheme.stats_options()); + let sample_data = ArrayAndStats::new(sample_array, scheme.stats_options()); let error_ctx = trace::enabled_error_context(&compress_ctx); let sample_ctx = compress_ctx.with_sampling(); - let compressed = match scheme.compress(compressor, &mut sample_data, sample_ctx, exec_ctx) { + let compressed = match scheme.compress(compressor, &sample_data, sample_ctx, exec_ctx) { Ok(compressed) => compressed, Err(err) => { trace::sample_compress_failed(scheme.id(), error_ctx.as_ref(), &err); diff --git a/vortex-compressor/src/scheme.rs b/vortex-compressor/src/scheme.rs index 24582b1bc2e..4c5e7f0e940 100644 --- a/vortex-compressor/src/scheme.rs +++ b/vortex-compressor/src/scheme.rs @@ -211,7 +211,7 @@ pub trait Scheme: Debug + Send + Sync { /// that are not all-null. fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate; @@ -224,7 +224,7 @@ pub trait Scheme: Debug + Send + Sync { fn compress( &self, compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult; diff --git a/vortex-compressor/src/stats/cache.rs b/vortex-compressor/src/stats/cache.rs index 5f53cb83d41..91f0bf711fb 100644 --- a/vortex-compressor/src/stats/cache.rs +++ b/vortex-compressor/src/stats/cache.rs @@ -5,7 +5,9 @@ use std::any::Any; use std::any::TypeId; +use std::sync::Arc; +use parking_lot::Mutex; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -20,42 +22,45 @@ use super::GenerateStatsOptions; use super::IntegerStats; use super::StringStats; +/// A single cache entry: a concrete [`TypeId`] paired with a type-erased value. +type StatsEntry = (TypeId, Arc); + /// Cache for compression statistics, keyed by concrete type. +/// +/// The cache is interior-mutable: entries can be inserted through a shared [`&StatsCache`] +/// borrow. Values are stored as [`Arc`] so that cached entries can be +/// cloned out of the lock cheaply and handed back to callers as [`Arc`]. struct StatsCache { // TODO(connor): We could further optimize this with a `SmallVec` here. /// The cache entries, keyed by [`TypeId`]. /// /// The total number of statistics types in this stats should be relatively small, so we use a /// vector instead of a hash map. - entries: Vec<(TypeId, Box)>, + entries: Arc>>, } impl StatsCache { /// Creates a new empty cache. fn new() -> Self { Self { - entries: Vec::new(), + entries: Arc::new(Mutex::new(Vec::new())), } } /// Returns a cached value, computing it on first access. - fn get_or_insert_with(&mut self, f: impl FnOnce() -> T) -> &T { + fn get_or_insert_with(&self, f: impl FnOnce() -> T) -> Arc { let type_id = TypeId::of::(); - let pos = self.entries.iter().position(|(id, _)| *id == type_id); + let mut guard = self.entries.lock(); - if let Some(pos) = pos { - self.entries[pos] - .1 - .downcast_ref::() + if let Some(pos) = guard.iter().position(|(id, _)| *id == type_id) { + Arc::clone(&guard[pos].1) + .downcast::() + .ok() .vortex_expect("we just checked the TypeID") } else { - self.entries.push((type_id, Box::new(f()))); - self.entries - .last() - .vortex_expect("just pushed") - .1 - .downcast_ref::() - .vortex_expect("we just checked the TypeID") + let new_arc: Arc = Arc::new(f()); + guard.push((type_id, Arc::clone(&new_arc) as Arc)); + new_arc } } } @@ -66,10 +71,16 @@ impl StatsCache { /// FoR bias subtraction), it must create a new [`ArrayAndStats`] so that stale stats from the /// original array are not reused. /// -/// Built-in stats are accessed via typed methods (`integer_stats`, `float_stats`, `string_stats`) -/// which generate stats lazily on first access using the stored [`GenerateStatsOptions`]. +/// Built-in stats are accessed via typed methods ([`integer_stats`], [`float_stats`], +/// [`string_stats`]) which generate stats lazily on first access using the stored +/// [`GenerateStatsOptions`]. /// -/// Extension schemes can use `get_or_insert_with` for custom stats types. +/// Extension schemes can use [`get_or_insert_with`] for custom stats types. +/// +/// [`integer_stats`]: ArrayAndStats::integer_stats +/// [`float_stats`]: ArrayAndStats::float_stats +/// [`string_stats`]: ArrayAndStats::string_stats +/// [`get_or_insert_with`]: ArrayAndStats::get_or_insert_with pub struct ArrayAndStats { /// The array. This is always in canonical form. array: ArrayRef, @@ -138,9 +149,8 @@ impl ArrayAndStats { } /// Returns bool stats, generating them lazily on first access. - pub fn bool_stats(&mut self, ctx: &mut ExecutionCtx) -> &BoolStats { + pub fn bool_stats(&self, ctx: &mut ExecutionCtx) -> Arc { let array = self.array.clone(); - self.cache.get_or_insert_with::(|| { let bool_array = array .as_opt::() @@ -150,13 +160,10 @@ impl ArrayAndStats { }) } - // TODO(connor): These should all have interior mutability instead!!! - /// Returns integer stats, generating them lazily on first access. - pub fn integer_stats(&mut self, ctx: &mut ExecutionCtx) -> &IntegerStats { + pub fn integer_stats(&self, ctx: &mut ExecutionCtx) -> Arc { let array = self.array.clone(); let opts = self.opts; - self.cache.get_or_insert_with::(|| { let primitive = array .as_opt::() @@ -167,10 +174,9 @@ impl ArrayAndStats { } /// Returns float stats, generating them lazily on first access. - pub fn float_stats(&mut self, ctx: &mut ExecutionCtx) -> &FloatStats { + pub fn float_stats(&self, ctx: &mut ExecutionCtx) -> Arc { let array = self.array.clone(); let opts = self.opts; - self.cache.get_or_insert_with::(|| { let primitive = array .as_opt::() @@ -181,10 +187,9 @@ impl ArrayAndStats { } /// Returns string stats, generating them lazily on first access. - pub fn string_stats(&mut self, ctx: &mut ExecutionCtx) -> &StringStats { + pub fn string_stats(&self, ctx: &mut ExecutionCtx) -> Arc { let array = self.array.clone(); let opts = self.opts; - self.cache.get_or_insert_with::(|| { let varbinview = array .as_opt::() @@ -195,7 +200,7 @@ impl ArrayAndStats { } /// For extension schemes with custom stats types. - pub fn get_or_insert_with(&mut self, f: impl FnOnce() -> T) -> &T { + pub fn get_or_insert_with(&self, f: impl FnOnce() -> T) -> Arc { self.cache.get_or_insert_with::(f) } } diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 49c21c193e9..153fab43aa9 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -12,9 +12,9 @@ pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::fmt(&self, f: &mut c impl vortex_compressor::scheme::Scheme for vortex_tensor::encodings::l2_denorm::L2DenormScheme -pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::expected_compression_ratio(&self, _data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::expected_compression_ratio(&self, _data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_tensor::encodings::l2_denorm::L2DenormScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool @@ -64,9 +64,9 @@ impl core::marker::StructuralPartialEq for vortex_tensor::encodings::turboquant: impl vortex_compressor::scheme::Scheme for vortex_tensor::encodings::turboquant::TurboQuantScheme -pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::compress(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::expected_compression_ratio(&self, data: &mut vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate +pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::expected_compression_ratio(&self, data: &vortex_compressor::stats::cache::ArrayAndStats, _compress_ctx: vortex_compressor::ctx::CompressorContext, _exec_ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_compressor::estimate::CompressionEstimate pub fn vortex_tensor::encodings::turboquant::TurboQuantScheme::matches(&self, canonical: &vortex_array::canonical::Canonical) -> bool diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index b21595db003..61a72d6a4d2 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -33,7 +33,7 @@ impl Scheme for L2DenormScheme { fn expected_compression_ratio( &self, - _data: &mut ArrayAndStats, + _data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -43,7 +43,7 @@ impl Scheme for L2DenormScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { diff --git a/vortex-tensor/src/encodings/turboquant/scheme.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs index c2ea8f91e1f..19007664abe 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -71,7 +71,7 @@ impl Scheme for TurboQuantScheme { fn expected_compression_ratio( &self, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { @@ -97,7 +97,7 @@ impl Scheme for TurboQuantScheme { fn compress( &self, _compressor: &CascadingCompressor, - data: &mut ArrayAndStats, + data: &ArrayAndStats, _compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, ) -> VortexResult { From 223d1dfad58cf995a233b662e96ddf04f80f90d0 Mon Sep 17 00:00:00 2001 From: Dmitrii Blaginin Date: Tue, 21 Apr 2026 18:47:46 +0100 Subject: [PATCH 153/250] ci: always run all lints (#7584) Zen Browser 2026-04-21 13 20 20 always run all checks and display their results right in the CI box Signed-off-by: blaginin Co-authored-by: Claude --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1067a62e1ce..c6b146d35b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,15 +214,51 @@ jobs: - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt - name: Rust Lint - Format + id: fmt + continue-on-error: true run: cargo +$NIGHTLY_TOOLCHAIN fmt --all --check - name: Rustc check + id: check + continue-on-error: true run: RUSTFLAGS="-D warnings" cargo check --profile ci --locked --all-features --all-targets - name: Rustc check (release) + id: check-release + continue-on-error: true run: RUSTFLAGS="-D warnings" cargo check --locked --all-features --all-targets --release - name: Rust Lint - Clippy All Features + id: clippy-all + continue-on-error: true run: cargo clippy --profile ci --locked --all-features --all-targets -- -D warnings - name: Rust Lint - Clippy Default Features + id: clippy-default + continue-on-error: true run: cargo clippy --profile ci --locked --all-targets -- -D warnings + - name: Check lint results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + const failed = Object.entries({ + fmt: '${{ steps.fmt.outcome }}', + check: '${{ steps.check.outcome }}', + 'check-release': '${{ steps.check-release.outcome }}', + 'clippy-all': '${{ steps.clippy-all.outcome }}', + 'clippy-default': '${{ steps.clippy-default.outcome }}', + }).filter(([, o]) => o === 'failure').map(([n]) => n); + if (!failed.length) return; + + const { data: { jobs } } = await github.rest.actions.listJobsForWorkflowRun({ + ...context.repo, run_id: context.runId, + }); + const url = jobs.find(j => j.name === context.job)?.check_run_url; + if (url) { + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(url.split('/').pop()), + output: { title: `${failed.join(', ')} — failing`, summary: '' }, + }); + } + core.setFailed(`Lint failed: ${failed.join(', ')}`); cpp-lint: name: "C/C++ (lint)" From 26a766f2754fefa3f8c18e7c6c90afb604643f49 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 21 Apr 2026 16:16:42 -0400 Subject: [PATCH 154/250] Allow Claude to write (#7294) * Split Claude automation into two workflows with separate trust boundaries: claude-write.yml for write-capable issue automation and follow-up edits on Claude-owned PRs, and claude-review.yml for read-only PR review. * Sets up a GitHub app (vortex-claude) so PRs pushed by Claude can run CI * Only claude-write can access the GitHub App private key, and only through the claude-automation environment. No other workflow gets that key. * claude-write uses a short-lived GitHub App installation token, not GITHUB_TOKEN, so PRs opened by Claude trigger normal pull_request CI. * claude-write runs only for trusted collaborators with write/maintain/admin access. * claude-write is issue-driven by default. It may also run from a PR conversation comment, but only when that PR is a same-repo PR originally opened by the Vortex Claude GitHub App (CLAUDE_APP_LOGIN). * claude-write refuses fork PR content entirely. There is no promotion path for forks. * claude-review has no GitHub App private key and no write-capable repo token. It uses the built-in GITHUB_TOKEN only for reading repo state and posting review comments. * claude-review can review same-repo PRs, but it must not create commits, push branches, open PRs, or update PR branches. * PR conversation comments on Claude-owned PRs are routed to claude-write; claude-review explicitly stands down on that path to avoid duplicate runs. * Both workflows reject bot-triggered invocations and gate access before running Claude logic. * Both workflows use persist-credentials: false on checkout so git operations do not silently fall back to the built-in GITHUB_TOKEN. * Fork PRs are always refused for Claude automation because fork content is treated as untrusted prompt input. --------- Signed-off-by: Nicholas Gates --- .github/workflows/claude-review.yml | 259 ++++++++++++++++++++++++++++ .github/workflows/claude-write.yml | 213 +++++++++++++++++++++++ .github/workflows/claude.yml | 66 ------- 3 files changed, 472 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/claude-review.yml create mode 100644 .github/workflows/claude-write.yml delete mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml new file mode 100644 index 00000000000..a8323f82b51 --- /dev/null +++ b/.github/workflows/claude-review.yml @@ -0,0 +1,259 @@ +name: Claude Code (Review) + +# Setup +# - This workflow does not need the GitHub App private key. +# - Do not attach the `claude-automation` environment here. +# - Store `CLAUDE_CODE_OAUTH_TOKEN` as a repository or organization Actions secret. +# - Create a repository or organization Actions variable: +# - CLAUDE_APP_LOGIN +# Set this to the bot login for the GitHub App, usually `[bot]`. +# - It may use the default GITHUB_TOKEN to read repository data and post review +# comments, but it must never be able to push commits or open branches. +# +# Why this workflow exists separately +# - PR review traffic is a different trust boundary from issue automation. +# - This workflow is intentionally read-only with respect to repository contents. +# - Fork PRs are refused outright. We do not "promote" or manually bless fork +# content into Claude. If a contributor wants Claude to implement something, +# a maintainer should restate the task on an issue and use claude-write.yml. +# - PR conversation comments on PRs already opened by the Claude App are handled by +# claude-write.yml so maintainers can ask Claude to make follow-up changes there. + +concurrency: + # `issue_comment` events on PRs expose the PR number via `github.event.issue.number`, + # so this falls back there when `github.event.pull_request.number` is unset. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.run_id }} + cancel-in-progress: true + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + pull_request_review: + types: [submitted] + +jobs: + gate: + name: Gate PR Trigger + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + outputs: + should_run: ${{ steps.gate.outputs.should_run }} + reason: ${{ steps.gate.outputs.reason }} + pull_number: ${{ steps.gate.outputs.pull_number }} + checkout_ref: ${{ steps.gate.outputs.checkout_ref }} + actor_has_write: ${{ steps.gate.outputs.actor_has_write }} + steps: + - name: Check whether this PR event is allowed to reach Claude + id: gate + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + const sender = context.payload.sender?.login ?? ''; + const senderType = context.payload.sender?.type ?? ''; + const trustedClaudeLogin = process.env.CLAUDE_APP_LOGIN ?? ''; + + async function getPermission(username) { + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username, + }); + return data.permission; + } catch (error) { + if (error.status === 404) { + return 'none'; + } + throw error; + } + } + + let mentioned = false; + let pullNumber = null; + + if (context.eventName === 'issue_comment') { + mentioned = (context.payload.comment?.body ?? '').includes('@claude'); + if (context.payload.issue?.pull_request) { + pullNumber = context.payload.issue.number; + } + } else if (context.eventName === 'pull_request_review_comment') { + mentioned = (context.payload.comment?.body ?? '').includes('@claude'); + pullNumber = context.payload.pull_request?.number ?? null; + } else if (context.eventName === 'pull_request_review') { + mentioned = (context.payload.review?.body ?? '').includes('@claude'); + pullNumber = context.payload.pull_request?.number ?? null; + } + + if (pullNumber) { + core.setOutput('pull_number', String(pullNumber)); + } + + let reason = ''; + + if (!mentioned) { + reason = 'not_mentioned'; + } else if (!pullNumber) { + reason = 'not_a_pr_event'; + } else if (senderType === 'Bot') { + reason = 'bot_sender_refused'; + } + + let actorHasWrite = 'false'; + if (!reason) { + const permission = await getPermission(sender); + core.setOutput('actor_permission', permission); + actorHasWrite = ['admin', 'maintain', 'write'].includes(permission) ? 'true' : 'false'; + if (actorHasWrite !== 'true') { + reason = 'actor_lacks_write'; + } + } + + if (!reason && context.eventName === 'issue_comment' && !trustedClaudeLogin) { + reason = 'missing_claude_app_login'; + } + + if (!reason) { + let pr = null; + const response = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + }); + pr = response.data; + + const headRepo = pr.head.repo; + const isFork = !headRepo || headRepo.fork || headRepo.full_name !== `${context.repo.owner}/${context.repo.repo}`; + core.setOutput('checkout_ref', pr.head.sha); + + if (isFork) { + reason = 'fork_pr_refused'; + } else if ( + context.eventName === 'issue_comment' && + trustedClaudeLogin && + (pr.user?.login ?? '') === trustedClaudeLogin + ) { + reason = 'claude_pr_uses_write_workflow'; + } + + if (!reason) { + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + per_page: 100, + }); + // Refuse PRs that modify `.github/` because workflow files define the + // automation policy this review job is enforcing. + if (files.some(f => f.filename.startsWith('.github/'))) { + reason = 'modifies_github_dir'; + } + } + } + + core.setOutput('actor_has_write', actorHasWrite); + core.setOutput('should_run', !reason ? 'true' : 'false'); + core.setOutput('reason', reason || 'allowed'); + env: + CLAUDE_APP_LOGIN: ${{ vars.CLAUDE_APP_LOGIN }} + + explain-refusal: + name: Explain Refusal + needs: gate + if: | + needs.gate.outputs.actor_has_write == 'true' && + ( + needs.gate.outputs.reason == 'fork_pr_refused' || + needs.gate.outputs.reason == 'modifies_github_dir' || + needs.gate.outputs.reason == 'missing_claude_app_login' + ) + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Comment with the refusal reason + uses: actions/github-script@v7 + with: + script: | + const reason = ${{ toJSON(needs.gate.outputs.reason) }}; + const messages = { + fork_pr_refused: [ + "Claude review automation is disabled for fork pull requests.", + "", + "Why:", + "- fork content is untrusted input", + "- this repository does not allow Claude to run against fork content", + "- there is no promotion path for forks", + "", + "If maintainers want Claude to implement a change, restate the task on an issue and use the issue-driven Claude workflow instead." + ].join("\n"), + modifies_github_dir: [ + "Claude review automation is disabled for pull requests that modify `.github/` files.", + "", + "Why:", + "- workflow and action files are part of the automation policy", + "- this review workflow refuses to evaluate automation changes from the same PR", + "", + "Ask a human reviewer to inspect workflow changes directly." + ].join("\n"), + missing_claude_app_login: [ + "Claude review automation is misconfigured for issue-comment triggers.", + "", + "Why:", + "- `CLAUDE_APP_LOGIN` is not set", + "- without that bot login, the review workflow cannot safely route comments on Claude-owned PRs to the write workflow", + "", + "Set the `CLAUDE_APP_LOGIN` Actions variable, then retry the command." + ].join("\n"), + }; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: Number(${{ needs.gate.outputs.pull_number }}), + body: messages[reason], + }); + + claude: + name: Run Claude PR Review + needs: gate + if: needs.gate.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + issues: write + pull-requests: write + actions: read + steps: + - name: Checkout same-repo PR contents + uses: actions/checkout@v6 + with: + ref: ${{ needs.gate.outputs.checkout_ref }} + fetch-depth: 1 + # Keep git credentials out of the workspace. This workflow is review-only + # and should never push changes. + persist-credentials: false + + - name: Run Claude Code in review-only mode + id: claude + uses: anthropics/claude-code-action@6cad158a175744eb2e76f7f5fd108ec63145598c + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # This workflow deliberately uses the built-in token because it only needs + # to read repository state and post review comments. It cannot write repo + # contents, and it has no access to the GitHub App private key. + github_token: ${{ github.token }} + + additional_permissions: | + actions: read + + claude_args: | + --allowedTools "Read,Grep,Glob,Bash(git diff:*),Bash(git show:*),Bash(git log:*),Bash(head:*),Bash(jq:*),Bash(rg:*)" + --system-prompt "You are the repository's read-only Claude review workflow. Review the current same-repo pull request and respond in GitHub. Never modify files, never create commits, never push branches, and never open or update pull requests. Fork pull requests are blocked before this job starts." diff --git a/.github/workflows/claude-write.yml b/.github/workflows/claude-write.yml new file mode 100644 index 00000000000..d4d4c6ad03f --- /dev/null +++ b/.github/workflows/claude-write.yml @@ -0,0 +1,213 @@ +name: Claude Code (Write) + +# Setup +# 1. Create a dedicated GitHub App and install it only on this repository. +# 2. Grant the App only: +# - Contents: Read & write +# - Issues: Read & write +# - Pull requests: Read & write +# 3. Create a GitHub Actions environment named `claude-automation`. +# 4. Store these environment secrets in `claude-automation`: +# - APP_ID +# - APP_PRIVATE_KEY +# - CLAUDE_CODE_OAUTH_TOKEN +# 5. Create a repository or organization Actions variable: +# - CLAUDE_APP_LOGIN +# Set this to the bot login for the GitHub App, usually `[bot]`. +# +# Why this workflow exists separately +# - This is the only workflow that can read the GitHub App private key. +# - It is primarily issue-driven. The only PR path it allows is a follow-up comment +# on a same-repo PR that was already opened by the Claude GitHub App. +# - Claude uses a short-lived GitHub App installation token, not GITHUB_TOKEN, so +# PRs opened by Claude trigger normal `pull_request` workflows. +# - A gate job runs before the environment is attached so untrusted events are +# rejected before the private key is exposed. +# - GitHub exposes the author of an app-created PR as the app's bot login rather +# than the numeric App ID in the PR payload, so the gate checks `CLAUDE_APP_LOGIN` +# instead of comparing directly to `APP_ID`. + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + +on: + issues: + types: [opened, assigned] + issue_comment: + types: [created] + +jobs: + gate: + name: Gate Issue Trigger + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + outputs: + should_run: ${{ steps.gate.outputs.should_run }} + reason: ${{ steps.gate.outputs.reason }} + checkout_ref: ${{ steps.gate.outputs.checkout_ref }} + steps: + - name: Check whether this event is allowed to reach Claude + id: gate + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + const sender = context.payload.sender?.login ?? ''; + const senderType = context.payload.sender?.type ?? ''; + const trustedClaudeLogin = process.env.CLAUDE_APP_LOGIN ?? ''; + + async function getPermission(username) { + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username, + }); + return data.permission; + } catch (error) { + if (error.status === 404) { + return 'none'; + } + throw error; + } + } + + let mentioned = false; + let reason = ''; + let checkoutRef = context.payload.repository?.default_branch ?? ''; + + if (context.eventName === 'issues') { + const title = context.payload.issue?.title ?? ''; + const body = context.payload.issue?.body ?? ''; + mentioned = title.includes('@claude') || body.includes('@claude'); + } else if (context.eventName === 'issue_comment') { + const body = context.payload.comment?.body ?? ''; + mentioned = body.includes('@claude'); + + if (context.payload.issue?.pull_request) { + const response = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number, + }); + const pr = response.data; + + checkoutRef = pr.head.sha; + + const headRepo = pr.head.repo; + const isFork = !headRepo || headRepo.fork || headRepo.full_name !== `${context.repo.owner}/${context.repo.repo}`; + if (isFork) { + reason = 'fork_pr_refused'; + } else if (!trustedClaudeLogin) { + reason = 'missing_claude_app_login'; + } else if ((pr.user?.login ?? '') !== trustedClaudeLogin) { + reason = 'pr_not_owned_by_claude_app'; + } + + if (!reason) { + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number, + per_page: 100, + }); + // Refuse workflow changes so write-capable automation never acts on + // `.github/` modifications from the same pull request. + if (files.some(f => f.filename.startsWith('.github/'))) { + reason = 'modifies_github_dir'; + } + } + } + } + + if (!reason && !mentioned) { + reason = 'not_mentioned'; + } + + if (!reason && senderType === 'Bot') { + reason = 'bot_sender_refused'; + } + + if (!reason) { + const permission = await getPermission(sender); + core.setOutput('actor_permission', permission); + if (!['admin', 'maintain', 'write'].includes(permission)) { + reason = 'actor_lacks_write'; + } + } + + core.setOutput('checkout_ref', checkoutRef); + core.setOutput('should_run', !reason ? 'true' : 'false'); + core.setOutput('reason', reason || 'allowed'); + env: + CLAUDE_APP_LOGIN: ${{ vars.CLAUDE_APP_LOGIN }} + + claude: + name: Run Claude Code + needs: gate + if: needs.gate.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + + environment: + # The App private key lives only in this environment so only this single job + # can mint a GitHub App installation token. + name: claude-automation + deployment: false + + permissions: + contents: read + issues: read + pull-requests: read + actions: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ needs.gate.outputs.checkout_ref }} + fetch-depth: 0 + # Do not leave the built-in GITHUB_TOKEN in git config. Claude should use + # only the GitHub App token generated below so its PRs trigger CI normally. + persist-credentials: false + + - name: Generate short-lived GitHub App token + id: app-token + uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-issues: write + permission-pull-requests: write + + - name: Setup Rust toolchain + uses: ./.github/actions/setup-rust + with: + enable-sccache: "false" + + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + with: + sync: false + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@6cad158a175744eb2e76f7f5fd108ec63145598c + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + github_token: ${{ steps.app-token.outputs.token }} + + # Claude may inspect CI state on related PRs, but this workflow is otherwise + # issue-driven and is the only place with write-capable GitHub credentials. + additional_permissions: | + actions: read + + claude_args: | + --allowedTools "Bash(cargo nextest:*),Bash(cargo check:*),Bash(cargo clippy:*),Bash(cargo fmt:*),Bash(uv run:*)" + --system-prompt "You are the repository's write-capable Claude workflow. You run only from trusted issue traffic and from trusted PR conversation comments on same-repo pull requests that were previously opened by the repository's Claude GitHub App. Create or update branches and pull requests using the provided GitHub App token. Do not use fork pull request content because those runs are blocked before this job starts." diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index 79553228a71..00000000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Claude Code - -concurrency: - # We shouldn't have multiple instances of Claude running on the same PR. - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - timeout-minutes: 120 - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 - with: - sync: false - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - - # This is an optional setting that allows Claude to read CI results on PRs - additional_permissions: | - actions: read - - # Optional: Customize the trigger phrase (default: @claude) - # trigger_phrase: "/claude" - - # Optional: Trigger when specific user is assigned to an issue - # assignee_trigger: "claude-bot" - - claude_args: | - --allowedTools "Bash(cargo nextest:*),Bash(cargo check:*),Bash(cargo clippy:*),Bash(cargo fmt:*),Bash(uv run:*)" - --system-prompt "You have been granted tools for editing files and running cargo commands (cargo nextest, cargo check, cargo clippy, cargo fmt) and uv for running pytest (e.g. via uv run --all-packages pytest)" - - # Optional: Custom environment variables for Claude - # claude_env: | - # NODE_ENV: test From 7324308da659b1436e10c846663b48063b892f80 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:35:24 -0400 Subject: [PATCH 155/250] Rename duplicated `ScalarFn` types (#7586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Renames: - `ScalarFn` -> `TypedScalarFn` - `struct ScalarFnVTable` -> `struct ScalarFn` - `trait ScalarFnVTable` -> `trait ScalarFnVTable` (not changed 🫠) ## Testing N/A --------- Signed-off-by: Connor Tsui --- encodings/datetime-parts/src/compute/rules.rs | 4 +- encodings/runend/src/rules.rs | 4 +- vortex-array/public-api.lock | 350 +++++++++--------- .../src/arrays/chunked/compute/rules.rs | 6 +- vortex-array/src/arrays/dict/compute/rules.rs | 6 +- .../src/arrays/extension/compute/rules.rs | 10 +- vortex-array/src/arrays/mod.rs | 2 +- vortex-array/src/arrays/scalar_fn/array.rs | 10 +- vortex-array/src/arrays/scalar_fn/plugin.rs | 4 +- vortex-array/src/arrays/scalar_fn/rules.rs | 25 +- .../src/arrays/scalar_fn/vtable/mod.rs | 22 +- .../src/arrays/scalar_fn/vtable/operations.rs | 16 +- .../src/arrays/scalar_fn/vtable/validity.rs | 13 +- vortex-array/src/arrow/executor/struct_.rs | 4 +- vortex-array/src/scalar_fn/erased.rs | 30 +- .../src/scalar_fn/fns/between/kernel.rs | 6 +- .../src/scalar_fn/fns/binary/compare.rs | 4 +- .../src/scalar_fn/fns/fill_null/kernel.rs | 6 +- vortex-array/src/scalar_fn/fns/like/kernel.rs | 6 +- .../src/scalar_fn/fns/list_contains/kernel.rs | 6 +- vortex-array/src/scalar_fn/fns/zip/kernel.rs | 6 +- vortex-array/src/scalar_fn/foreign.rs | 5 +- vortex-array/src/scalar_fn/mod.rs | 4 +- vortex-array/src/scalar_fn/plugin.rs | 4 +- vortex-array/src/scalar_fn/typed.rs | 10 +- vortex-array/src/scalar_fn/vtable.rs | 4 +- vortex-duckdb/src/datasource.rs | 4 +- vortex-tensor/public-api.lock | 10 +- .../src/encodings/turboquant/tests/mod.rs | 6 +- .../src/scalar_fns/cosine_similarity.rs | 8 +- vortex-tensor/src/scalar_fns/inner_product.rs | 10 +- vortex-tensor/src/scalar_fns/l2_denorm.rs | 12 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 10 +- .../src/scalar_fns/sorf_transform/mod.rs | 8 +- 34 files changed, 320 insertions(+), 315 deletions(-) diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index 533de399081..bc9f0205add 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -10,8 +10,8 @@ use vortex_array::arrays::Filter; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::filter::FilterReduceAdaptor; use vortex_array::arrays::scalar_fn::AnyScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -use vortex_array::arrays::scalar_fn::ScalarFnVTable; use vortex_array::arrays::slice::SliceReduceAdaptor; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; @@ -95,7 +95,7 @@ impl ArrayParentReduceRule for DTPComparisonPushDownRule { fn reduce_parent( &self, child: ArrayView<'_, DateTimeParts>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, child_idx: usize, ) -> VortexResult> { // Only handle comparison operations (Binary comparisons or Between) diff --git a/encodings/runend/src/rules.rs b/encodings/runend/src/rules.rs index 0abfc3fa54e..64c3287ae79 100644 --- a/encodings/runend/src/rules.rs +++ b/encodings/runend/src/rules.rs @@ -10,8 +10,8 @@ use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::scalar_fn::AnyScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; -use vortex_array::arrays::scalar_fn::ScalarFnVTable; use vortex_array::dtype::DType; use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; @@ -43,7 +43,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { fn reduce_parent( &self, run_end: ArrayView<'_, RunEnd>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, child_idx: usize, ) -> VortexResult> { for (idx, child) in parent.iter_children().enumerate() { diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 1f7b1320b5c..1e4a8a58ee5 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -3844,7 +3844,7 @@ pub fn vortex_array::arrays::scalar_fn::AnyScalarFn::fmt(&self, f: &mut core::fm impl vortex_array::matcher::Matcher for vortex_array::arrays::scalar_fn::AnyScalarFn -pub type vortex_array::arrays::scalar_fn::AnyScalarFn::Match<'a> = vortex_array::ArrayView<'a, vortex_array::arrays::scalar_fn::ScalarFnVTable> +pub type vortex_array::arrays::scalar_fn::AnyScalarFn::Match<'a> = vortex_array::ArrayView<'a, vortex_array::arrays::scalar_fn::ScalarFn> pub fn vortex_array::arrays::scalar_fn::AnyScalarFn::matches(array: &vortex_array::ArrayRef) -> bool @@ -3868,77 +3868,77 @@ pub fn vortex_array::arrays::scalar_fn::ExactScalarFn::matches(array: &vortex pub fn vortex_array::arrays::scalar_fn::ExactScalarFn::try_match(array: &vortex_array::ArrayRef) -> core::option::Option -pub struct vortex_array::arrays::scalar_fn::ScalarFnArrayView<'a, F: vortex_array::scalar_fn::ScalarFnVTable> +pub struct vortex_array::arrays::scalar_fn::ScalarFn -pub vortex_array::arrays::scalar_fn::ScalarFnArrayView::options: &'a ::Options +impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFn -pub vortex_array::arrays::scalar_fn::ScalarFnArrayView::vtable: &'a F +pub fn vortex_array::arrays::scalar_fn::ScalarFn::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFn -impl core::ops::deref::Deref for vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F> +impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::Target = vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::deref(&self) -> &Self::Target +impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::scalar_fn::ScalarFnArrayView<'a, F: vortex_array::scalar_fn::ScalarFnVTable> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub vortex_array::arrays::scalar_fn::ScalarFnArrayView::options: &'a ::Options -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub vortex_array::arrays::scalar_fn::ScalarFnArrayView::vtable: &'a F -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +impl core::ops::deref::Deref for vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F> -impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::Target = vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayView<'_, F>::deref(&self) -> &Self::Target -pub trait vortex_array::arrays::scalar_fn::ScalarFnArrayExt: vortex_array::TypedArrayRef +pub trait vortex_array::arrays::scalar_fn::ScalarFnArrayExt: vortex_array::TypedArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::child_at(&self, idx: usize) -> &vortex_array::ArrayRef @@ -3954,7 +3954,7 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::nchildren(&self) -> us pub fn vortex_array::arrays::scalar_fn::ScalarFnArrayExt::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef -impl> vortex_array::arrays::scalar_fn::ScalarFnArrayExt for T +impl> vortex_array::arrays::scalar_fn::ScalarFnArrayExt for T pub fn T::child_at(&self, idx: usize) -> &vortex_array::ArrayRef @@ -3986,7 +3986,7 @@ pub fn V::try_new_array(&self, len: usize, options: Self::Options, children: imp pub fn V::try_new_array(&self, len: usize, options: Self::Options, children: impl core::convert::Into>) -> vortex_error::VortexResult -pub type vortex_array::arrays::scalar_fn::ScalarFnArray = vortex_array::Array +pub type vortex_array::arrays::scalar_fn::ScalarFnArray = vortex_array::Array pub mod vortex_array::arrays::shared @@ -6234,63 +6234,63 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::ScalarFnVTable +pub struct vortex_array::arrays::ScalarFn -impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnVTable +pub fn vortex_array::arrays::scalar_fn::ScalarFn::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFn -impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::arrays::scalar_fn::ScalarFn::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>) -> vortex_error::VortexResult pub struct vortex_array::arrays::Shared @@ -6740,7 +6740,7 @@ pub type vortex_array::arrays::PatchedArray = vortex_array::Array -pub type vortex_array::arrays::ScalarFnArray = vortex_array::Array +pub type vortex_array::arrays::ScalarFnArray = vortex_array::Array pub type vortex_array::arrays::SharedArray = vortex_array::Array @@ -13100,7 +13100,7 @@ pub fn vortex_array::AnyColumnar::try_match<'a>(array: &'a vortex_array::ArrayRe impl vortex_array::matcher::Matcher for vortex_array::arrays::scalar_fn::AnyScalarFn -pub type vortex_array::arrays::scalar_fn::AnyScalarFn::Match<'a> = vortex_array::ArrayView<'a, vortex_array::arrays::scalar_fn::ScalarFnVTable> +pub type vortex_array::arrays::scalar_fn::AnyScalarFn::Match<'a> = vortex_array::ArrayView<'a, vortex_array::arrays::scalar_fn::ScalarFn> pub fn vortex_array::arrays::scalar_fn::AnyScalarFn::matches(array: &vortex_array::ArrayRef) -> bool @@ -17450,18 +17450,6 @@ pub fn vortex_array::scalar_fn::ForeignScalarFnVTable::stat_falsification(&self, pub fn vortex_array::scalar_fn::ForeignScalarFnVTable::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult> -pub struct vortex_array::scalar_fn::ScalarFn - -impl vortex_array::scalar_fn::ScalarFn - -pub fn vortex_array::scalar_fn::ScalarFn::erased(self) -> vortex_array::scalar_fn::ScalarFnRef - -pub fn vortex_array::scalar_fn::ScalarFn::new(vtable: V, options: ::Options) -> Self - -pub fn vortex_array::scalar_fn::ScalarFn::options(&self) -> &::Options - -pub fn vortex_array::scalar_fn::ScalarFn::vtable(&self) -> &V - pub struct vortex_array::scalar_fn::ScalarFnOptions<'a> impl vortex_array::scalar_fn::ScalarFnOptions<'_> @@ -17500,9 +17488,9 @@ pub fn vortex_array::scalar_fn::ScalarFnRef::as_opt vortex_error::VortexResult> -pub fn vortex_array::scalar_fn::ScalarFnRef::downcast(self) -> alloc::sync::Arc> +pub fn vortex_array::scalar_fn::ScalarFnRef::downcast(self) -> alloc::sync::Arc> -pub fn vortex_array::scalar_fn::ScalarFnRef::downcast_ref(&self) -> core::option::Option<&vortex_array::scalar_fn::ScalarFn> +pub fn vortex_array::scalar_fn::ScalarFnRef::downcast_ref(&self) -> core::option::Option<&vortex_array::scalar_fn::TypedScalarFnInstance> pub fn vortex_array::scalar_fn::ScalarFnRef::execute(&self, args: &dyn vortex_array::scalar_fn::ExecutionArgs, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -17518,7 +17506,7 @@ pub fn vortex_array::scalar_fn::ScalarFnRef::return_dtype(&self, arg_types: &[vo pub fn vortex_array::scalar_fn::ScalarFnRef::signature(&self) -> vortex_array::scalar_fn::ScalarFnSignature<'_> -pub fn vortex_array::scalar_fn::ScalarFnRef::try_downcast(self) -> core::result::Result>, vortex_array::scalar_fn::ScalarFnRef> +pub fn vortex_array::scalar_fn::ScalarFnRef::try_downcast(self) -> core::result::Result>, vortex_array::scalar_fn::ScalarFnRef> pub fn vortex_array::scalar_fn::ScalarFnRef::validity(&self, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult @@ -17556,6 +17544,18 @@ pub fn vortex_array::scalar_fn::ScalarFnSignature<'_>::is_fallible(&self) -> boo pub fn vortex_array::scalar_fn::ScalarFnSignature<'_>::is_null_sensitive(&self) -> bool +pub struct vortex_array::scalar_fn::TypedScalarFnInstance + +impl vortex_array::scalar_fn::TypedScalarFnInstance + +pub fn vortex_array::scalar_fn::TypedScalarFnInstance::erased(self) -> vortex_array::scalar_fn::ScalarFnRef + +pub fn vortex_array::scalar_fn::TypedScalarFnInstance::new(vtable: V, options: ::Options) -> Self + +pub fn vortex_array::scalar_fn::TypedScalarFnInstance::options(&self) -> &::Options + +pub fn vortex_array::scalar_fn::TypedScalarFnInstance::vtable(&self) -> &V + pub struct vortex_array::scalar_fn::VecExecutionArgs impl vortex_array::scalar_fn::VecExecutionArgs @@ -20062,45 +20062,45 @@ pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::A pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -20222,9 +20222,9 @@ impl vortex_array::OperationsVTable for pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice @@ -21034,45 +21034,45 @@ pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::A pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -21204,9 +21204,9 @@ impl vortex_array::ValidityVTable for vortex_a pub fn vortex_array::arrays::null::Null::validity(_array: vortex_array::ArrayView<'_, vortex_array::arrays::null::Null>) -> vortex_error::VortexResult -impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>) -> vortex_error::VortexResult impl vortex_array::ValidityVTable for vortex_array::arrays::slice::Slice @@ -21748,9 +21748,9 @@ impl vortex_array::Array pub fn vortex_array::Array::new(len: usize) -> Self -impl vortex_array::Array +impl vortex_array::Array -pub fn vortex_array::Array::try_new(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_new(scalar_fn: vortex_array::scalar_fn::ScalarFnRef, children: alloc::vec::Vec, len: usize) -> vortex_error::VortexResult impl vortex_array::Array @@ -23730,45 +23730,45 @@ pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::A pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -24070,9 +24070,9 @@ impl vortex_array::OperationsVTable for pub fn vortex_array::arrays::patched::Patched::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::patched::Patched>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::OperationsVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>, index: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::OperationsVTable for vortex_array::arrays::slice::Slice @@ -24950,45 +24950,45 @@ pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::A pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> -impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFn::ArrayData = vortex_array::arrays::scalar_fn::array::ScalarFnData -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable +pub type vortex_array::arrays::scalar_fn::ScalarFn::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFn::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFn::id(&self) -> vortex_array::ArrayId -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize +pub fn vortex_array::arrays::scalar_fn::ScalarFn::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::serialize(_array: vortex_array::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String +pub fn vortex_array::arrays::scalar_fn::ScalarFn::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validate(&self, data: &vortex_array::arrays::scalar_fn::array::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::slice::Slice @@ -25120,9 +25120,9 @@ impl vortex_array::ValidityVTable for vortex_a pub fn vortex_array::arrays::null::Null::validity(_array: vortex_array::ArrayView<'_, vortex_array::arrays::null::Null>) -> vortex_error::VortexResult -impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFnVTable +impl vortex_array::ValidityVTable for vortex_array::arrays::scalar_fn::ScalarFn -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFnVTable>) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFn::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::scalar_fn::ScalarFn>) -> vortex_error::VortexResult impl vortex_array::ValidityVTable for vortex_array::arrays::slice::Slice diff --git a/vortex-array/src/arrays/chunked/compute/rules.rs b/vortex-array/src/arrays/chunked/compute/rules.rs index 8ef63a188c3..d8d324a8e86 100644 --- a/vortex-array/src/arrays/chunked/compute/rules.rs +++ b/vortex-array/src/arrays/chunked/compute/rules.rs @@ -11,8 +11,8 @@ use crate::arrays::Chunked; use crate::arrays::ChunkedArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; +use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; -use crate::arrays::ScalarFnVTable; use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::scalar_fn::AnyScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; @@ -38,7 +38,7 @@ impl ArrayParentReduceRule for ChunkedUnaryScalarFnPushDownRule { fn reduce_parent( &self, array: ArrayView<'_, Chunked>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, _child_idx: usize, ) -> VortexResult> { if parent.nchildren() != 1 { @@ -73,7 +73,7 @@ impl ArrayParentReduceRule for ChunkedConstantScalarFnPushDownRule { fn reduce_parent( &self, array: ArrayView<'_, Chunked>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, child_idx: usize, ) -> VortexResult> { for (idx, child) in parent.iter_children().enumerate() { diff --git a/vortex-array/src/arrays/dict/compute/rules.rs b/vortex-array/src/arrays/dict/compute/rules.rs index 23fc05d08d6..f6fe816a6cc 100644 --- a/vortex-array/src/arrays/dict/compute/rules.rs +++ b/vortex-array/src/arrays/dict/compute/rules.rs @@ -13,8 +13,8 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; +use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; -use crate::arrays::ScalarFnVTable; use crate::arrays::dict::DictArraySlotsExt; use crate::arrays::filter::FilterReduceAdaptor; use crate::arrays::scalar_fn::AnyScalarFn; @@ -51,7 +51,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnValuesPushDownRule { fn reduce_parent( &self, array: ArrayView<'_, Dict>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, child_idx: usize, ) -> VortexResult> { // Check that the scalar function can actually be pushed down. @@ -156,7 +156,7 @@ impl ArrayParentReduceRule for DictionaryScalarFnCodesPullUpRule { fn reduce_parent( &self, array: ArrayView<'_, Dict>, - parent: ArrayView<'_, ScalarFnVTable>, + parent: ArrayView<'_, ScalarFn>, child_idx: usize, ) -> VortexResult> { // Don't attempt to pull up if there are less than 2 siblings. diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 568fe1c061c..b6e2d5a1e06 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -91,7 +91,7 @@ mod tests { use crate::arrays::ExtensionArray; use crate::arrays::FilterArray; use crate::arrays::PrimitiveArray; - use crate::arrays::ScalarFnVTable; + use crate::arrays::ScalarFn; use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnFactoryExt; @@ -221,7 +221,7 @@ mod tests { .unwrap(); let optimized = scalar_fn_array.optimize_recursive().unwrap(); - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); let children = scalar_fn.children(); let constant = children[0] .as_opt::() @@ -284,7 +284,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // The first child should still be an ExtensionArray (no pushdown happened) - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when ext types differ" @@ -309,7 +309,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // No pushdown should happen because sibling is not a constant - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when sibling is not constant" @@ -332,7 +332,7 @@ mod tests { let optimized = scalar_fn_array.optimize().unwrap(); // No pushdown should happen because constant is not an extension scalar - let scalar_fn = optimized.as_opt::().unwrap(); + let scalar_fn = optimized.as_opt::().unwrap(); assert!( scalar_fn.children()[0].as_opt::().is_some(), "Expected first child to remain ExtensionArray when constant is not extension" diff --git a/vortex-array/src/arrays/mod.rs b/vortex-array/src/arrays/mod.rs index 2597708919a..212f4bbe619 100644 --- a/vortex-array/src/arrays/mod.rs +++ b/vortex-array/src/arrays/mod.rs @@ -75,8 +75,8 @@ pub use primitive::Primitive; pub use primitive::PrimitiveArray; pub mod scalar_fn; +pub use scalar_fn::ScalarFn; pub use scalar_fn::ScalarFnArray; -pub use scalar_fn::ScalarFnVTable; pub mod shared; pub use shared::Shared; diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index d7d0118d02e..4a82951ebdb 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -12,7 +12,7 @@ use crate::ArrayRef; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::scalar_fn::ScalarFnRef; // ScalarFnArray has a variable number of slots (one per child) @@ -49,7 +49,7 @@ impl ScalarFnData { } } -pub trait ScalarFnArrayExt: TypedArrayRef { +pub trait ScalarFnArrayExt: TypedArrayRef { fn scalar_fn(&self) -> &ScalarFnRef { &self.scalar_fn } @@ -80,9 +80,9 @@ pub trait ScalarFnArrayExt: TypedArrayRef { self.iter_children().cloned().collect() } } -impl> ScalarFnArrayExt for T {} +impl> ScalarFnArrayExt for T {} -impl Array { +impl Array { /// Create a new ScalarFnArray from a scalar function and its children. pub fn try_new( scalar_fn: ScalarFnRef, @@ -92,7 +92,7 @@ impl Array { let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); let dtype = scalar_fn.return_dtype(&arg_dtypes)?; let data = ScalarFnData::build(scalar_fn.clone(), children.clone(), len)?; - let vtable = ScalarFnVTable { id: scalar_fn.id() }; + let vtable = ScalarFn { id: scalar_fn.id() }; Ok(unsafe { Array::from_parts_unchecked( ArrayParts::new(vtable, dtype, len, data) diff --git a/vortex-array/src/arrays/scalar_fn/plugin.rs b/vortex-array/src/arrays/scalar_fn/plugin.rs index 1fb656323d9..c685503a370 100644 --- a/vortex-array/src/arrays/scalar_fn/plugin.rs +++ b/vortex-array/src/arrays/scalar_fn/plugin.rs @@ -13,8 +13,8 @@ use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayView; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::scalar_fn::ScalarFn; use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::TypedScalarFnInstance; use crate::serde::ArrayChildren; /// An adapter for enabling a scalar function to be serialized as an array. @@ -83,7 +83,7 @@ impl ArrayPlugin for ScalarFnArrayPlugi &self.0, dtype, len, metadata, children, session, )?; Ok(ScalarFnArray::try_new( - ScalarFn::new(self.0.clone(), parts.options).erased(), + TypedScalarFnInstance::new(self.0.clone(), parts.options).erased(), parts.children, len, )? diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 30b46914b8e..1e9563cf9de 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -14,8 +14,8 @@ use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Filter; +use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; -use crate::arrays::ScalarFnVTable; use crate::arrays::Slice; use crate::arrays::StructArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; @@ -31,10 +31,10 @@ use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::fns::pack::Pack; use crate::validity::Validity; -pub(super) const RULES: ReduceRuleSet = +pub(super) const RULES: ReduceRuleSet = ReduceRuleSet::new(&[&ScalarFnPackToStructRule, &ScalarFnAbstractReduceRule]); -pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ +pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&ScalarFnUnaryFilterPushDownRule), ParentRuleSet::lift(&ScalarFnSliceReduceRule), ]); @@ -42,8 +42,8 @@ pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::ne /// Converts a ScalarFnArray with Pack into a StructArray directly. #[derive(Debug)] struct ScalarFnPackToStructRule; -impl ArrayReduceRule for ScalarFnPackToStructRule { - fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { +impl ArrayReduceRule for ScalarFnPackToStructRule { + fn reduce(&self, array: ArrayView<'_, ScalarFn>) -> VortexResult> { let Some(pack_options) = array.scalar_fn().as_opt::() else { return Ok(None); }; @@ -67,12 +67,12 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { #[derive(Debug)] struct ScalarFnSliceReduceRule; -impl ArrayParentReduceRule for ScalarFnSliceReduceRule { +impl ArrayParentReduceRule for ScalarFnSliceReduceRule { type Parent = Slice; fn reduce_parent( &self, - array: ArrayView<'_, ScalarFnVTable>, + array: ArrayView<'_, ScalarFn>, parent: ArrayView<'_, Slice>, _child_idx: usize, ) -> VortexResult> { @@ -91,8 +91,8 @@ impl ArrayParentReduceRule for ScalarFnSliceReduceRule { #[derive(Debug)] struct ScalarFnAbstractReduceRule; -impl ArrayReduceRule for ScalarFnAbstractReduceRule { - fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { +impl ArrayReduceRule for ScalarFnAbstractReduceRule { + fn reduce(&self, array: ArrayView<'_, ScalarFn>) -> VortexResult> { if let Some(reduced) = array .scalar_fn() .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len() })? @@ -119,8 +119,7 @@ impl ReduceNode for ArrayRef { } fn scalar_fn(&self) -> Option<&ScalarFnRef> { - self.as_opt::() - .map(|a| a.data().scalar_fn()) + self.as_opt::().map(|a| a.data().scalar_fn()) } fn child(&self, idx: usize) -> ReduceNodeRef { @@ -164,12 +163,12 @@ impl ReduceCtx for ArrayReduceCtx { #[derive(Debug)] struct ScalarFnUnaryFilterPushDownRule; -impl ArrayParentReduceRule for ScalarFnUnaryFilterPushDownRule { +impl ArrayParentReduceRule for ScalarFnUnaryFilterPushDownRule { type Parent = Filter; fn reduce_parent( &self, - child: ArrayView<'_, ScalarFnVTable>, + child: ArrayView<'_, ScalarFn>, parent: ArrayView<'_, Filter>, _child_idx: usize, ) -> VortexResult> { diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 3f70088966b..34c472eac07 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -45,11 +45,11 @@ use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::VecExecutionArgs; use crate::serde::ArrayChildren; -/// A [`ScalarFnVTable`]-encoded Vortex array. -pub type ScalarFnArray = Array; +/// A [`ScalarFn`]-encoded Vortex array. +pub type ScalarFnArray = Array; #[derive(Clone, Debug)] -pub struct ScalarFnVTable { +pub struct ScalarFn { pub(super) id: ScalarFnId, } @@ -65,7 +65,7 @@ impl ArrayEq for ScalarFnData { } } -impl VTable for ScalarFnVTable { +impl VTable for ScalarFn { type ArrayData = ScalarFnData; type OperationsVTable = Self; type ValidityVTable = Self; @@ -173,7 +173,7 @@ pub trait ScalarFnFactoryExt: scalar_fn::ScalarFnVTable { options: Self::Options, children: impl Into>, ) -> VortexResult { - let scalar_fn = scalar_fn::ScalarFn::new(self.clone(), options).erased(); + let scalar_fn = scalar_fn::TypedScalarFnInstance::new(self.clone(), options).erased(); let children = children.into(); vortex_ensure!( @@ -187,7 +187,7 @@ pub trait ScalarFnFactoryExt: scalar_fn::ScalarFnVTable { let data = ScalarFnData { scalar_fn: scalar_fn.clone(), }; - let vtable = ScalarFnVTable { id: scalar_fn.id() }; + let vtable = ScalarFn { id: scalar_fn.id() }; Ok(unsafe { Array::from_parts_unchecked( ArrayParts::new(vtable, dtype, len, data) @@ -203,14 +203,14 @@ impl ScalarFnFactoryExt for V {} #[derive(Debug)] pub struct AnyScalarFn; impl Matcher for AnyScalarFn { - type Match<'a> = ArrayView<'a, ScalarFnVTable>; + type Match<'a> = ArrayView<'a, ScalarFn>; fn matches(array: &ArrayRef) -> bool { - array.is::() + array.is::() } fn try_match(array: &ArrayRef) -> Option> { - array.as_opt::() + array.as_opt::() } } @@ -222,7 +222,7 @@ impl Matcher for ExactScalarFn { type Match<'a> = ScalarFnArrayView<'a, F>; fn matches(array: &ArrayRef) -> bool { - if let Some(scalar_fn_array) = array.as_opt::() { + if let Some(scalar_fn_array) = array.as_opt::() { scalar_fn_array.data().scalar_fn().is::() } else { false @@ -230,7 +230,7 @@ impl Matcher for ExactScalarFn { } fn try_match(array: &ArrayRef) -> Option> { - let scalar_fn_array = array.as_opt::()?; + let scalar_fn_array = array.as_opt::()?; let scalar_fn_data = scalar_fn_array.data(); let scalar_fn = scalar_fn_data.scalar_fn().downcast_ref::()?; Some(ScalarFnArrayView { diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index 224a8e2f4c2..1f710af3cdf 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -9,14 +9,14 @@ use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::arrays::ConstantArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::scalar_fn::vtable::ScalarFnVTable; +use crate::arrays::scalar_fn::vtable::ScalarFn; use crate::columnar::Columnar; use crate::scalar::Scalar; use crate::scalar_fn::VecExecutionArgs; -impl OperationsVTable for ScalarFnVTable { +impl OperationsVTable for ScalarFn { fn scalar_at( - array: ArrayView<'_, ScalarFnVTable>, + array: ArrayView<'_, ScalarFn>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -66,7 +66,7 @@ mod tests { use crate::arrays::PrimitiveArray; use crate::arrays::ScalarFnArray; use crate::assert_arrays_eq; - use crate::scalar_fn::ScalarFn; + use crate::scalar_fn::TypedScalarFnInstance; use crate::scalar_fn::fns::binary::Binary; use crate::scalar_fn::fns::operators::Operator; use crate::validity::Validity; @@ -76,7 +76,7 @@ mod tests { let lhs = buffer![1i32, 2, 3].into_array(); let rhs = buffer![10i32, 20, 30].into_array(); - let scalar_fn = ScalarFn::new(Binary, Operator::Add).erased(); + let scalar_fn = TypedScalarFnInstance::new(Binary, Operator::Add).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; let result = scalar_fn_array @@ -94,7 +94,7 @@ mod tests { let lhs = buffer![2i32, 3, 4].into_array(); let rhs = buffer![5i32, 6, 7].into_array(); - let scalar_fn = ScalarFn::new(Binary, Operator::Mul).erased(); + let scalar_fn = TypedScalarFnInstance::new(Binary, Operator::Mul).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; let result = scalar_fn_array @@ -116,7 +116,7 @@ mod tests { ) .into_array(); - let scalar_fn = ScalarFn::new(Binary, Operator::Add).erased(); + let scalar_fn = TypedScalarFnInstance::new(Binary, Operator::Add).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; let result = scalar_fn_array @@ -138,7 +138,7 @@ mod tests { let lhs = buffer![1i32, 5, 3].into_array(); let rhs = buffer![2i32, 5, 1].into_array(); - let scalar_fn = ScalarFn::new(Binary, Operator::Eq).erased(); + let scalar_fn = TypedScalarFnInstance::new(Binary, Operator::Eq).erased(); let scalar_fn_array = ScalarFnArray::try_new(scalar_fn, vec![lhs, rhs], 3)?; let result = scalar_fn_array diff --git a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs index 2884530f8ba..2ac376155e3 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/validity.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/validity.rs @@ -12,10 +12,10 @@ use crate::array::ValidityVTable; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::vtable::ArrayExpr; use crate::arrays::scalar_fn::vtable::FakeEq; -use crate::arrays::scalar_fn::vtable::ScalarFnVTable; +use crate::arrays::scalar_fn::vtable::ScalarFn; use crate::expr::Expression; use crate::expr::lit; -use crate::scalar_fn::ScalarFn; +use crate::scalar_fn::TypedScalarFnInstance; use crate::scalar_fn::VecExecutionArgs; use crate::scalar_fn::fns::literal::Literal; use crate::scalar_fn::fns::root::Root; @@ -50,15 +50,18 @@ fn execute_expr(expr: &Expression, row_count: usize) -> VortexResult { Ok(expr.scalar_fn().execute(&args, &mut ctx)?.into_array()) } -impl ValidityVTable for ScalarFnVTable { - fn validity(array: ArrayView<'_, ScalarFnVTable>) -> VortexResult { +impl ValidityVTable for ScalarFn { + fn validity(array: ArrayView<'_, ScalarFn>) -> VortexResult { let inputs: Vec<_> = array .iter_children() .map(|child| { if let Some(scalar) = child.as_constant() { return Ok(lit(scalar)); } - Expression::try_new(ScalarFn::new(ArrayExpr, FakeEq(child.clone())).erased(), []) + Expression::try_new( + TypedScalarFnInstance::new(ArrayExpr, FakeEq(child.clone())).erased(), + [], + ) }) .collect::>()?; diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 88b62abd4c9..909fd78059d 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -16,7 +16,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::Chunked; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::Struct; use crate::arrays::StructArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; @@ -69,7 +69,7 @@ pub(super) fn to_arrow_struct( }; // We can also short-circuit if the array is a `pack` scalar function: - if let Some(array) = array.as_opt::() + if let Some(array) = array.as_opt::() && let Some(_pack_options) = array.scalar_fn().as_opt::() { let DType::Struct(struct_fields, _) = array.dtype() else { diff --git a/vortex-array/src/scalar_fn/erased.rs b/vortex-array/src/scalar_fn/erased.rs index 369627964e2..10e82d25455 100644 --- a/vortex-array/src/scalar_fn/erased.rs +++ b/vortex-array/src/scalar_fn/erased.rs @@ -35,15 +35,15 @@ use crate::scalar_fn::fns::is_not_null::IsNotNull; use crate::scalar_fn::options::ScalarFnOptions; use crate::scalar_fn::signature::ScalarFnSignature; use crate::scalar_fn::typed::DynScalarFn; -use crate::scalar_fn::typed::ScalarFn; +use crate::scalar_fn::typed::TypedScalarFnInstance; /// A type-erased scalar function, pairing a vtable with bound options behind a trait object. /// /// This stores a [`ScalarFnVTable`] and its options behind an `Arc`, allowing /// heterogeneous storage inside [`Expression`] and [`crate::arrays::ScalarFnArray`]. /// -/// Use [`super::ScalarFn::new()`] to construct, and [`super::ScalarFn::erased()`] to obtain a -/// [`ScalarFnRef`]. +/// Use [`super::TypedScalarFnInstance::new()`] to construct, and [`super::TypedScalarFnInstance::erased()`] to +/// obtain a [`ScalarFnRef`]. #[derive(Clone)] pub struct ScalarFnRef(pub(super) Arc); @@ -55,14 +55,14 @@ impl ScalarFnRef { /// Returns whether the scalar function is of the given vtable type. pub fn is(&self) -> bool { - self.0.as_any().is::>() + self.0.as_any().is::>() } /// Returns the typed options for this scalar function if it matches the given vtable type. pub fn as_opt(&self) -> Option<&V::Options> { self.0 .as_any() - .downcast_ref::>() + .downcast_ref::>() .map(|sf| sf.options()) } @@ -76,24 +76,26 @@ impl ScalarFnRef { .vortex_expect("Expression options type mismatch") } - /// Downcast to the concrete [`ScalarFn`]. + /// Downcast to the concrete [`TypedScalarFnInstance`]. /// /// Returns `Err(self)` if the downcast fails. - pub fn try_downcast(self) -> Result>, ScalarFnRef> { - if self.0.as_any().is::>() { - let ptr = Arc::into_raw(self.0) as *const ScalarFn; + pub fn try_downcast( + self, + ) -> Result>, ScalarFnRef> { + if self.0.as_any().is::>() { + let ptr = Arc::into_raw(self.0) as *const TypedScalarFnInstance; Ok(unsafe { Arc::from_raw(ptr) }) } else { Err(self) } } - /// Downcast to the concrete [`ScalarFn`]. + /// Downcast to the concrete [`TypedScalarFnInstance`]. /// /// # Panics /// /// Panics if the downcast fails. - pub fn downcast(self) -> Arc> { + pub fn downcast(self) -> Arc> { self.try_downcast::() .map_err(|this| { vortex_err!( @@ -105,9 +107,9 @@ impl ScalarFnRef { .vortex_expect("Failed to downcast ScalarFnRef") } - /// Try to downcast into a typed [`ScalarFn`]. - pub fn downcast_ref(&self) -> Option<&ScalarFn> { - self.0.as_any().downcast_ref::>() + /// Try to downcast into a typed [`TypedScalarFnInstance`]. + pub fn downcast_ref(&self) -> Option<&TypedScalarFnInstance> { + self.0.as_any().downcast_ref::>() } /// The type-erased options for this scalar function. diff --git a/vortex-array/src/scalar_fn/fns/between/kernel.rs b/vortex-array/src/scalar_fn/fns/between/kernel.rs index 3bf543940c4..ee4fb688982 100644 --- a/vortex-array/src/scalar_fn/fns/between/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/between/kernel.rs @@ -11,7 +11,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -64,7 +64,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let children = scalar_fn_array.children(); let lower = &children[1]; @@ -99,7 +99,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let children = scalar_fn_array.children(); let lower = &children[1]; diff --git a/vortex-array/src/scalar_fn/fns/binary/compare.rs b/vortex-array/src/scalar_fn/fns/binary/compare.rs index 4c2a25258b0..546501fbb5e 100644 --- a/vortex-array/src/scalar_fn/fns/binary/compare.rs +++ b/vortex-array/src/scalar_fn/fns/binary/compare.rs @@ -19,7 +19,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::Constant; use crate::arrays::ConstantArray; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -74,7 +74,7 @@ where }; // Get the ScalarFnArray to access children - let Some(scalar_fn_array) = parent.as_opt::() else { + let Some(scalar_fn_array) = parent.as_opt::() else { return Ok(None); }; // Normalize so `array` is always LHS, swapping the operator if needed diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index f5e8df294dd..eea3dd6ef7b 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -12,7 +12,7 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::Constant; use crate::arrays::ConstantArray; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -123,7 +123,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let fill_value = scalar_fn_array .get_child(1) @@ -159,7 +159,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let fill_value = scalar_fn_array .get_child(1) diff --git a/vortex-array/src/scalar_fn/fns/like/kernel.rs b/vortex-array/src/scalar_fn/fns/like/kernel.rs index 0489504cc93..b3b683212ff 100644 --- a/vortex-array/src/scalar_fn/fns/like/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/like/kernel.rs @@ -8,7 +8,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -69,7 +69,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let pattern = scalar_fn_array.get_child(1); let options = *parent.options; @@ -98,7 +98,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let pattern = scalar_fn_array.get_child(1); let options = *parent.options; diff --git a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs index 7bc643606a7..563600bfeee 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/kernel.rs @@ -8,7 +8,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -67,7 +67,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let list = scalar_fn_array.get_child(0); ::list_contains(list, array) @@ -96,7 +96,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let list = scalar_fn_array.get_child(0); ::list_contains(list, array, ctx) diff --git a/vortex-array/src/scalar_fn/fns/zip/kernel.rs b/vortex-array/src/scalar_fn/fns/zip/kernel.rs index 5cfefee70d7..575c2f4c55a 100644 --- a/vortex-array/src/scalar_fn/fns/zip/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/zip/kernel.rs @@ -8,7 +8,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ScalarFnVTable; +use crate::arrays::ScalarFn; use crate::arrays::scalar_fn::ExactScalarFn; use crate::arrays::scalar_fn::ScalarFnArrayExt; use crate::arrays::scalar_fn::ScalarFnArrayView; @@ -68,7 +68,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let if_false = scalar_fn_array.get_child(1); let mask_array = scalar_fn_array.get_child(2); @@ -97,7 +97,7 @@ where return Ok(None); } let scalar_fn_array = parent - .as_opt::() + .as_opt::() .vortex_expect("ExactScalarFn matcher confirmed ScalarFnArray"); let if_false = scalar_fn_array.get_child(1); let mask_array = scalar_fn_array.get_child(2); diff --git a/vortex-array/src/scalar_fn/foreign.rs b/vortex-array/src/scalar_fn/foreign.rs index 5cf896c129d..d2abd3b9a95 100644 --- a/vortex-array/src/scalar_fn/foreign.rs +++ b/vortex-array/src/scalar_fn/foreign.rs @@ -17,10 +17,10 @@ use crate::expr::Expression; use crate::scalar_fn::Arity; use crate::scalar_fn::ChildName; use crate::scalar_fn::ExecutionArgs; -use crate::scalar_fn::ScalarFn; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::TypedScalarFnInstance; /// Options payload for a foreign scalar function. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -58,7 +58,8 @@ impl ForeignScalarFnVTable { } pub fn make_scalar_fn(id: ScalarFnId, metadata: Vec, arity: usize) -> ScalarFnRef { - ScalarFn::new(Self::new(id), ForeignScalarFnOptions::new(metadata, arity)).erased() + TypedScalarFnInstance::new(Self::new(id), ForeignScalarFnOptions::new(metadata, arity)) + .erased() } } diff --git a/vortex-array/src/scalar_fn/mod.rs b/vortex-array/src/scalar_fn/mod.rs index ce00fc08358..6f8ba1b6544 100644 --- a/vortex-array/src/scalar_fn/mod.rs +++ b/vortex-array/src/scalar_fn/mod.rs @@ -39,11 +39,11 @@ pub type ScalarFnId = Id; /// Private module to seal [`typed::DynScalarFn`]. mod sealed { use crate::scalar_fn::ScalarFnVTable; - use crate::scalar_fn::typed::ScalarFn; + use crate::scalar_fn::typed::TypedScalarFnInstance; /// Marker trait to prevent external implementations of [`super::typed::DynScalarFn`]. pub(crate) trait Sealed {} /// This can be the **only** implementor for [`super::typed::DynScalarFn`]. - impl Sealed for ScalarFn {} + impl Sealed for TypedScalarFnInstance {} } diff --git a/vortex-array/src/scalar_fn/plugin.rs b/vortex-array/src/scalar_fn/plugin.rs index bafaf2b6137..ef06bc3be0e 100644 --- a/vortex-array/src/scalar_fn/plugin.rs +++ b/vortex-array/src/scalar_fn/plugin.rs @@ -7,10 +7,10 @@ use std::sync::Arc; use vortex_error::VortexResult; use vortex_session::VortexSession; -use crate::scalar_fn::ScalarFn; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTable; +use crate::scalar_fn::TypedScalarFnInstance; /// Reference-counted pointer to a scalar function plugin. pub type ScalarFnPluginRef = Arc; @@ -43,6 +43,6 @@ impl ScalarFnPlugin for V { fn deserialize(&self, metadata: &[u8], session: &VortexSession) -> VortexResult { let options = ScalarFnVTable::deserialize(self, metadata, session)?; - Ok(ScalarFn::new(self.clone(), options).erased()) + Ok(TypedScalarFnInstance::new(self.clone(), options).erased()) } } diff --git a/vortex-array/src/scalar_fn/typed.rs b/vortex-array/src/scalar_fn/typed.rs index 88072ff4002..a2ef9549bff 100644 --- a/vortex-array/src/scalar_fn/typed.rs +++ b/vortex-array/src/scalar_fn/typed.rs @@ -40,14 +40,14 @@ use crate::scalar_fn::SimplifyCtx; /// You can construct one via [`new()`], and erase the type with [`erased()`] to obtain a /// [`ScalarFnRef`]. /// -/// [`new()`]: ScalarFn::new -/// [`erased()`]: ScalarFn::erased -pub struct ScalarFn { +/// [`new()`]: TypedScalarFnInstance::new +/// [`erased()`]: TypedScalarFnInstance::erased +pub struct TypedScalarFnInstance { vtable: V, options: V::Options, } -impl ScalarFn { +impl TypedScalarFnInstance { /// Create a new typed scalar function instance. pub fn new(vtable: V, options: V::Options) -> Self { Self { vtable, options } @@ -121,7 +121,7 @@ pub(super) trait DynScalarFn: 'static + Send + Sync + super::sealed::Sealed { fn options_debug(&self, f: &mut Formatter<'_>) -> fmt::Result; } -impl DynScalarFn for ScalarFn { +impl DynScalarFn for TypedScalarFnInstance { #[inline(always)] fn as_any(&self) -> &dyn Any { self diff --git a/vortex-array/src/scalar_fn/vtable.rs b/vortex-array/src/scalar_fn/vtable.rs index c9588e8aa13..7c4f65cedff 100644 --- a/vortex-array/src/scalar_fn/vtable.rs +++ b/vortex-array/src/scalar_fn/vtable.rs @@ -22,9 +22,9 @@ use crate::dtype::DType; use crate::expr::Expression; use crate::expr::StatsCatalog; use crate::expr::stats::Stat; -use crate::scalar_fn::ScalarFn; use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnRef; +use crate::scalar_fn::TypedScalarFnInstance; /// This trait defines the interface for scalar function vtables, including methods for /// serialization, deserialization, validation, child naming, return type computation, @@ -388,7 +388,7 @@ impl Display for EmptyOptions { pub trait ScalarFnVTableExt: ScalarFnVTable { /// Bind this vtable with the given options into a [`ScalarFnRef`]. fn bind(&self, options: Self::Options) -> ScalarFnRef { - ScalarFn::new(self.clone(), options).erased() + TypedScalarFnInstance::new(self.clone(), options).erased() } /// Create a new expression with this vtable and the given options and children. diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index 40ed1315e0b..ed416b088a2 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -21,7 +21,7 @@ use tracing::debug; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::VortexSessionExecute; -use vortex::array::arrays::ScalarFnVTable; +use vortex::array::arrays::ScalarFn; use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; use vortex::array::arrays::scalar_fn::ScalarFnArrayExt; @@ -420,7 +420,7 @@ impl TableFunction for T { let array_result: StructArray = if let Some(array) = array_result.as_opt::() { array.into_owned() - } else if let Some(array) = array_result.as_opt::() + } else if let Some(array) = array_result.as_opt::() && let Some(pack_options) = array.scalar_fn().as_opt::() { StructArray::new( diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 153fab43aa9..2f0080f18e8 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -258,7 +258,7 @@ pub struct vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity impl vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity -pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::new() -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::new() -> vortex_array::scalar_fn::typed::TypedScalarFnInstance pub fn vortex_tensor::scalar_fns::cosine_similarity::CosineSimilarity::try_new_array(lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult @@ -300,7 +300,7 @@ pub struct vortex_tensor::scalar_fns::inner_product::InnerProduct impl vortex_tensor::scalar_fns::inner_product::InnerProduct -pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::new() -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::new() -> vortex_array::scalar_fn::typed::TypedScalarFnInstance pub fn vortex_tensor::scalar_fns::inner_product::InnerProduct::try_new_array(lhs: vortex_array::array::erased::ArrayRef, rhs: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult @@ -342,7 +342,7 @@ pub struct vortex_tensor::scalar_fns::l2_denorm::L2Denorm impl vortex_tensor::scalar_fns::l2_denorm::L2Denorm -pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new() -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new() -> vortex_array::scalar_fn::typed::TypedScalarFnInstance pub unsafe fn vortex_tensor::scalar_fns::l2_denorm::L2Denorm::new_array_unchecked(normalized: vortex_array::array::erased::ArrayRef, norms: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult @@ -390,7 +390,7 @@ pub struct vortex_tensor::scalar_fns::l2_norm::L2Norm impl vortex_tensor::scalar_fns::l2_norm::L2Norm -pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::new() -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::new() -> vortex_array::scalar_fn::typed::TypedScalarFnInstance pub fn vortex_tensor::scalar_fns::l2_norm::L2Norm::try_new_array(child: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult @@ -478,7 +478,7 @@ pub struct vortex_tensor::scalar_fns::sorf_transform::SorfTransform impl vortex_tensor::scalar_fns::sorf_transform::SorfTransform -pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::new(options: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions) -> vortex_array::scalar_fn::typed::ScalarFn +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::new(options: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions) -> vortex_array::scalar_fn::typed::TypedScalarFnInstance pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::try_new_array(options: &vortex_tensor::scalar_fns::sorf_transform::SorfOptions, child: vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult diff --git a/vortex-tensor/src/encodings/turboquant/tests/mod.rs b/vortex-tensor/src/encodings/turboquant/tests/mod.rs index 7dfe53c444a..ec4182dcc3d 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/mod.rs @@ -19,7 +19,7 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::ScalarFnVTable; +use vortex_array::arrays::ScalarFn; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -74,7 +74,7 @@ fn make_vector_ext(fsl: &FixedSizeListArray) -> ArrayRef { /// Unwrap an L2Denorm ScalarFnArray into (sorf_child, norms_child). fn unwrap_l2denorm(encoded: &ArrayRef) -> (ArrayRef, ArrayRef) { let sfn = encoded - .as_opt::() + .as_opt::() .expect("expected ScalarFnArray (L2Denorm)"); (sfn.child_at(0).clone(), sfn.child_at(1).clone()) } @@ -86,7 +86,7 @@ fn unwrap_codes_centroids_norms( ) -> VortexResult<(PrimitiveArray, PrimitiveArray, PrimitiveArray)> { let (sorf_child, norms_child) = unwrap_l2denorm(encoded); let padded_vector_child = sorf_child - .as_opt::() + .as_opt::() .expect("expected SorfTransform ScalarFnArray") .child_at(0) .clone(); diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index 9f1cd39fa1a..f325bf2a205 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -24,9 +24,9 @@ use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; -use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::TypedScalarFnInstance; use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; use vortex_buffer::Buffer; @@ -62,9 +62,9 @@ use crate::utils::validate_binary_tensor_float_inputs; pub struct CosineSimilarity; impl CosineSimilarity { - /// Creates a new [`ScalarFn`] wrapping the cosine similarity operation. - pub fn new() -> ScalarFn { - ScalarFn::new(CosineSimilarity, EmptyOptions) + /// Creates a new [`TypedScalarFnInstance`] wrapping the cosine similarity operation. + pub fn new() -> TypedScalarFnInstance { + TypedScalarFnInstance::new(CosineSimilarity, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the cosine similarity between `lhs` and diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index f001c74498e..7bff5165edb 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -17,8 +17,8 @@ use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeList; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn as ScalarFnArrayEncoding; use vortex_array::arrays::ScalarFnArray; -use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -39,9 +39,9 @@ use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; -use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::TypedScalarFnInstance; use vortex_array::serde::ArrayChildren; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -76,9 +76,9 @@ use crate::utils::validate_binary_tensor_float_inputs; pub struct InnerProduct; impl InnerProduct { - /// Creates a new [`ScalarFn`] wrapping the inner product operation. - pub fn new() -> ScalarFn { - ScalarFn::new(InnerProduct, EmptyOptions) + /// Creates a new [`TypedScalarFnInstance`] wrapping the inner product operation. + pub fn new() -> TypedScalarFnInstance { + TypedScalarFnInstance::new(InnerProduct, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the inner product between `lhs` and diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 8474216156a..fa6e787eeac 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -18,8 +18,8 @@ use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn as ScalarFnArrayEncoding; use vortex_array::arrays::ScalarFnArray; -use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; @@ -42,9 +42,9 @@ use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; -use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::TypedScalarFnInstance; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::serde::ArrayChildren; use vortex_array::validity::Validity; @@ -89,12 +89,12 @@ use crate::utils::validate_tensor_float_input; pub struct L2Denorm; impl L2Denorm { - /// Creates a new [`ScalarFn`] wrapping the L2 denormalization operation. + /// Creates a new [`TypedScalarFnInstance`] wrapping the L2 denormalization operation. /// /// This is a low-level scalar-function descriptor constructor. To build a semantically valid /// [`L2Denorm`] array, prefer [`try_new_array`](Self::try_new_array). - pub fn new() -> ScalarFn { - ScalarFn::new(L2Denorm, EmptyOptions) + pub fn new() -> TypedScalarFnInstance { + TypedScalarFnInstance::new(L2Denorm, EmptyOptions) } /// Constructs a validated [`ScalarFnArray`] that lazily re-applies `norms` to `normalized`. @@ -1119,7 +1119,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); let original = normalize_as_l2_denorm(input, &mut ctx)?.into_array(); - let scalar_fn_array = original.as_::(); + let scalar_fn_array = original.as_::(); let children = scalar_fn_array.children(); let plugin = ScalarFnArrayPlugin::new(L2Denorm); diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index d9341b282db..5a8172939b7 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -14,8 +14,8 @@ use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn as ScalarFnArrayEncoding; use vortex_array::arrays::ScalarFnArray; -use vortex_array::arrays::ScalarFnVTable as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; @@ -33,9 +33,9 @@ use vortex_array::scalar_fn::Arity; use vortex_array::scalar_fn::ChildName; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::ExecutionArgs; -use vortex_array::scalar_fn::ScalarFn; use vortex_array::scalar_fn::ScalarFnId; use vortex_array::scalar_fn::ScalarFnVTable; +use vortex_array::scalar_fn::TypedScalarFnInstance; use vortex_array::serde::ArrayChildren; use vortex_buffer::Buffer; use vortex_error::VortexExpect; @@ -65,9 +65,9 @@ use crate::utils::validate_tensor_float_input; pub struct L2Norm; impl L2Norm { - /// Creates a new [`ScalarFn`] wrapping the L2 norm operation. - pub fn new() -> ScalarFn { - ScalarFn::new(L2Norm, EmptyOptions) + /// Creates a new [`TypedScalarFnInstance`] wrapping the L2 norm operation. + pub fn new() -> TypedScalarFnInstance { + TypedScalarFnInstance::new(L2Norm, EmptyOptions) } /// Constructs a [`ScalarFnArray`] that lazily computes the L2 norm over `child`. diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs index 96e6af414ca..c974d9a9f09 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs @@ -47,7 +47,7 @@ use std::fmt::Formatter; use vortex_array::ArrayRef; use vortex_array::arrays::ScalarFnArray; use vortex_array::dtype::PType; -use vortex_array::scalar_fn::ScalarFn; +use vortex_array::scalar_fn::TypedScalarFnInstance; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -89,9 +89,9 @@ pub struct SorfOptions { } impl SorfTransform { - /// Creates a new [`ScalarFn`] wrapping the SORF inverse transform with the given options. - pub fn new(options: &SorfOptions) -> ScalarFn { - ScalarFn::new(SorfTransform, options.clone()) + /// Creates a new [`TypedScalarFnInstance`] wrapping the SORF inverse transform with the given options. + pub fn new(options: &SorfOptions) -> TypedScalarFnInstance { + TypedScalarFnInstance::new(SorfTransform, options.clone()) } /// Constructs a validated [`ScalarFnArray`] that lazily applies the inverse SORF transform. From f1522814d708d324532e292102ecfdfafc19c19c Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:53:54 -0400 Subject: [PATCH 156/250] Implement smarter sampler (#7585) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 Right now, we have a very naive sampling strategy in the compressor. We will run the estimation function in the order of the `Scheme` declaration, regardless of how expensive it is to estimate things. This changes the sampling algorithm to first do all of the cheap estimations, and only when all of them fail do we do the more expensive sampling / callbacks. Edit: so benchmarks show no real change, but theoretically if someone comes along and has a really slow compress scheme, this should lessen the impact of that. ## Testing Will run benchmarks. Signed-off-by: Connor Tsui --- vortex-btrblocks/src/schemes/integer.rs | 32 +- vortex-compressor/public-api.lock | 28 +- .../src/builtins/constant/float.rs | 2 +- .../src/builtins/constant/string.rs | 2 +- vortex-compressor/src/compressor.rs | 303 ++++++++++++++++-- vortex-compressor/src/ctx.rs | 1 + vortex-compressor/src/estimate.rs | 36 ++- 7 files changed, 350 insertions(+), 54 deletions(-) diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 7e6c3503239..7dc1fccad80 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -17,6 +17,7 @@ use vortex_compressor::builtins::FloatDictScheme; use vortex_compressor::builtins::StringDictScheme; use vortex_compressor::estimate::CompressionEstimate; use vortex_compressor::estimate::DeferredEstimate; +use vortex_compressor::estimate::EstimateScore; use vortex_compressor::estimate::EstimateVerdict; use vortex_compressor::scheme::AncestorExclusion; use vortex_compressor::scheme::ChildSelection; @@ -737,20 +738,29 @@ impl Scheme for SequenceScheme { return CompressionEstimate::Verdict(EstimateVerdict::Skip); } - // TODO(connor): Why do we sequence encode the whole thing and then throw it away? And then - // why do we divide the ratio by 2??? - + // TODO(connor): `sequence_encode` allocates the encoded array just to confirm feasibility. + // A cheaper `is_sequence` probe would let us skip the allocation entirely. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, _ctx, exec_ctx| { - let Some(encoded) = sequence_encode(data.array_as_primitive(), exec_ctx)? else { - // If we are unable to sequence encode this array, make sure we skip. + |_compressor, data, best_so_far, _ctx, exec_ctx| { + // `SequenceArray` stores exactly two scalars (base and multiplier), so the best + // achievable compression ratio is `array_len / 2`. + let compressed_size = 2usize; + let max_ratio = data.array_len() as f64 / compressed_size as f64; + + // If we cannot beat the best so far, then we do not want to even try sequence + // encoding the data. + let threshold = best_so_far.and_then(EstimateScore::finite_ratio); + if threshold.is_some_and(|t| max_ratio <= t) { return Ok(EstimateVerdict::Skip); - }; + } - // TODO(connor): This doesn't really make sense? - // Since two values are required to store base and multiplier the compression ratio is - // divided by 2. - Ok(EstimateVerdict::Ratio(encoded.len() as f64 / 2.0)) + // TODO(connor): We should pass this array back to the compressor in the case that + // we do want to sequence encode this so that we do not need to recompress. + if sequence_encode(data.array_as_primitive(), exec_ctx)?.is_none() { + return Ok(EstimateVerdict::Skip); + } + // TODO(connor): Should we get the actual ratio here? + Ok(EstimateVerdict::Ratio(max_ratio)) }, ))) } diff --git a/vortex-compressor/public-api.lock b/vortex-compressor/public-api.lock index 05cbad5ad6f..d93f2c44305 100644 --- a/vortex-compressor/public-api.lock +++ b/vortex-compressor/public-api.lock @@ -326,6 +326,32 @@ impl core::fmt::Debug for vortex_compressor::estimate::DeferredEstimate pub fn vortex_compressor::estimate::DeferredEstimate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum vortex_compressor::estimate::EstimateScore + +pub vortex_compressor::estimate::EstimateScore::FiniteCompression(f64) + +pub vortex_compressor::estimate::EstimateScore::ZeroBytes + +impl vortex_compressor::estimate::EstimateScore + +pub fn vortex_compressor::estimate::EstimateScore::finite_ratio(self) -> core::option::Option + +impl core::clone::Clone for vortex_compressor::estimate::EstimateScore + +pub fn vortex_compressor::estimate::EstimateScore::clone(&self) -> vortex_compressor::estimate::EstimateScore + +impl core::cmp::PartialEq for vortex_compressor::estimate::EstimateScore + +pub fn vortex_compressor::estimate::EstimateScore::eq(&self, other: &vortex_compressor::estimate::EstimateScore) -> bool + +impl core::fmt::Debug for vortex_compressor::estimate::EstimateScore + +pub fn vortex_compressor::estimate::EstimateScore::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl core::marker::Copy for vortex_compressor::estimate::EstimateScore + +impl core::marker::StructuralPartialEq for vortex_compressor::estimate::EstimateScore + pub enum vortex_compressor::estimate::EstimateVerdict pub vortex_compressor::estimate::EstimateVerdict::AlwaysUse @@ -338,7 +364,7 @@ impl core::fmt::Debug for vortex_compressor::estimate::EstimateVerdict pub fn vortex_compressor::estimate::EstimateVerdict::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &vortex_compressor::stats::ArrayAndStats, vortex_compressor::ctx::CompressorContext, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) +pub type vortex_compressor::estimate::EstimateFn = (dyn core::ops::function::FnOnce(&vortex_compressor::CascadingCompressor, &vortex_compressor::stats::ArrayAndStats, core::option::Option, vortex_compressor::ctx::CompressorContext, &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + core::marker::Send + core::marker::Sync) pub mod vortex_compressor::scheme diff --git a/vortex-compressor/src/builtins/constant/float.rs b/vortex-compressor/src/builtins/constant/float.rs index 1445a6cca2d..501e3cd9623 100644 --- a/vortex-compressor/src/builtins/constant/float.rs +++ b/vortex-compressor/src/builtins/constant/float.rs @@ -66,7 +66,7 @@ impl Scheme for FloatConstantScheme { // This is an expensive check, but in practice the distinct count is known because we often // include dictionary encoding in our set of schemes, so we rarely call this. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, _ctx, exec_ctx| { + |_compressor, data, _best_so_far, _ctx, exec_ctx| { if is_constant(data.array(), exec_ctx)? { Ok(EstimateVerdict::AlwaysUse) } else { diff --git a/vortex-compressor/src/builtins/constant/string.rs b/vortex-compressor/src/builtins/constant/string.rs index 68ccfe14e8a..fcd6138bca2 100644 --- a/vortex-compressor/src/builtins/constant/string.rs +++ b/vortex-compressor/src/builtins/constant/string.rs @@ -60,7 +60,7 @@ impl Scheme for StringConstantScheme { // This is an expensive check, but the alternative of not compressing a constant array is // far less preferable. CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, data, _ctx, exec_ctx| { + |_compressor, data, _best_so_far, _ctx, exec_ctx| { if is_constant(data.array(), exec_ctx)? { Ok(EstimateVerdict::AlwaysUse) } else { diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 0f67619e3cb..aa18b61637b 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -349,7 +349,17 @@ impl CascadingCompressor { /// Calls [`expected_compression_ratio`] on each candidate and returns the winning scheme along /// with its resolved winner estimate, or `None` if no scheme beats the canonical encoding. - /// Ties are broken by registration order (earlier in the list wins). + /// + /// Selection runs in two passes. Pass 1 evaluates every immediate + /// [`CompressionEstimate::Verdict`] and tracks the running best. [`Scheme`]s returning + /// [`CompressionEstimate::Deferred`] are stashed for pass 2 so that we do not make any + /// expensive computations if we don't have to. + /// + /// Pass 2 evaluates the deferred work and, for each [`DeferredEstimate::Callback`], passes the + /// current best [`EstimateScore`] as an early-exit hint so the callback can return + /// [`EstimateVerdict::Skip`] without doing expensive work when it cannot beat the threshold. + /// + /// Ties are broken by registration order within each pass. /// /// [`expected_compression_ratio`]: Scheme::expected_compression_ratio fn choose_best_scheme( @@ -360,40 +370,61 @@ impl CascadingCompressor { exec_ctx: &mut ExecutionCtx, ) -> VortexResult> { let mut best: Option<(&'static dyn Scheme, EstimateScore)> = None; + let mut deferred: Vec<(&'static dyn Scheme, DeferredEstimate)> = Vec::new(); - // TODO(connor): Rather than computing the deferred estimates eagerly, it would be better to - // look at all quick estimates and see if it makes sense to sample at all. + // Pass 1: evaluate every immediate verdict. Stash deferred work for pass 2. for &scheme in schemes { - let verdict = - match scheme.expected_compression_ratio(data, compress_ctx.clone(), exec_ctx) { - CompressionEstimate::Verdict(verdict) => verdict, - CompressionEstimate::Deferred(DeferredEstimate::Sample) => { - let score = estimate_compression_ratio_with_sampling( - scheme, - self, - data.array(), - compress_ctx.clone(), - exec_ctx, - )?; - if is_better_score(score, &best) { - best = Some((scheme, score)); - } - continue; - } - CompressionEstimate::Deferred(DeferredEstimate::Callback(callback)) => { - callback(self, data, compress_ctx.clone(), exec_ctx)? + match scheme.expected_compression_ratio(data, compress_ctx.clone(), exec_ctx) { + CompressionEstimate::Verdict(EstimateVerdict::Skip) => {} + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) => { + return Ok(Some((scheme, WinnerEstimate::AlwaysUse))); + } + CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio)) => { + let score = EstimateScore::FiniteCompression(ratio); + + if is_better_score(score, &best) { + best = Some((scheme, score)); } - }; + } + CompressionEstimate::Deferred(deferred_estimate) => { + deferred.push((scheme, deferred_estimate)); + } + } + } + + // Pass 2: run deferred work. Callbacks receive the current best as a threshold so they can + // short-circuit with `Skip` when they cannot beat it. + for (scheme, deferred_estimate) in deferred { + let threshold: Option = best.map(|(_, score)| score); + match deferred_estimate { + DeferredEstimate::Sample => { + let score = estimate_compression_ratio_with_sampling( + self, + scheme, + data.array(), + compress_ctx.clone(), + exec_ctx, + )?; - match verdict { - EstimateVerdict::Skip => {} - EstimateVerdict::AlwaysUse => return Ok(Some((scheme, WinnerEstimate::AlwaysUse))), - EstimateVerdict::Ratio(ratio) => { - let score = EstimateScore::FiniteCompression(ratio); if is_better_score(score, &best) { best = Some((scheme, score)); } } + DeferredEstimate::Callback(callback) => { + match callback(self, data, threshold, compress_ctx.clone(), exec_ctx)? { + EstimateVerdict::Skip => {} + EstimateVerdict::AlwaysUse => { + return Ok(Some((scheme, WinnerEstimate::AlwaysUse))); + } + EstimateVerdict::Ratio(ratio) => { + let score = EstimateScore::FiniteCompression(ratio); + + if is_better_score(score, &best) { + best = Some((scheme, score)); + } + } + } + } } } @@ -757,7 +788,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::AlwaysUse), + |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::AlwaysUse), ))) } @@ -791,7 +822,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::Skip), + |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Skip), ))) } @@ -825,7 +856,7 @@ mod tests { _exec_ctx: &mut ExecutionCtx, ) -> CompressionEstimate { CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( - |_compressor, _data, _ctx, _exec_ctx| Ok(EstimateVerdict::Ratio(3.0)), + |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(3.0)), ))) } @@ -1201,6 +1232,216 @@ mod tests { Ok(()) } + // Observer helper used by threshold-related tests. Captures the `best_so_far` value the + // compressor passes to its deferred callback. `OBSERVER_LOCK` serializes tests that share + // `OBSERVED_THRESHOLD` so they do not race. + static OBSERVER_LOCK: Mutex<()> = Mutex::new(()); + static OBSERVED_THRESHOLD: Mutex>> = Mutex::new(None); + + #[derive(Debug)] + struct ThresholdObservingScheme; + + impl Scheme for ThresholdObservingScheme { + fn scheme_name(&self) -> &'static str { + "test.threshold_observing" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, best_so_far, _ctx, _exec_ctx| { + *OBSERVED_THRESHOLD.lock() = Some(best_so_far); + Ok(EstimateVerdict::Skip) + }, + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[derive(Debug)] + struct CallbackMatchingRatioScheme; + + impl Scheme for CallbackMatchingRatioScheme { + fn scheme_name(&self) -> &'static str { + "test.callback_matching_ratio" + } + + fn matches(&self, canonical: &Canonical) -> bool { + matches_integer_primitive(canonical) + } + + fn expected_compression_ratio( + &self, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, _data, _ctx, _exec_ctx, _best_so_far| Ok(EstimateVerdict::Ratio(2.0)), + ))) + } + + fn compress( + &self, + _compressor: &CascadingCompressor, + _data: &ArrayAndStats, + _compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + unreachable!("test helper should never be selected for compression") + } + } + + #[test] + fn callback_always_use_overrides_pass_one_best() -> VortexResult<()> { + // `HugeRatioScheme` returns an immediate `Ratio(100.0)` in pass 1; + // `CallbackAlwaysUseScheme` returns `AlwaysUse` from its deferred callback in pass 2. + // The deferred `AlwaysUse` must still win. + let compressor = CascadingCompressor::new(vec![&HugeRatioScheme, &CallbackAlwaysUseScheme]); + let schemes: [&'static dyn Scheme; 2] = [&HugeRatioScheme, &CallbackAlwaysUseScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::AlwaysUse)) + if scheme.id() == CallbackAlwaysUseScheme.id() + )); + Ok(()) + } + + #[test] + fn threshold_reflects_pass_one_best() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_THRESHOLD.lock() = None; + + let compressor = + CascadingCompressor::new(vec![&DirectRatioScheme, &ThresholdObservingScheme]); + let schemes: [&'static dyn Scheme; 2] = [&DirectRatioScheme, &ThresholdObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + let observed = *OBSERVED_THRESHOLD.lock(); + assert!(matches!( + observed, + Some(Some(EstimateScore::FiniteCompression(r))) if r == 2.0 + )); + Ok(()) + } + + #[test] + fn threshold_is_none_when_only_prior_is_zero_bytes() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_THRESHOLD.lock() = None; + + let compressor = + CascadingCompressor::new(vec![&ZeroBytesSamplingScheme, &ThresholdObservingScheme]); + let schemes: [&'static dyn Scheme; 2] = + [&ZeroBytesSamplingScheme, &ThresholdObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + // The observing callback was invoked (outer `Some`) and `best_so_far` was `None` (inner + // `None`) because the zero-byte sample is never stored as the best. + let observed = *OBSERVED_THRESHOLD.lock(); + assert_eq!(observed, Some(None)); + Ok(()) + } + + #[test] + fn threshold_is_none_when_no_prior_scheme() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_THRESHOLD.lock() = None; + + let compressor = CascadingCompressor::new(vec![&ThresholdObservingScheme]); + let schemes: [&'static dyn Scheme; 1] = [&ThresholdObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + let observed = *OBSERVED_THRESHOLD.lock(); + assert_eq!(observed, Some(None)); + Ok(()) + } + + #[test] + fn threshold_updates_from_earlier_deferred_callback() -> VortexResult<()> { + let _guard = OBSERVER_LOCK.lock(); + *OBSERVED_THRESHOLD.lock() = None; + + // Both schemes are deferred. The first callback registers `Ratio(3.0)`; the second + // callback must observe it as its threshold. + let compressor = + CascadingCompressor::new(vec![&CallbackRatioScheme, &ThresholdObservingScheme]); + let schemes: [&'static dyn Scheme; 2] = [&CallbackRatioScheme, &ThresholdObservingScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + compressor.choose_best_scheme(&schemes, &data, CompressorContext::new(), &mut exec_ctx)?; + + let observed = *OBSERVED_THRESHOLD.lock(); + assert!(matches!( + observed, + Some(Some(EstimateScore::FiniteCompression(r))) if r == 3.0 + )); + Ok(()) + } + + #[test] + fn ratio_tie_between_immediate_and_deferred_favors_immediate() -> VortexResult<()> { + // Both schemes produce the same `Ratio(2.0)`, one from pass 1 (immediate) and one from + // pass 2 (deferred callback). Pass 1 locks in first, and strict `>` tie-breaking means + // the deferred callback's equal ratio cannot displace it. + let compressor = + CascadingCompressor::new(vec![&CallbackMatchingRatioScheme, &DirectRatioScheme]); + let schemes: [&'static dyn Scheme; 2] = [&CallbackMatchingRatioScheme, &DirectRatioScheme]; + let data = estimate_test_data(); + let mut exec_ctx = SESSION.create_execution_ctx(); + + let winner = compressor.choose_best_scheme( + &schemes, + &data, + CompressorContext::new(), + &mut exec_ctx, + )?; + + assert!(matches!( + winner, + Some((scheme, WinnerEstimate::Score(EstimateScore::FiniteCompression(r)))) + if scheme.id() == DirectRatioScheme.id() && r == 2.0 + )); + Ok(()) + } + #[test] fn all_null_array_compresses_to_constant() -> VortexResult<()> { let array = PrimitiveArray::new( @@ -1244,8 +1485,8 @@ mod tests { // "this must be present since `DictScheme` declared that we need distinct values" let mut exec_ctx = SESSION.create_execution_ctx(); let score = estimate_compression_ratio_with_sampling( - &FloatDictScheme, &compressor, + &FloatDictScheme, &array, ctx, &mut exec_ctx, diff --git a/vortex-compressor/src/ctx.rs b/vortex-compressor/src/ctx.rs index 576fc8002a6..4e0619ff8ee 100644 --- a/vortex-compressor/src/ctx.rs +++ b/vortex-compressor/src/ctx.rs @@ -30,6 +30,7 @@ pub struct CompressorContext { /// Merged stats options from all eligible schemes at this compression site. merged_stats_options: GenerateStatsOptions, + // TODO(connor): Replace this with an `im::Vector` /// The cascade chain: `(scheme_id, child_index)` pairs from root to current depth. /// Used for self-exclusion, push rules ([`descendant_exclusions`]), and pull rules /// ([`ancestor_exclusions`]). diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 22a99fbccdc..9fbf434352a 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -23,12 +23,21 @@ use crate::trace; /// Closure type for [`DeferredEstimate::Callback`]. /// -/// The compressor calls this with the same arguments it would pass to sampling. The closure must -/// resolve directly to a terminal [`EstimateVerdict`]. +/// The compressor calls this with the same arguments it would pass to sampling, plus the best +/// [`EstimateScore`] observed so far (if any). The closure must resolve directly to a terminal +/// [`EstimateVerdict`]. +/// +/// The `best_so_far` threshold is an early-exit hint. If your scheme's maximum achievable +/// compression ratio is not strictly greater than +/// `best_so_far.and_then(EstimateScore::finite_ratio)`, you should return +/// [`EstimateVerdict::Skip`]. Returning a ratio equal to the threshold is permitted but will +/// lose to the prior best due to strict `>` tie-breaking in the selector. Use the threshold +/// only as an early-exit hint, never to perform additional work. #[rustfmt::skip] pub type EstimateFn = dyn FnOnce( &CascadingCompressor, &ArrayAndStats, + Option, CompressorContext, &mut ExecutionCtx, ) -> VortexResult @@ -85,17 +94,23 @@ pub enum DeferredEstimate { /// Use this only when the scheme needs to perform trial encoding or other costly checks to /// determine its compression ratio. The callback returns an [`EstimateVerdict`] directly, so /// it cannot request more sampling or another deferred callback. + /// + /// The compressor evaluates all immediate [`CompressionEstimate::Verdict`] results before + /// invoking any deferred callback, and passes the best [`EstimateScore`] observed so far to + /// the callback. This lets the callback return [`EstimateVerdict::Skip`] without performing + /// expensive work when its maximum achievable ratio cannot beat the current best. See + /// [`EstimateFn`] for the full contract. Callback(Box), } /// Ranked estimate used for comparing non-terminal compression candidates. #[derive(Debug, Clone, Copy, PartialEq)] -pub(super) enum EstimateScore { +pub enum EstimateScore { /// A finite compression ratio. Higher means a smaller amount of data, so it is better. FiniteCompression(f64), /// Trial compression produced a 0-byte output. /// - /// This has no finite trace ratio and is not eligible for scheme selection. + /// This has no finite ratio and is not eligible for scheme selection. /// /// TODO(connor): A zero-byte sample usually means the sampler happened to hit an all-null /// sample. Improve this logic so we can distinguish real zero-byte wins from sampling artifacts. @@ -112,8 +127,11 @@ impl EstimateScore { } } - /// Returns the traceable numeric ratio, omitting the zero-byte special case. - pub(super) fn trace_ratio(self) -> Option { + /// Returns the finite compression ratio, or [`None`] for the zero-byte special case. + /// + /// Callers comparing a scheme's maximum achievable ratio against a "best so far" threshold + /// should use this to extract a numeric value from an [`EstimateScore`]. + pub fn finite_ratio(self) -> Option { match self { Self::FiniteCompression(ratio) => Some(ratio), Self::ZeroBytes => None, @@ -156,7 +174,7 @@ impl WinnerEstimate { pub(super) fn trace_ratio(self) -> Option { match self { Self::AlwaysUse => None, - Self::Score(score) => score.trace_ratio(), + Self::Score(score) => score.finite_ratio(), } } } @@ -178,8 +196,8 @@ pub(super) fn is_better_score( /// /// Returns an error if sample compression fails. pub(super) fn estimate_compression_ratio_with_sampling( - scheme: &S, compressor: &CascadingCompressor, + scheme: &S, array: &ArrayRef, compress_ctx: CompressorContext, exec_ctx: &mut ExecutionCtx, @@ -212,7 +230,7 @@ pub(super) fn estimate_compression_ratio_with_sampling( // Single DEBUG event per sampled scheme. Downstream tooling can join this with the eventual // `scheme.compress_result` on the same scheme to compute sample-vs-full divergence. - trace::sample_result(scheme.id(), before, after, score.trace_ratio()); + trace::sample_result(scheme.id(), before, after, score.finite_ratio()); Ok(score) } From 452a4a3cb2e75f2e6b100664c6a4a75e335d187e Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:56:54 -0400 Subject: [PATCH 157/250] Remove `ExtensionData` (#7587) ## Summary This was unfortunately not caught in the big refactor. ## Testing Existing tests pass (and actually caught a bug in the extension array constructor that just happened to be hidden by `ExtensionData` panicking. Signed-off-by: Connor Tsui --- vortex-array/public-api.lock | 68 ++------ vortex-array/src/arrays/extension/array.rs | 145 +++--------------- vortex-array/src/arrays/extension/mod.rs | 1 - .../src/arrays/extension/vtable/mod.rs | 98 +++++++----- vortex-btrblocks/src/schemes/temporal.rs | 1 + vortex-tensor/src/encodings/l2_denorm.rs | 1 + 6 files changed, 101 insertions(+), 213 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 1e4a8a58ee5..8ee3c97f17b 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2302,7 +2302,7 @@ pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -2338,7 +2338,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension @@ -2368,44 +2368,16 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Ex pub fn vortex_array::arrays::Extension::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Extension>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub struct vortex_array::arrays::extension::ExtensionData - -impl vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef - -pub fn vortex_array::arrays::extension::ExtensionData::new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self - -pub unsafe fn vortex_array::arrays::extension::ExtensionData::new_unchecked(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> Self - -pub fn vortex_array::arrays::extension::ExtensionData::try_new(ext_dtype: vortex_array::dtype::extension::ExtDTypeRef, storage_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult - -impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::clone(&self) -> vortex_array::arrays::extension::ExtensionData - -impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::fmt::Display for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool - -impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) - pub trait vortex_array::arrays::extension::ExtensionArrayExt: vortex_array::TypedArrayRef +pub fn vortex_array::arrays::extension::ExtensionArrayExt::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef + pub fn vortex_array::arrays::extension::ExtensionArrayExt::storage_array(&self) -> &vortex_array::ArrayRef impl> vortex_array::arrays::extension::ExtensionArrayExt for T +pub fn T::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef + pub fn T::storage_array(&self) -> &vortex_array::ArrayRef pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array @@ -5558,7 +5530,7 @@ pub fn vortex_array::arrays::Extension::scalar_at(array: vortex_array::ArrayView impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -5594,7 +5566,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::ValidityChild for vortex_array::arrays::Extension @@ -19464,7 +19436,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -19500,7 +19472,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -20436,7 +20408,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -20472,7 +20444,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -22686,10 +22658,6 @@ impl vortex_array::ArrayEq for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool -impl vortex_array::ArrayEq for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_eq(&self, _other: &Self, _precision: vortex_array::Precision) -> bool - impl vortex_array::ArrayEq for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::array_eq(&self, other: &Self, precision: vortex_array::Precision) -> bool @@ -22794,10 +22762,6 @@ impl vortex_array::ArrayHash for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) -impl vortex_array::ArrayHash for vortex_array::arrays::extension::ExtensionData - -pub fn vortex_array::arrays::extension::ExtensionData::array_hash(&self, _state: &mut H, _precision: vortex_array::Precision) - impl vortex_array::ArrayHash for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::array_hash(&self, state: &mut H, precision: vortex_array::Precision) @@ -23132,7 +23096,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -23168,7 +23132,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter @@ -24352,7 +24316,7 @@ pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::array impl vortex_array::VTable for vortex_array::arrays::Extension -pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::ArrayData = vortex_array::EmptyArrayData pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension @@ -24388,7 +24352,7 @@ pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::validate(&self, _data: &vortex_array::EmptyArrayData, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option]) -> vortex_error::VortexResult<()> impl vortex_array::VTable for vortex_array::arrays::Filter diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 0a79774a2b5..b42f7b63603 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -1,13 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fmt::Display; -use std::fmt::Formatter; - use vortex_error::VortexExpect; use vortex_error::VortexResult; +use vortex_error::vortex_ensure_eq; use crate::ArrayRef; +use crate::EmptyArrayData; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -22,113 +21,14 @@ pub(super) const STORAGE_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; -/// An extension array that wraps another array with additional type information. -/// -/// **⚠️ Unstable API**: This is an experimental feature that may change significantly -/// in future versions. The extension type system is still evolving. -/// -/// Unlike Apache Arrow's extension arrays, Vortex extension arrays provide a more flexible -/// mechanism for adding semantic meaning to existing array types without requiring -/// changes to the core type system. -/// -/// ## Design Philosophy -/// -/// Extension arrays serve as a type-safe wrapper that: -/// - Preserves the underlying storage format and operations -/// - Adds semantic type information via `ExtDType` -/// - Enables custom serialization and deserialization logic -/// - Allows domain-specific interpretations of generic data -/// -/// ## Storage and Type Relationship -/// -/// The extension array maintains a strict contract: -/// - **Storage array**: Contains the actual data in a standard Vortex encoding -/// - **Extension type**: Defines how to interpret the storage data semantically -/// - **Type safety**: The storage array's dtype must match the extension type's storage dtype -/// -/// ## Use Cases -/// -/// Extension arrays are ideal for: -/// - **Custom numeric types**: Units of measurement, currencies -/// - **Temporal types**: Custom date/time formats, time zones, calendars -/// - **Domain-specific types**: UUIDs, IP addresses, geographic coordinates -/// - **Encoded types**: Base64 strings, compressed data, encrypted values -/// -/// ## Validity and Operations -/// -/// Extension arrays delegate validity and most operations to their storage array: -/// - Validity is inherited from the underlying storage -/// - Slicing preserves the extension type -/// - Scalar access wraps storage scalars with extension metadata -#[derive(Clone, Debug)] -pub struct ExtensionData { - /// The storage dtype. This **must** be a [`Extension::DType`] variant. - pub(super) ext_dtype: ExtDTypeRef, -} - -impl Display for ExtensionData { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "ext_dtype: {}", self.ext_dtype) - } -} - -impl ExtensionData { - /// Constructs a new `ExtensionArray`. - /// - /// # Panics - /// - /// Panics if the storage array in not compatible with the extension dtype. - pub fn new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { - Self::try_new(ext_dtype, storage_dtype).vortex_expect("Failed to create `ExtensionArray`") - } - - /// Tries to construct a new `ExtensionArray`. - /// - /// # Errors - /// - /// Returns an error if the storage array in not compatible with the extension dtype. - pub fn try_new(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> VortexResult { - // TODO(connor): Replace these statements once we add `validate_storage_array`. - // ext_dtype.validate_storage_array(&storage_array)?; - assert_eq!( - ext_dtype.storage_dtype(), - storage_dtype, - "ExtensionArray: storage_dtype must match storage array DType", - ); - - // SAFETY: we validate that the inputs are valid above. - Ok(unsafe { Self::new_unchecked(ext_dtype, storage_dtype) }) - } - - /// Creates a new `ExtensionArray`. - /// - /// # Safety - /// - /// The caller must ensure that the storage array is compatible with the extension dtype. In - /// other words, they must know that `ext_dtype.validate_storage_array(&storage_array)` has been - /// called successfully on this storage array. - pub unsafe fn new_unchecked(ext_dtype: ExtDTypeRef, storage_dtype: &DType) -> Self { - // TODO(connor): Replace these statements once we add `validate_storage_array`. - // #[cfg(debug_assertions)] - // ext_dtype - // .validate_storage_array(&storage_array) - // .vortex_expect("[Debug Assertion]: Invalid storage array for `ExtensionArray`"); - debug_assert_eq!( - ext_dtype.storage_dtype(), - storage_dtype, - "ExtensionArray: storage_dtype must match storage array DType", - ); - - Self { ext_dtype } - } - - /// The extension dtype of this array. - pub fn ext_dtype(&self) -> &ExtDTypeRef { - &self.ext_dtype +pub trait ExtensionArrayExt: TypedArrayRef { + fn ext_dtype(&self) -> &ExtDTypeRef { + self.as_ref() + .dtype() + .as_extension_opt() + .vortex_expect("extension array somehow did not have an extension dtype") } -} -pub trait ExtensionArrayExt: TypedArrayRef { fn storage_array(&self) -> &ArrayRef { self.as_ref().slots()[STORAGE_SLOT] .as_ref() @@ -144,26 +44,24 @@ impl Array { /// /// Panics if the storage array is not compatible with the extension dtype. pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - let dtype = DType::Extension(ext_dtype.clone()); - let len = storage_array.len(); - let data = ExtensionData::new(ext_dtype, storage_array.dtype()); - unsafe { - Array::from_parts_unchecked( - ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), - ) - } + Self::try_new(ext_dtype, storage_array).vortex_expect("Unable to create `ExtensionArray`") } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { - let dtype = DType::Extension(ext_dtype.clone()); + vortex_ensure_eq!( + ext_dtype.storage_dtype(), + storage_array.dtype(), + "Tried to create an `ExtensionArray` with an incompatible storage array" + ); + + let dtype = DType::Extension(ext_dtype); let len = storage_array.len(); - let data = ExtensionData::try_new(ext_dtype, storage_array.dtype())?; - Ok(unsafe { - Array::from_parts_unchecked( - ArrayParts::new(Extension, dtype, len, data).with_slots(vec![Some(storage_array)]), - ) - }) + + let parts = ArrayParts::new(Extension, dtype, len, EmptyArrayData) + .with_slots(vec![Some(storage_array)]); + + Ok(unsafe { Array::from_parts_unchecked(parts) }) } /// Creates a new [`ExtensionArray`](crate::arrays::ExtensionArray) from a vtable, metadata, and @@ -176,6 +74,7 @@ impl Array { let ext_dtype = ExtDType::::try_with_vtable(vtable, metadata, storage_array.dtype().clone())? .erased(); + Self::try_new(ext_dtype, storage_array) } } diff --git a/vortex-array/src/arrays/extension/mod.rs b/vortex-array/src/arrays/extension/mod.rs index 802545025c1..57bc9df5ead 100644 --- a/vortex-array/src/arrays/extension/mod.rs +++ b/vortex-array/src/arrays/extension/mod.rs @@ -3,7 +3,6 @@ mod array; pub use array::ExtensionArrayExt; -pub use array::ExtensionData; pub use vtable::ExtensionArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 82ad85e701f..b048c81fff7 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -1,29 +1,26 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::hash::Hasher; - use kernel::PARENT_KERNELS; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; use vortex_session::registry::CachedId; -use crate::ArrayEq; -use crate::ArrayHash; use crate::ArrayRef; +use crate::EmptyArrayData; use crate::ExecutionCtx; use crate::ExecutionResult; -use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::array::ValidityVTableFromChild; -use crate::arrays::extension::ExtensionData; use crate::arrays::extension::array::SLOT_NAMES; use crate::arrays::extension::array::STORAGE_SLOT; use crate::arrays::extension::compute::rules::PARENT_RULES; @@ -36,24 +33,52 @@ mod kernel; mod operations; mod validity; +/// An extension array that wraps another array with additional type information. +/// +/// **⚠️ Unstable API**: This is an experimental feature that may change significantly +/// in future versions. The extension type system is still evolving. +/// +/// Unlike Apache Arrow's extension arrays, Vortex extension arrays provide a more flexible +/// mechanism for adding semantic meaning to existing array types without requiring +/// changes to the core type system. +/// +/// ## Design Philosophy +/// +/// Extension arrays serve as a type-safe wrapper that: +/// - Preserves the underlying storage format and operations +/// - Adds semantic type information via `ExtDType` +/// - Enables custom serialization and deserialization logic +/// - Allows domain-specific interpretations of generic data +/// +/// ## Storage and Type Relationship +/// +/// The extension array maintains a strict contract: +/// - **Storage array**: Contains the actual data in a standard Vortex encoding +/// - **Extension type**: Defines how to interpret the storage data semantically +/// - **Type safety**: The storage array's dtype must match the extension type's storage dtype +/// +/// ## Use Cases +/// +/// Extension arrays are ideal for: +/// - **Custom numeric types**: Units of measurement, currencies +/// - **Temporal types**: Custom date/time formats, time zones, calendars +/// - **Domain-specific types**: UUIDs, IP addresses, geographic coordinates +/// - **Encoded types**: Base64 strings, compressed data, encrypted values +/// +/// ## Validity and Operations +/// +/// Extension arrays delegate validity and most operations to their storage array: +/// - Validity is inherited from the underlying storage +/// - Slicing preserves the extension type +/// - Scalar access wraps storage scalars with extension metadata #[derive(Clone, Debug)] pub struct Extension; /// A [`Extension`]-encoded Vortex array. pub type ExtensionArray = Array; -impl ArrayHash for ExtensionData { - fn array_hash(&self, _state: &mut H, _precision: Precision) {} -} - -impl ArrayEq for ExtensionData { - fn array_eq(&self, _other: &Self, _precision: Precision) -> bool { - true - } -} - impl VTable for Extension { - type ArrayData = ExtensionData; + type ArrayData = EmptyArrayData; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; @@ -65,28 +90,30 @@ impl VTable for Extension { fn validate( &self, - data: &ExtensionData, + _data: &EmptyArrayData, dtype: &DType, len: usize, slots: &[Option], ) -> VortexResult<()> { - _ = data; let storage = slots[STORAGE_SLOT] .as_ref() .vortex_expect("ExtensionArray storage slot"); - vortex_ensure!( - storage.len() == len, - "ExtensionArray length {} does not match outer length {}", + vortex_ensure_eq!( + storage.len(), + len, + "ExtensionArray length {} does not match outer length {len}", storage.len(), - len ); - let actual_dtype = DType::Extension(data.ext_dtype.clone()); - vortex_ensure!( - &actual_dtype == dtype, - "ExtensionArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype + let ext_dtype = dtype + .as_extension_opt() + .ok_or_else(|| vortex_err!("not an extension dtype"))?; + + let actual_dtype = DType::Extension(ext_dtype.clone()); + vortex_ensure_eq!( + &actual_dtype, + dtype, + "ExtensionArray dtype {actual_dtype} does not match outer dtype {dtype}", ); Ok(()) @@ -120,7 +147,7 @@ impl VTable for Extension { _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult> { + ) -> VortexResult> { if !metadata.is_empty() { vortex_bail!( "ExtensionArray expects empty metadata, got {} bytes", @@ -134,13 +161,10 @@ impl VTable for Extension { vortex_bail!("Expected 1 child, got {}", children.len()); } let storage = children.get(0, ext_dtype.storage_dtype(), len)?; - Ok(crate::array::ArrayParts::new( - self.clone(), - dtype.clone(), - len, - ExtensionData::new(ext_dtype.clone(), storage.dtype()), + Ok( + ArrayParts::new(self.clone(), dtype.clone(), len, EmptyArrayData) + .with_slots(vec![Some(storage)]), ) - .with_slots(vec![Some(storage)])) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { diff --git a/vortex-btrblocks/src/schemes/temporal.rs b/vortex-btrblocks/src/schemes/temporal.rs index 73671da0ff3..73aa9eedfb4 100644 --- a/vortex-btrblocks/src/schemes/temporal.rs +++ b/vortex-btrblocks/src/schemes/temporal.rs @@ -12,6 +12,7 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::TemporalArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::dtype::extension::Matcher; use vortex_array::extension::datetime::AnyTemporal; diff --git a/vortex-tensor/src/encodings/l2_denorm.rs b/vortex-tensor/src/encodings/l2_denorm.rs index 61a72d6a4d2..6cb4fcb0626 100644 --- a/vortex-tensor/src/encodings/l2_denorm.rs +++ b/vortex-tensor/src/encodings/l2_denorm.rs @@ -5,6 +5,7 @@ use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; +use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_compressor::CascadingCompressor; use vortex_compressor::ctx::CompressorContext; use vortex_compressor::estimate::CompressionEstimate; From 029fb663576973ce7b518227fa49980ebbed6eb3 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 22 Apr 2026 14:47:47 +0100 Subject: [PATCH 158/250] feat: GPU dyn dispatch patches support (#7563) Integrates the structural plumbing as well as applying the patches in source and scalar ops within the GPU dynamic dispatch kernel. None of the CUDA dynamic dispatch benchmarks regressed by adding patches support. As part of this change, the fastlanes lane count for a given type is now determined at compile time via `FL_LANES`. --------- Signed-off-by: Alexander Droste --- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 12 +- vortex-cuda/kernels/src/bit_unpack_16.cu | 78 +-- .../kernels/src/bit_unpack_16_lanes.cuh | 32 +- vortex-cuda/kernels/src/bit_unpack_32.cu | 142 ++--- .../kernels/src/bit_unpack_32_lanes.cuh | 64 +-- vortex-cuda/kernels/src/bit_unpack_64.cu | 270 ++++----- .../kernels/src/bit_unpack_64_lanes.cuh | 128 ++--- vortex-cuda/kernels/src/bit_unpack_8.cu | 46 +- .../kernels/src/bit_unpack_8_lanes.cuh | 16 +- vortex-cuda/kernels/src/dynamic_dispatch.cu | 105 +++- vortex-cuda/kernels/src/dynamic_dispatch.h | 21 +- vortex-cuda/kernels/src/fastlanes_common.cuh | 19 +- vortex-cuda/kernels/src/patches.cuh | 15 +- vortex-cuda/src/bit_unpack_gen.rs | 35 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 513 ++++++++++++++++-- .../src/dynamic_dispatch/plan_builder.rs | 149 +++-- vortex-cuda/src/kernel/encodings/bitpacked.rs | 17 +- vortex-cuda/src/kernel/mod.rs | 1 + vortex-cuda/src/kernel/patches/types.rs | 91 +++- 19 files changed, 1190 insertions(+), 564 deletions(-) diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 5b81a2c2120..488e22f38d8 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -123,7 +123,7 @@ struct BenchRunner { } impl BenchRunner { - fn new(array: &vortex::array::ArrayRef, len: usize, cuda_ctx: &CudaExecutionCtx) -> Self { + fn new(array: &vortex::array::ArrayRef, len: usize, cuda_ctx: &mut CudaExecutionCtx) -> Self { let plan = match DispatchPlan::new(array).vortex_expect("build_dyn_dispatch_plan") { DispatchPlan::Fused(plan) => plan, _ => unreachable!("encoding not fusable"), @@ -201,7 +201,7 @@ fn bench_for_bitpacked(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); - let bench_runner = BenchRunner::new(&array, n, &cuda_ctx); + let bench_runner = BenchRunner::new(&array, n, &mut cuda_ctx); b.iter_custom(|iters| { let mut total_time = Duration::ZERO; @@ -246,7 +246,7 @@ fn bench_dict_bp_codes(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); - let bench_runner = BenchRunner::new(&array, n, &cuda_ctx); + let bench_runner = BenchRunner::new(&array, n, &mut cuda_ctx); b.iter_custom(|iters| { let mut total_time = Duration::ZERO; @@ -290,7 +290,7 @@ fn bench_runend(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); - let bench_runner = BenchRunner::new(&array, n, &cuda_ctx); + let bench_runner = BenchRunner::new(&array, n, &mut cuda_ctx); b.iter_custom(|iters| { let mut total_time = Duration::ZERO; @@ -344,7 +344,7 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); - let bench_runner = BenchRunner::new(&array, n, &cuda_ctx); + let bench_runner = BenchRunner::new(&array, n, &mut cuda_ctx); b.iter_custom(|iters| { let mut total_time = Duration::ZERO; @@ -409,7 +409,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); - let bench_runner = BenchRunner::new(&array, n, &cuda_ctx); + let bench_runner = BenchRunner::new(&array, n, &mut cuda_ctx); b.iter_custom(|iters| { let mut total_time = Duration::ZERO; diff --git a/vortex-cuda/kernels/src/bit_unpack_16.cu b/vortex-cuda/kernels/src/bit_unpack_16.cu index 3c05baf2011..a784df201d3 100644 --- a/vortex-cuda/kernels/src/bit_unpack_16.cu +++ b/vortex-cuda/kernels/src/bit_unpack_16.cu @@ -4,19 +4,19 @@ template __device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint16_t shared_out[1024]; + __shared__ uint16_t shared_out[FL_CHUNK]; // Step 1: Unpack into shared memory #pragma unroll - for (int i = 0; i < 2; i++) { - _bit_unpack_16_lane(in, shared_out, reference, thread_idx * 2 + i); + for (int i = 0; i < FL_LANES / 32; i++) { + _bit_unpack_16_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); } __syncwarp(); // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); - while (patch.index != 1024) { + while (patch.index != FL_CHUNK) { shared_out[patch.index] = patch.value; patch = cursor.next(); } @@ -24,7 +24,7 @@ __device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *_ // Step 3: Copy to global memory #pragma unroll - for (int i = 0; i < 32; i++) { + for (int i = 0; i < FL_CHUNK / 32; i++) { auto idx = i * 32 + thread_idx; out[idx] = shared_out[idx]; } @@ -32,120 +32,120 @@ __device__ void _bit_unpack_16_device(const uint16_t *__restrict in, uint16_t *_ extern "C" __global__ void bit_unpack_16_0bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 0 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 0)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<0>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_1bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 1 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 1)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<1>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_2bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 2 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 2)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<2>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_3bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 3 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 3)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<3>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_4bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 4 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 4)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<4>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_5bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 5 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 5)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<5>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_6bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 6 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 6)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<6>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_7bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 7 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 7)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<7>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_8bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 8 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 8)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<8>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_9bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 9 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 9)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<9>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_10bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 10 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 10)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<10>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_11bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 11 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 11)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<11>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_12bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 12 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 12)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<12>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_13bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 13 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 13)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<13>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_14bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 14 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 14)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<14>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_15bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 15 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 15)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<15>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_16_16bw_32t(const uint16_t *__restrict full_in, uint16_t *__restrict full_out, uint16_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 16 / sizeof(uint16_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 16)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_16_device<16>(in, out, reference, thread_idx, patches); } diff --git a/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh index 48c62fde2d4..ff992fa4ffb 100644 --- a/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh +++ b/vortex-cuda/kernels/src/bit_unpack_16_lanes.cuh @@ -19,7 +19,7 @@ __device__ void _bit_unpack_16_lane<0>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<1>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -59,7 +59,7 @@ __device__ void _bit_unpack_16_lane<1>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<2>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -101,7 +101,7 @@ __device__ void _bit_unpack_16_lane<2>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<3>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -145,7 +145,7 @@ __device__ void _bit_unpack_16_lane<3>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<4>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -191,7 +191,7 @@ __device__ void _bit_unpack_16_lane<4>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<5>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -239,7 +239,7 @@ __device__ void _bit_unpack_16_lane<5>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<6>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -289,7 +289,7 @@ __device__ void _bit_unpack_16_lane<6>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<7>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -341,7 +341,7 @@ __device__ void _bit_unpack_16_lane<7>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<8>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -395,7 +395,7 @@ __device__ void _bit_unpack_16_lane<8>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<9>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -451,7 +451,7 @@ __device__ void _bit_unpack_16_lane<9>(const uint16_t *__restrict in, uint16_t * template <> __device__ void _bit_unpack_16_lane<10>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -509,7 +509,7 @@ __device__ void _bit_unpack_16_lane<10>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<11>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -569,7 +569,7 @@ __device__ void _bit_unpack_16_lane<11>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<12>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -631,7 +631,7 @@ __device__ void _bit_unpack_16_lane<12>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<13>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -695,7 +695,7 @@ __device__ void _bit_unpack_16_lane<13>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<14>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -761,7 +761,7 @@ __device__ void _bit_unpack_16_lane<14>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<15>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; uint16_t src; uint16_t tmp; src = in[lane]; @@ -829,7 +829,7 @@ __device__ void _bit_unpack_16_lane<15>(const uint16_t *__restrict in, uint16_t template <> __device__ void _bit_unpack_16_lane<16>(const uint16_t *__restrict in, uint16_t *__restrict out, uint16_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 64; + constexpr unsigned int LANE_COUNT = FL_LANES; #pragma unroll for (int row = 0; row < 16; row++) { out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; diff --git a/vortex-cuda/kernels/src/bit_unpack_32.cu b/vortex-cuda/kernels/src/bit_unpack_32.cu index 97906f612c4..3f8fcb5c227 100644 --- a/vortex-cuda/kernels/src/bit_unpack_32.cu +++ b/vortex-cuda/kernels/src/bit_unpack_32.cu @@ -4,19 +4,19 @@ template __device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint32_t shared_out[1024]; + __shared__ uint32_t shared_out[FL_CHUNK]; // Step 1: Unpack into shared memory #pragma unroll - for (int i = 0; i < 1; i++) { - _bit_unpack_32_lane(in, shared_out, reference, thread_idx * 1 + i); + for (int i = 0; i < FL_LANES / 32; i++) { + _bit_unpack_32_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); } __syncwarp(); // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); - while (patch.index != 1024) { + while (patch.index != FL_CHUNK) { shared_out[patch.index] = patch.value; patch = cursor.next(); } @@ -24,7 +24,7 @@ __device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *_ // Step 3: Copy to global memory #pragma unroll - for (int i = 0; i < 32; i++) { + for (int i = 0; i < FL_CHUNK / 32; i++) { auto idx = i * 32 + thread_idx; out[idx] = shared_out[idx]; } @@ -32,232 +32,232 @@ __device__ void _bit_unpack_32_device(const uint32_t *__restrict in, uint32_t *_ extern "C" __global__ void bit_unpack_32_0bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 0 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 0)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<0>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_1bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 1 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 1)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<1>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_2bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 2 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 2)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<2>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_3bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 3 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 3)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<3>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_4bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 4 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 4)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<4>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_5bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 5 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 5)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<5>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_6bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 6 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 6)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<6>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_7bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 7 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 7)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<7>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_8bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 8 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 8)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<8>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_9bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 9 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 9)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<9>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_10bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 10 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 10)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<10>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_11bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 11 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 11)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<11>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_12bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 12 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 12)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<12>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_13bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 13 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 13)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<13>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_14bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 14 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 14)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<14>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_15bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 15 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 15)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<15>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_16bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 16 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 16)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<16>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_17bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 17 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 17)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<17>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_18bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 18 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 18)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<18>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_19bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 19 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 19)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<19>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_20bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 20 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 20)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<20>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_21bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 21 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 21)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<21>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_22bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 22 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 22)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<22>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_23bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 23 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 23)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<23>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_24bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 24 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 24)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<24>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_25bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 25 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 25)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<25>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_26bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 26 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 26)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<26>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_27bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 27 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 27)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<27>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_28bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 28 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 28)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<28>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_29bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 29 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 29)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<29>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_30bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 30 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 30)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<30>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_31bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 31 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 31)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<31>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_32_32bw_32t(const uint32_t *__restrict full_in, uint32_t *__restrict full_out, uint32_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 32 / sizeof(uint32_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 32)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_32_device<32>(in, out, reference, thread_idx, patches); } diff --git a/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh index 4d1206e9f88..58d4195e3aa 100644 --- a/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh +++ b/vortex-cuda/kernels/src/bit_unpack_32_lanes.cuh @@ -19,7 +19,7 @@ __device__ void _bit_unpack_32_lane<0>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<1>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -91,7 +91,7 @@ __device__ void _bit_unpack_32_lane<1>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<2>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -165,7 +165,7 @@ __device__ void _bit_unpack_32_lane<2>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<3>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -241,7 +241,7 @@ __device__ void _bit_unpack_32_lane<3>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<4>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -319,7 +319,7 @@ __device__ void _bit_unpack_32_lane<4>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<5>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -399,7 +399,7 @@ __device__ void _bit_unpack_32_lane<5>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<6>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -481,7 +481,7 @@ __device__ void _bit_unpack_32_lane<6>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<7>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -565,7 +565,7 @@ __device__ void _bit_unpack_32_lane<7>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<8>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -651,7 +651,7 @@ __device__ void _bit_unpack_32_lane<8>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<9>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -739,7 +739,7 @@ __device__ void _bit_unpack_32_lane<9>(const uint32_t *__restrict in, uint32_t * template <> __device__ void _bit_unpack_32_lane<10>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -829,7 +829,7 @@ __device__ void _bit_unpack_32_lane<10>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<11>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -921,7 +921,7 @@ __device__ void _bit_unpack_32_lane<11>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<12>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1015,7 +1015,7 @@ __device__ void _bit_unpack_32_lane<12>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<13>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1111,7 +1111,7 @@ __device__ void _bit_unpack_32_lane<13>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<14>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1209,7 +1209,7 @@ __device__ void _bit_unpack_32_lane<14>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<15>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1309,7 +1309,7 @@ __device__ void _bit_unpack_32_lane<15>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<16>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1411,7 +1411,7 @@ __device__ void _bit_unpack_32_lane<16>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<17>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1515,7 +1515,7 @@ __device__ void _bit_unpack_32_lane<17>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<18>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1621,7 +1621,7 @@ __device__ void _bit_unpack_32_lane<18>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<19>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1729,7 +1729,7 @@ __device__ void _bit_unpack_32_lane<19>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<20>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1839,7 +1839,7 @@ __device__ void _bit_unpack_32_lane<20>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<21>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -1951,7 +1951,7 @@ __device__ void _bit_unpack_32_lane<21>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<22>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2065,7 +2065,7 @@ __device__ void _bit_unpack_32_lane<22>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<23>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2181,7 +2181,7 @@ __device__ void _bit_unpack_32_lane<23>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<24>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2299,7 +2299,7 @@ __device__ void _bit_unpack_32_lane<24>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<25>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2419,7 +2419,7 @@ __device__ void _bit_unpack_32_lane<25>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<26>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2541,7 +2541,7 @@ __device__ void _bit_unpack_32_lane<26>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<27>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2665,7 +2665,7 @@ __device__ void _bit_unpack_32_lane<27>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<28>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2791,7 +2791,7 @@ __device__ void _bit_unpack_32_lane<28>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<29>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -2919,7 +2919,7 @@ __device__ void _bit_unpack_32_lane<29>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<30>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -3049,7 +3049,7 @@ __device__ void _bit_unpack_32_lane<30>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<31>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; uint32_t src; uint32_t tmp; src = in[lane]; @@ -3181,7 +3181,7 @@ __device__ void _bit_unpack_32_lane<31>(const uint32_t *__restrict in, uint32_t template <> __device__ void _bit_unpack_32_lane<32>(const uint32_t *__restrict in, uint32_t *__restrict out, uint32_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 32; + constexpr unsigned int LANE_COUNT = FL_LANES; #pragma unroll for (int row = 0; row < 32; row++) { out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; diff --git a/vortex-cuda/kernels/src/bit_unpack_64.cu b/vortex-cuda/kernels/src/bit_unpack_64.cu index 6270f4f8261..ebe0b125369 100644 --- a/vortex-cuda/kernels/src/bit_unpack_64.cu +++ b/vortex-cuda/kernels/src/bit_unpack_64.cu @@ -4,19 +4,19 @@ template __device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint64_t shared_out[1024]; + __shared__ uint64_t shared_out[FL_CHUNK]; // Step 1: Unpack into shared memory #pragma unroll - for (int i = 0; i < 1; i++) { - _bit_unpack_64_lane(in, shared_out, reference, thread_idx * 1 + i); + for (int i = 0; i < FL_LANES / 16; i++) { + _bit_unpack_64_lane(in, shared_out, reference, thread_idx * (FL_LANES / 16) + i); } __syncwarp(); // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 16); auto patch = cursor.next(); - while (patch.index != 1024) { + while (patch.index != FL_CHUNK) { shared_out[patch.index] = patch.value; patch = cursor.next(); } @@ -24,7 +24,7 @@ __device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *_ // Step 3: Copy to global memory #pragma unroll - for (int i = 0; i < 64; i++) { + for (int i = 0; i < FL_CHUNK / 16; i++) { auto idx = i * 16 + thread_idx; out[idx] = shared_out[idx]; } @@ -32,456 +32,456 @@ __device__ void _bit_unpack_64_device(const uint64_t *__restrict in, uint64_t *_ extern "C" __global__ void bit_unpack_64_0bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 0 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 0)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<0>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_1bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 1 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 1)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<1>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_2bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 2 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 2)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<2>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_3bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 3 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 3)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<3>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_4bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 4 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 4)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<4>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_5bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 5 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 5)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<5>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_6bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 6 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 6)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<6>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_7bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 7 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 7)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<7>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_8bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 8 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 8)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<8>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_9bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 9 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 9)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<9>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_10bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 10 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 10)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<10>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_11bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 11 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 11)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<11>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_12bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 12 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 12)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<12>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_13bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 13 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 13)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<13>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_14bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 14 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 14)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<14>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_15bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 15 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 15)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<15>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_16bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 16 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 16)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<16>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_17bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 17 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 17)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<17>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_18bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 18 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 18)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<18>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_19bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 19 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 19)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<19>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_20bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 20 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 20)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<20>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_21bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 21 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 21)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<21>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_22bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 22 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 22)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<22>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_23bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 23 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 23)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<23>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_24bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 24 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 24)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<24>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_25bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 25 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 25)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<25>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_26bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 26 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 26)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<26>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_27bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 27 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 27)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<27>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_28bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 28 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 28)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<28>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_29bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 29 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 29)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<29>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_30bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 30 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 30)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<30>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_31bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 31 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 31)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<31>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_32bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 32 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 32)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<32>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_33bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 33 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 33)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<33>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_34bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 34 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 34)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<34>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_35bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 35 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 35)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<35>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_36bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 36 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 36)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<36>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_37bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 37 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 37)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<37>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_38bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 38 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 38)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<38>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_39bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 39 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 39)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<39>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_40bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 40 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 40)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<40>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_41bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 41 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 41)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<41>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_42bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 42 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 42)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<42>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_43bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 43 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 43)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<43>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_44bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 44 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 44)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<44>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_45bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 45 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 45)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<45>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_46bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 46 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 46)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<46>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_47bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 47 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 47)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<47>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_48bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 48 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 48)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<48>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_49bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 49 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 49)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<49>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_50bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 50 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 50)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<50>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_51bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 51 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 51)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<51>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_52bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 52 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 52)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<52>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_53bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 53 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 53)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<53>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_54bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 54 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 54)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<54>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_55bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 55 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 55)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<55>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_56bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 56 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 56)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<56>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_57bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 57 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 57)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<57>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_58bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 58 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 58)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<58>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_59bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 59 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 59)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<59>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_60bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 60 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 60)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<60>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_61bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 61 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 61)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<61>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_62bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 62 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 62)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<62>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_63bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 63 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 63)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<63>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_64_64bw_16t(const uint64_t *__restrict full_in, uint64_t *__restrict full_out, uint64_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 64 / sizeof(uint64_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 64)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_64_device<64>(in, out, reference, thread_idx, patches); } diff --git a/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh index 0f20fee4e7d..afc17737b40 100644 --- a/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh +++ b/vortex-cuda/kernels/src/bit_unpack_64_lanes.cuh @@ -19,7 +19,7 @@ __device__ void _bit_unpack_64_lane<0>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<1>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -155,7 +155,7 @@ __device__ void _bit_unpack_64_lane<1>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<2>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -293,7 +293,7 @@ __device__ void _bit_unpack_64_lane<2>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<3>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -433,7 +433,7 @@ __device__ void _bit_unpack_64_lane<3>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<4>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -575,7 +575,7 @@ __device__ void _bit_unpack_64_lane<4>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<5>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -719,7 +719,7 @@ __device__ void _bit_unpack_64_lane<5>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<6>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -865,7 +865,7 @@ __device__ void _bit_unpack_64_lane<6>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<7>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1013,7 +1013,7 @@ __device__ void _bit_unpack_64_lane<7>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<8>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1163,7 +1163,7 @@ __device__ void _bit_unpack_64_lane<8>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<9>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1315,7 +1315,7 @@ __device__ void _bit_unpack_64_lane<9>(const uint64_t *__restrict in, uint64_t * template <> __device__ void _bit_unpack_64_lane<10>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1469,7 +1469,7 @@ __device__ void _bit_unpack_64_lane<10>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<11>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1625,7 +1625,7 @@ __device__ void _bit_unpack_64_lane<11>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<12>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1783,7 +1783,7 @@ __device__ void _bit_unpack_64_lane<12>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<13>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -1943,7 +1943,7 @@ __device__ void _bit_unpack_64_lane<13>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<14>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2105,7 +2105,7 @@ __device__ void _bit_unpack_64_lane<14>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<15>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2269,7 +2269,7 @@ __device__ void _bit_unpack_64_lane<15>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<16>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2435,7 +2435,7 @@ __device__ void _bit_unpack_64_lane<16>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<17>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2603,7 +2603,7 @@ __device__ void _bit_unpack_64_lane<17>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<18>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2773,7 +2773,7 @@ __device__ void _bit_unpack_64_lane<18>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<19>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -2945,7 +2945,7 @@ __device__ void _bit_unpack_64_lane<19>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<20>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -3119,7 +3119,7 @@ __device__ void _bit_unpack_64_lane<20>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<21>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -3295,7 +3295,7 @@ __device__ void _bit_unpack_64_lane<21>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<22>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -3473,7 +3473,7 @@ __device__ void _bit_unpack_64_lane<22>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<23>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -3653,7 +3653,7 @@ __device__ void _bit_unpack_64_lane<23>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<24>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -3835,7 +3835,7 @@ __device__ void _bit_unpack_64_lane<24>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<25>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4019,7 +4019,7 @@ __device__ void _bit_unpack_64_lane<25>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<26>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4205,7 +4205,7 @@ __device__ void _bit_unpack_64_lane<26>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<27>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4393,7 +4393,7 @@ __device__ void _bit_unpack_64_lane<27>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<28>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4583,7 +4583,7 @@ __device__ void _bit_unpack_64_lane<28>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<29>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4775,7 +4775,7 @@ __device__ void _bit_unpack_64_lane<29>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<30>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -4969,7 +4969,7 @@ __device__ void _bit_unpack_64_lane<30>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<31>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -5165,7 +5165,7 @@ __device__ void _bit_unpack_64_lane<31>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<32>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -5363,7 +5363,7 @@ __device__ void _bit_unpack_64_lane<32>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<33>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -5563,7 +5563,7 @@ __device__ void _bit_unpack_64_lane<33>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<34>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -5765,7 +5765,7 @@ __device__ void _bit_unpack_64_lane<34>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<35>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -5969,7 +5969,7 @@ __device__ void _bit_unpack_64_lane<35>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<36>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -6175,7 +6175,7 @@ __device__ void _bit_unpack_64_lane<36>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<37>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -6383,7 +6383,7 @@ __device__ void _bit_unpack_64_lane<37>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<38>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -6593,7 +6593,7 @@ __device__ void _bit_unpack_64_lane<38>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<39>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -6805,7 +6805,7 @@ __device__ void _bit_unpack_64_lane<39>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<40>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -7019,7 +7019,7 @@ __device__ void _bit_unpack_64_lane<40>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<41>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -7235,7 +7235,7 @@ __device__ void _bit_unpack_64_lane<41>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<42>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -7453,7 +7453,7 @@ __device__ void _bit_unpack_64_lane<42>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<43>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -7673,7 +7673,7 @@ __device__ void _bit_unpack_64_lane<43>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<44>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -7895,7 +7895,7 @@ __device__ void _bit_unpack_64_lane<44>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<45>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -8119,7 +8119,7 @@ __device__ void _bit_unpack_64_lane<45>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<46>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -8345,7 +8345,7 @@ __device__ void _bit_unpack_64_lane<46>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<47>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -8573,7 +8573,7 @@ __device__ void _bit_unpack_64_lane<47>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<48>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -8803,7 +8803,7 @@ __device__ void _bit_unpack_64_lane<48>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<49>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -9035,7 +9035,7 @@ __device__ void _bit_unpack_64_lane<49>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<50>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -9269,7 +9269,7 @@ __device__ void _bit_unpack_64_lane<50>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<51>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -9505,7 +9505,7 @@ __device__ void _bit_unpack_64_lane<51>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<52>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -9743,7 +9743,7 @@ __device__ void _bit_unpack_64_lane<52>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<53>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -9983,7 +9983,7 @@ __device__ void _bit_unpack_64_lane<53>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<54>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -10225,7 +10225,7 @@ __device__ void _bit_unpack_64_lane<54>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<55>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -10469,7 +10469,7 @@ __device__ void _bit_unpack_64_lane<55>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<56>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -10715,7 +10715,7 @@ __device__ void _bit_unpack_64_lane<56>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<57>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -10963,7 +10963,7 @@ __device__ void _bit_unpack_64_lane<57>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<58>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -11213,7 +11213,7 @@ __device__ void _bit_unpack_64_lane<58>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<59>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -11465,7 +11465,7 @@ __device__ void _bit_unpack_64_lane<59>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<60>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -11719,7 +11719,7 @@ __device__ void _bit_unpack_64_lane<60>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<61>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -11975,7 +11975,7 @@ __device__ void _bit_unpack_64_lane<61>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<62>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -12233,7 +12233,7 @@ __device__ void _bit_unpack_64_lane<62>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<63>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; uint64_t src; uint64_t tmp; src = in[lane]; @@ -12493,7 +12493,7 @@ __device__ void _bit_unpack_64_lane<63>(const uint64_t *__restrict in, uint64_t template <> __device__ void _bit_unpack_64_lane<64>(const uint64_t *__restrict in, uint64_t *__restrict out, uint64_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 16; + constexpr unsigned int LANE_COUNT = FL_LANES; #pragma unroll for (int row = 0; row < 64; row++) { out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; diff --git a/vortex-cuda/kernels/src/bit_unpack_8.cu b/vortex-cuda/kernels/src/bit_unpack_8.cu index 064970d650f..b2fcfd26f04 100644 --- a/vortex-cuda/kernels/src/bit_unpack_8.cu +++ b/vortex-cuda/kernels/src/bit_unpack_8.cu @@ -4,19 +4,19 @@ template __device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, int thread_idx, GPUPatches& patches) { - __shared__ uint8_t shared_out[1024]; + __shared__ uint8_t shared_out[FL_CHUNK]; // Step 1: Unpack into shared memory #pragma unroll - for (int i = 0; i < 4; i++) { - _bit_unpack_8_lane(in, shared_out, reference, thread_idx * 4 + i); + for (int i = 0; i < FL_LANES / 32; i++) { + _bit_unpack_8_lane(in, shared_out, reference, thread_idx * (FL_LANES / 32) + i); } __syncwarp(); // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); auto patch = cursor.next(); - while (patch.index != 1024) { + while (patch.index != FL_CHUNK) { shared_out[patch.index] = patch.value; patch = cursor.next(); } @@ -24,7 +24,7 @@ __device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__re // Step 3: Copy to global memory #pragma unroll - for (int i = 0; i < 32; i++) { + for (int i = 0; i < FL_CHUNK / 32; i++) { auto idx = i * 32 + thread_idx; out[idx] = shared_out[idx]; } @@ -32,64 +32,64 @@ __device__ void _bit_unpack_8_device(const uint8_t *__restrict in, uint8_t *__re extern "C" __global__ void bit_unpack_8_0bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 0 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 0)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<0>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_1bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 1 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 1)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<1>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_2bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 2 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 2)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<2>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_3bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 3 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 3)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<3>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_4bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 4 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 4)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<4>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_5bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 5 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 5)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<5>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_6bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 6 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 6)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<6>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_7bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 7 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 7)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<7>(in, out, reference, thread_idx, patches); } extern "C" __global__ void bit_unpack_8_8bw_32t(const uint8_t *__restrict full_in, uint8_t *__restrict full_out, uint8_t reference, GPUPatches patches) { int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * 8 / sizeof(uint8_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * 8)); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_8_device<8>(in, out, reference, thread_idx, patches); } diff --git a/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh b/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh index 64fb8ce39a8..d9c3550cb18 100644 --- a/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh +++ b/vortex-cuda/kernels/src/bit_unpack_8_lanes.cuh @@ -19,7 +19,7 @@ __device__ void _bit_unpack_8_lane<0>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<1>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -43,7 +43,7 @@ __device__ void _bit_unpack_8_lane<1>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<2>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -69,7 +69,7 @@ __device__ void _bit_unpack_8_lane<2>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<3>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -97,7 +97,7 @@ __device__ void _bit_unpack_8_lane<3>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<4>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -127,7 +127,7 @@ __device__ void _bit_unpack_8_lane<4>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<5>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -159,7 +159,7 @@ __device__ void _bit_unpack_8_lane<5>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<6>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -193,7 +193,7 @@ __device__ void _bit_unpack_8_lane<6>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<7>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; uint8_t src; uint8_t tmp; src = in[lane]; @@ -229,7 +229,7 @@ __device__ void _bit_unpack_8_lane<7>(const uint8_t *__restrict in, uint8_t *__r template <> __device__ void _bit_unpack_8_lane<8>(const uint8_t *__restrict in, uint8_t *__restrict out, uint8_t reference, unsigned int lane) { - unsigned int LANE_COUNT = 128; + constexpr unsigned int LANE_COUNT = FL_LANES; #pragma unroll for (int row = 0; row < 8; row++) { out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.cu b/vortex-cuda/kernels/src/dynamic_dispatch.cu index d3950485435..9a413af17c1 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.cu +++ b/vortex-cuda/kernels/src/dynamic_dispatch.cu @@ -52,6 +52,7 @@ #include "bit_unpack.cuh" #include "dynamic_dispatch.h" +#include "patches.cuh" #include "types.cuh" // ═══════════════════════════════════════════════════════════════════════════ @@ -109,8 +110,12 @@ __shared__ uint64_t runend_cursors[BLOCK_SIZE]; // ═══════════════════════════════════════════════════════════════════════════ /// Apply one scalar operation to N values in registers. +/// +/// `abs_pos` is the absolute output position of the first value to process. +/// It is used by scalar operations that apply patches, e.g. ALP. template -__device__ inline void scalar_op(T *values, const struct ScalarOp &op, char *__restrict smem) { +__device__ inline void +scalar_op(T *values, const struct ScalarOp &op, char *__restrict smem, uint64_t abs_pos = 0) { switch (op.op_code) { case ScalarOp::FOR: { const T ref = static_cast(op.params.frame_of_ref.reference); @@ -134,6 +139,33 @@ __device__ inline void scalar_op(T *values, const struct ScalarOp &op, char *__r float r = static_cast(static_cast(values[i])) * f * e; values[i] = static_cast(__float_as_uint(r)); } + // Apply ALP patches: override positions whose float value couldn't + // be reconstructed through the ALP encode/decode cycle. + // Per-value cursor — with a slice offset, a tile's N values can + // straddle two FL chunks, so each value needs its own lookup. + if (op.params.alp.patches_ptr != 0) { + const auto &patches = *reinterpret_cast(op.params.alp.patches_ptr); + // The sliced chunk_offsets array starts at original chunk + // (offset / FL_CHUNK). PatchesCursor indexes from 0, so + // subtract that base to get the index into chunk_offsets. + const uint32_t chunk_start = patches.offset / FL_CHUNK; +#pragma unroll + for (uint32_t i = 0; i < N; ++i) { + uint64_t my_pos = (N > 1) ? abs_pos + i * blockDim.x + threadIdx.x : abs_pos; + uint64_t orig = my_pos + patches.offset; + uint32_t chunk = static_cast(orig / FL_CHUNK) - chunk_start; + uint32_t within = static_cast(orig % FL_CHUNK); + PatchesCursor cursor(patches, chunk, 0, 1); + auto patch = cursor.next(); + while (patch.index != FL_CHUNK) { + if (patch.index == within) { + values[i] = patch.value; + break; + } + patch = cursor.next(); + } + } + } break; } case ScalarOp::DICT: { @@ -149,6 +181,24 @@ __device__ inline void scalar_op(T *values, const struct ScalarOp &op, char *__r } } +// ═══════════════════════════════════════════════════════════════════════════ +// Patches +// ═══════════════════════════════════════════════════════════════════════════ + +/// Scatter patches for a single chunk into `out` using PatchesCursor. +/// All threads in the block cooperate. Caller must issue __syncthreads() +/// afterward if other threads read from `out`. +template +__device__ __forceinline__ void +scatter_patches_chunk(const GPUPatches &patches, T *__restrict out, uint32_t chunk) { + PatchesCursor cursor(patches, chunk, threadIdx.x, blockDim.x); + auto patch = cursor.next(); + while (patch.index != FL_CHUNK) { + out[patch.index] = patch.value; + patch = cursor.next(); + } +} + // ═══════════════════════════════════════════════════════════════════════════ // Source ops // ═══════════════════════════════════════════════════════════════════════════ @@ -162,11 +212,8 @@ __device__ inline void bitunpack(const T *__restrict packed, uint64_t chunk_start, uint32_t chunk_len, const struct SourceOp &src) { - constexpr uint32_t T_BITS = sizeof(T) * 8; - constexpr uint32_t FL_CHUNK = 1024; - constexpr uint32_t LANES = FL_CHUNK / T_BITS; const uint32_t bw = src.params.bitunpack.bit_width; - const uint32_t words_per_block = LANES * bw; + const uint32_t words_per_block = FL_LANES * bw; const uint32_t elem_off = src.params.bitunpack.element_offset; const uint32_t dst_off = (chunk_start + elem_off) % FL_CHUNK; const uint64_t first_block = (chunk_start + elem_off) / FL_CHUNK; @@ -177,9 +224,15 @@ __device__ inline void bitunpack(const T *__restrict packed, for (uint32_t c = 0; c < n_chunks; ++c) { const T *src_chunk = packed + (first_block + c) * words_per_block; T *chunk_dst = dst + c * FL_CHUNK; - for (uint32_t lane = threadIdx.x; lane < LANES; lane += blockDim.x) { + for (uint32_t lane = threadIdx.x; lane < FL_LANES; lane += blockDim.x) { bit_unpack_lane(src_chunk, chunk_dst, 0, lane, bw); } + // Apply BitPacked patches inline, matching the standalone kernel pattern. + if (src.params.bitunpack.patches_ptr != 0) { + __syncthreads(); + const auto &patches = *reinterpret_cast(src.params.bitunpack.patches_ptr); + scatter_patches_chunk(patches, chunk_dst, first_block + c); + } } } @@ -282,15 +335,12 @@ __device__ void execute_output_stage(T *__restrict output, // Cap at 4 values per thread per tile to minimise register pressure. constexpr uint32_t VALUES_PER_TILE = (32 / sizeof(T)) < 4 ? (32 / sizeof(T)) : 4; const uint32_t tile_size = blockDim.x * VALUES_PER_TILE; + const auto &src = stage.source; const void *raw_input = reinterpret_cast(stage.input_ptr); const PTypeTag ptype = stage.source_ptype; if (src.op_code == SourceOp::RUNEND) { - // Seed each thread's cursor with the run containing its first - // strided position. The RUNEND arm in source_op advances the - // cursor monotonically, so this avoids a full binary search on - // every element. const T *ends = reinterpret_cast(smem + src.params.runend.ends_smem_byte_offset); runend_cursors[threadIdx.x] = upper_bound(ends, src.params.runend.num_runs, @@ -301,10 +351,6 @@ __device__ void execute_output_stage(T *__restrict output, uint32_t chunk_len; const T *smem_src = nullptr; - // BITUNPACK uses smem scratch, so the outer loop advances one - // chunk at a time. LOAD, SEQUENCE, and RUNEND need no smem - // scratch, so chunk_len = block_len (single outer iteration); - // tiling happens in the inner tile_idx loop. if (src.op_code == SourceOp::BITUNPACK) { chunk_len = bitunpack_tile_len(stage, block_len, elem_idx); T *scratch = reinterpret_cast(smem + stage.smem_byte_offset); @@ -313,10 +359,10 @@ __device__ void execute_output_stage(T *__restrict output, block_start + elem_idx, chunk_len, src); - constexpr uint32_t FL_CHUNK = 1024; // FastLanes chunk size const uint32_t align = (block_start + elem_idx + src.params.bitunpack.element_offset) % FL_CHUNK; smem_src = scratch + align; - // Write barrier: all threads finished bitunpack, safe to read from scratch. + // Write barrier: all threads finished bitunpack (and any + // patches), safe to read from scratch. __syncthreads(); } else { chunk_len = block_len; @@ -337,7 +383,7 @@ __device__ void execute_output_stage(T *__restrict output, smem); for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) { - scalar_op(values, stage.scalar_ops[op], smem); + scalar_op(values, stage.scalar_ops[op], smem, tile_start); } #pragma unroll @@ -359,7 +405,7 @@ __device__ void execute_output_stage(T *__restrict output, source_op(&val, src, raw_input, ptype, smem_src, i, gpos, smem); for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) { - scalar_op(&val, stage.scalar_ops[op], smem); + scalar_op(&val, stage.scalar_ops[op], smem, gpos); } __stcs(&output[gpos], val); } @@ -392,24 +438,27 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) { const auto &src = stage.source; if (src.op_code == SourceOp::BITUNPACK) { + T *raw_smem = smem_out; bitunpack(reinterpret_cast(stage.input_ptr), smem_out, 0, stage.len, src); - smem_out += src.params.bitunpack.element_offset % SMEM_TILE_SIZE; // Write barrier: cooperative bitunpack finished, safe to read - // decoded elements in the scalar-op loop below. + // decoded elements below. __syncthreads(); + smem_out += src.params.bitunpack.element_offset % SMEM_TILE_SIZE; + if (stage.num_scalar_ops > 0) { - for (uint32_t i = threadIdx.x; i < stage.len; i += blockDim.x) { - T val = smem_out[i]; + for (uint32_t elem_idx = threadIdx.x; elem_idx < stage.len; elem_idx += blockDim.x) { + T val = smem_out[elem_idx]; for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) { - scalar_op(&val, stage.scalar_ops[op], smem); + scalar_op(&val, stage.scalar_ops[op], smem, elem_idx); } - smem_out[i] = val; + smem_out[elem_idx] = val; } // Write barrier: scalar ops applied in-place, smem region is // now fully populated for subsequent stages to read. __syncthreads(); } + } else { if (src.op_code == SourceOp::RUNEND) { // Seed each thread's cursor with the run containing its first @@ -421,13 +470,13 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) { upper_bound(ends, src.params.runend.num_runs, threadIdx.x + src.params.runend.offset); } const void *raw_input = reinterpret_cast(stage.input_ptr); - for (uint32_t i = threadIdx.x; i < stage.len; i += blockDim.x) { + for (uint32_t elem_idx = threadIdx.x; elem_idx < stage.len; elem_idx += blockDim.x) { T val; - source_op(&val, src, raw_input, stage.source_ptype, nullptr, 0, i, smem); + source_op(&val, src, raw_input, stage.source_ptype, nullptr, 0, elem_idx, smem); for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) { - scalar_op(&val, stage.scalar_ops[op], smem); + scalar_op(&val, stage.scalar_ops[op], smem, elem_idx); } - smem_out[i] = val; + smem_out[elem_idx] = val; } // Write barrier: smem region is fully populated for subsequent // stages to read. diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.h b/vortex-cuda/kernels/src/dynamic_dispatch.h index 95540c51581..9bba3f50a04 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.h +++ b/vortex-cuda/kernels/src/dynamic_dispatch.h @@ -103,11 +103,18 @@ extern "C" { #endif /// Parameters for source ops, which decode data into a stage's shared memory region. +/// +/// patches_ptr lives on the union variant that owns it (BitunpackParams, +/// AlpParams) — not per-stage — so the pointer is tied to its op. +/// Adding a u64 can grow the union and every ScalarOp/SourceOp read in the +/// tile loop; SourceParams was already 24 B (no change), ScalarParams grew +/// 8→16 B (no measurable impact — tile loop is compute-bound). union SourceParams { /// Unpack FastLanes bit-packed data. struct BitunpackParams { uint8_t bit_width; uint32_t element_offset; // Sub-byte offset + uint64_t patches_ptr; // device pointer to GPUPatches struct (0 = none) } bitunpack; /// Copy from global to shared memory. @@ -157,6 +164,7 @@ union ScalarParams { struct AlpParams { float f; float e; + uint64_t patches_ptr; // device pointer to GPUPatches struct (0 = none) } alp; /// Dictionary gather: use current value as index into decoded values in smem. @@ -193,6 +201,7 @@ struct PackedStage { uint32_t len; // number of elements this stage produces struct SourceOp source; + uint8_t num_scalar_ops; enum PTypeTag source_ptype; // PType produced by the source op }; @@ -220,11 +229,12 @@ struct __attribute__((aligned(8))) PlanHeader { /// change the type; the final output PType is given by the last scalar op's /// `output_ptype` (or `source_ptype` if there are no scalar ops). struct Stage { - uint64_t input_ptr; // encoded input in global memory - uint32_t smem_byte_offset; // byte offset within dynamic shared memory - uint32_t len; // elements produced - enum PTypeTag source_ptype; // PType produced by the source op - struct SourceOp source; // source decode op + uint64_t input_ptr; // encoded input in global memory + uint32_t smem_byte_offset; // byte offset within dynamic shared memory + uint32_t len; // elements produced + enum PTypeTag source_ptype; // PType produced by the source op + struct SourceOp source; // source decode op + uint8_t num_scalar_ops; // number of scalar ops const struct ScalarOp *scalar_ops; // scalar decode ops }; @@ -247,6 +257,7 @@ __device__ inline Stage parse_stage(const uint8_t *&cursor) { .len = packed_stage->len, .source_ptype = packed_stage->source_ptype, .source = packed_stage->source, + .num_scalar_ops = packed_stage->num_scalar_ops, .scalar_ops = ops, }; diff --git a/vortex-cuda/kernels/src/fastlanes_common.cuh b/vortex-cuda/kernels/src/fastlanes_common.cuh index 660a1c554f4..8536de26789 100644 --- a/vortex-cuda/kernels/src/fastlanes_common.cuh +++ b/vortex-cuda/kernels/src/fastlanes_common.cuh @@ -8,8 +8,25 @@ // FastLanes ordering array __constant__ int FL_ORDER[] = {0, 4, 2, 6, 1, 5, 3, 7}; +// FastLanes organises every 1024-element vector into a transposed layout +// of FL_LANES columns × (1024 / FL_LANES) rows. Each column is a "lane" +// that can be processed independently of every other lane, which is what +// makes all FastLanes encodings (FFOR, DELTA, RLE, ALP, …) fully +// data-parallel. One CUDA thread or one CPU SIMD lane handles one +// FastLanes lane. +// +// Paper: https://ir.cwi.nl/pub/35881/35881.pdf +// Repo: https://github.com/cwida/FastLanes + +/// FastLanes chunk size in elements. +constexpr uint32_t FL_CHUNK = 1024; + +/// Number of FastLanes lanes for element type T (1024 / bit-width). +template +constexpr uint32_t FL_LANES = FL_CHUNK / (sizeof(T) * 8); + // Compute the index in the FastLanes layout #define INDEX(row, lane) (FL_ORDER[row / 8] * 16 + (row % 8) * 128 + lane) // Create a mask with 'width' bits set -#define MASK(T, width) (((T)1 << width) - 1) \ No newline at end of file +#define MASK(T, width) (((T)1 << width) - 1) diff --git a/vortex-cuda/kernels/src/patches.cuh b/vortex-cuda/kernels/src/patches.cuh index 66a97b935e9..fa9ff18def9 100644 --- a/vortex-cuda/kernels/src/patches.cuh +++ b/vortex-cuda/kernels/src/patches.cuh @@ -3,6 +3,7 @@ #pragma once +#include "fastlanes_common.cuh" #include "patches.h" /// Load a chunk offset value, dispatching on the runtime type. @@ -21,8 +22,8 @@ __device__ inline uint32_t load_chunk_offset(const GPUPatches &patches, uint32_t } /// A single patch: a within-chunk index and its replacement value. -/// A sentinel patch has index == 1024, which can never match a valid -/// within-chunk position (0–1023). +/// A sentinel patch has index == FL_CHUNK, which can never match a valid +/// within-chunk position (0–FL_CHUNK-1). template struct Patch { uint16_t index; @@ -38,7 +39,7 @@ struct Patch { /// /// PatchesCursor cursor(patches, blockIdx.x, thread_idx, 32); /// auto patch = cursor.next(); -/// while (patch.index != 1024) { +/// while (patch.index != FL_CHUNK) { /// shared_out[patch.index] = patch.value; /// patch = cursor.next(); /// } @@ -89,15 +90,15 @@ public: // The iterator returns indices relative to the start of the chunk. // `chunk_base` is the index of the first element within a chunk, accounting // for the slice offset. - chunk_base = chunk * 1024 + patches.offset; - chunk_base -= min(chunk_base, patches.offset % 1024); + chunk_base = chunk * FL_CHUNK + patches.offset; + chunk_base -= min(chunk_base, patches.offset % FL_CHUNK); } /// Return the current patch (with within-chunk index) and advance, /// or a sentinel {1024, 0} if exhausted. __device__ Patch next() { if (remaining == 0) { - return {1024, T {}}; + return {FL_CHUNK, T {}}; } uint16_t within_chunk = static_cast(*indices - chunk_base); Patch patch = {within_chunk, *values}; @@ -110,6 +111,6 @@ public: private: const uint32_t *indices; const T *values; - uint8_t remaining; + uint32_t remaining; uint32_t chunk_base; }; diff --git a/vortex-cuda/src/bit_unpack_gen.rs b/vortex-cuda/src/bit_unpack_gen.rs index 8d5eda920cd..2482c0996b8 100644 --- a/vortex-cuda/src/bit_unpack_gen.rs +++ b/vortex-cuda/src/bit_unpack_gen.rs @@ -48,12 +48,7 @@ fn write_row(output: &mut impl Write, bits: usize, bit_width: usize, row: usize) /// loop. For all other bit widths, emits pre-computed per-row bit extraction /// with register-cached `src` words — identical to the original hand-unrolled /// codegen, preserving minimal memory loads and zero extra work. -fn generate_lane_decoder( - output: &mut impl Write, - bits: usize, - lanes: usize, - bit_width: usize, -) -> io::Result<()> { +fn generate_lane_decoder(output: &mut impl Write, bits: usize, bit_width: usize) -> io::Result<()> { if bit_width == 0 { write!( output, @@ -71,7 +66,7 @@ __device__ void _bit_unpack_{bits}_lane<0>(const uint{bits}_t *__restrict in, ui output, r#"template <> __device__ void _bit_unpack_{bits}_lane<{bit_width}>(const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, unsigned int lane) {{ - unsigned int LANE_COUNT = {lanes}; + constexpr unsigned int LANE_COUNT = FL_LANES; #pragma unroll for (int row = 0; row < {bits}; row++) {{ out[INDEX(row, lane)] = in[LANE_COUNT * row + lane] + reference; @@ -84,7 +79,7 @@ __device__ void _bit_unpack_{bits}_lane<{bit_width}>(const uint{bits}_t *__restr output, r#"template <> __device__ void _bit_unpack_{bits}_lane<{bit_width}>(const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, unsigned int lane) {{ - unsigned int LANE_COUNT = {lanes}; + constexpr unsigned int LANE_COUNT = FL_LANES; uint{bits}_t src; uint{bits}_t tmp; src = in[lane]; @@ -142,29 +137,25 @@ __device__ __noinline__ void bit_unpack_{bits}_lane( fn generate_device_kernel_template( output: &mut impl Write, bits: usize, - lanes: usize, thread_count: usize, ) -> io::Result<()> { - let per_thread_loop_count = lanes / thread_count; - let shared_copy_ncount = 1024 / thread_count; - write!( output, r#"template __device__ void _bit_unpack_{bits}_device(const uint{bits}_t *__restrict in, uint{bits}_t *__restrict out, uint{bits}_t reference, int thread_idx, GPUPatches& patches) {{ - __shared__ uint{bits}_t shared_out[1024]; + __shared__ uint{bits}_t shared_out[FL_CHUNK]; // Step 1: Unpack into shared memory #pragma unroll - for (int i = 0; i < {per_thread_loop_count}; i++) {{ - _bit_unpack_{bits}_lane(in, shared_out, reference, thread_idx * {per_thread_loop_count} + i); + for (int i = 0; i < FL_LANES / {thread_count}; i++) {{ + _bit_unpack_{bits}_lane(in, shared_out, reference, thread_idx * (FL_LANES / {thread_count}) + i); }} __syncwarp(); // Step 2: Apply patches to shared memory in parallel PatchesCursor cursor(patches, blockIdx.x, thread_idx, {thread_count}); auto patch = cursor.next(); - while (patch.index != 1024) {{ + while (patch.index != FL_CHUNK) {{ shared_out[patch.index] = patch.value; patch = cursor.next(); }} @@ -172,7 +163,7 @@ __device__ void _bit_unpack_{bits}_device(const uint{bits}_t *__restrict in, uin // Step 3: Copy to global memory #pragma unroll - for (int i = 0; i < {shared_copy_ncount}; i++) {{ + for (int i = 0; i < FL_CHUNK / {thread_count}; i++) {{ auto idx = i * {thread_count} + thread_idx; out[idx] = shared_out[idx]; }} @@ -194,8 +185,8 @@ fn generate_global_kernel( output, r#"extern "C" __global__ void {func_name}(const uint{bits}_t *__restrict full_in, uint{bits}_t *__restrict full_out, uint{bits}_t reference, GPUPatches patches) {{ int thread_idx = threadIdx.x; - auto in = full_in + (blockIdx.x * (128 * {bit_width} / sizeof(uint{bits}_t))); - auto out = full_out + (blockIdx.x * 1024); + auto in = full_in + (blockIdx.x * (FL_LANES * {bit_width})); + auto out = full_out + (blockIdx.x * FL_CHUNK); _bit_unpack_{bits}_device<{bit_width}>(in, out, reference, thread_idx, patches); }} "# @@ -210,7 +201,6 @@ fn generate_global_kernel( /// not pull in the 129 standalone bit-unpack kernel entry points. pub fn generate_cuda_unpack_lanes(output: &mut impl Write) -> io::Result<()> { let bits = T::T; - let lanes = T::LANES; write!( output, @@ -230,7 +220,7 @@ __device__ void _bit_unpack_{bits}_lane(const uint{bits}_t *__restrict in, uint{ // Lane-decoder template specializations (one per bit width). for bit_width in 0..=bits { - generate_lane_decoder(output, bits, lanes, bit_width)?; + generate_lane_decoder(output, bits, bit_width)?; writeln!(output)?; } @@ -250,7 +240,6 @@ pub fn generate_cuda_unpack_kernels( thread_count: usize, ) -> io::Result<()> { let bits = T::T; - let lanes = T::LANES; write!( output, @@ -262,7 +251,7 @@ pub fn generate_cuda_unpack_kernels( )?; // Device kernel template (written once, instantiated per bit width). - generate_device_kernel_template(output, bits, lanes, thread_count)?; + generate_device_kernel_template(output, bits, thread_count)?; writeln!(output)?; // Thin extern "C" global-kernel wrappers (one per bit width). diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 847fa39340d..46aa35d0465 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -313,6 +313,7 @@ impl SourceOp { bitunpack: SourceParams_BitunpackParams { bit_width, element_offset: u32::from(element_offset), + patches_ptr: 0, }, }, } @@ -393,7 +394,11 @@ impl ScalarOp { op_code: ScalarOp_ScalarOpCode_ALP, output_ptype: PTypeTag_PTYPE_F32, params: ScalarParams { - alp: ScalarParams_AlpParams { f, e }, + alp: ScalarParams_AlpParams { + f, + e, + patches_ptr: 0, + }, }, } } @@ -493,6 +498,7 @@ impl MaterializedPlan { #[cfg(test)] mod tests { + use std::ops::Range; use std::sync::Arc; use cudarc::driver::DevicePtr; @@ -515,6 +521,7 @@ mod tests { use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; + use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArrayExt; use vortex::encodings::runend::RunEnd; @@ -550,7 +557,7 @@ mod tests { fn dispatch_plan( array: &vortex::array::ArrayRef, - ctx: &CudaExecutionCtx, + ctx: &mut CudaExecutionCtx, ) -> VortexResult { match DispatchPlan::new(array)? { DispatchPlan::Fused(plan) => plan.materialize(ctx), @@ -778,8 +785,8 @@ mod tests { .collect(); let bp = bitpacked_array_u32(bit_width, len); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&bp.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -803,8 +810,8 @@ mod tests { let bp = bitpacked_array_u32(bit_width, len); let for_arr = FoR::try_new(bp.into_array(), Scalar::from(reference))?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&for_arr.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&for_arr.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -830,7 +837,7 @@ mod tests { let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); - let plan = dispatch_plan(&re.into_array(), &cuda_ctx)?; + let plan = dispatch_plan(&re.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -870,8 +877,8 @@ mod tests { let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&dict.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -907,8 +914,8 @@ mod tests { None, ); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&tree.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&tree.into_array(), &mut cuda_ctx)?; let actual = run_dispatch_plan_f32(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -940,8 +947,8 @@ mod tests { )?; let zz = ZigZag::try_new(bp.into_array())?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&zz.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&zz.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -970,7 +977,7 @@ mod tests { let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); let for_arr = FoR::try_new(re.into_array(), Scalar::from(reference))?; - let plan = dispatch_plan(&for_arr.into_array(), &cuda_ctx)?; + let plan = dispatch_plan(&for_arr.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -998,8 +1005,8 @@ mod tests { let dict = DictArray::try_new(codes_prim.into_array(), values_prim.into_array())?; let for_arr = FoR::try_new(dict.into_array(), Scalar::from(reference))?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&for_arr.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&for_arr.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -1030,8 +1037,8 @@ mod tests { let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); let dict = DictArray::try_new(codes_for.into_array(), values_prim.into_array())?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&dict.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -1059,8 +1066,8 @@ mod tests { let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&dict.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -1203,8 +1210,8 @@ mod tests { let expected: Vec = data[slice_start..slice_end].to_vec(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1258,8 +1265,8 @@ mod tests { let sliced = zz.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1308,8 +1315,8 @@ mod tests { .map(|&c| dict_values[c as usize]) .collect(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1357,8 +1364,8 @@ mod tests { let sliced = bp.into_array().slice(slice_start..slice_end)?; let expected: Vec = data[slice_start..slice_end].to_vec(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1410,8 +1417,8 @@ mod tests { let sliced = for_arr.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1475,8 +1482,8 @@ mod tests { let sliced = dict.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&sliced, &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&sliced, &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1507,8 +1514,8 @@ mod tests { let seq = Sequence::try_new_typed(base, multiplier, Nullability::NonNullable, len)?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&seq.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&seq.into_array(), &mut cuda_ctx)?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -1540,8 +1547,8 @@ mod tests { let seq = Sequence::try_new_typed(base, multiplier, Nullability::NonNullable, len)?; - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let plan = dispatch_plan(&seq.into_array(), &cuda_ctx)?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&seq.into_array(), &mut cuda_ctx)?; let actual_u32 = run_dynamic_dispatch_plan( &cuda_ctx, @@ -2127,4 +2134,438 @@ mod tests { assert_eq!(prim.validity()?.nullability(), Nullability::Nullable); Ok(()) } + + // --------------------------------------------------------------- + // Patch tests — fused dynamic dispatch with exception values + // --------------------------------------------------------------- + + #[rstest] + #[case::unsliced(3000, None)] + #[case::mid_slice(5000, Some(500..3500))] + #[case::start_slice(5000, Some(0..1000))] + #[case::chunk_aligned(5000, Some(1024..3000))] + #[crate::test] + fn test_bitpacked_with_patches( + #[case] len: usize, + #[case] slice_range: Option>, + ) -> VortexResult<()> { + let bit_width: u8 = 4; + let max_val = (1u32 << bit_width) - 1; + let values: Vec = (0..len) + .map(|i| { + if i % 100 == 0 { + 1000 + } else { + (i as u32) % (max_val + 1) + } + }) + .collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let (array, expected) = if let Some(range) = slice_range { + let sliced = bp.into_array().slice(range.clone())?; + (sliced, values[range].to_vec()) + } else { + (bp.into_array(), values) + }; + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&array, &mut cuda_ctx)?; + let actual = run_dynamic_dispatch_plan( + &cuda_ctx, + expected.len(), + &plan.dispatch_plan, + plan.shared_mem_bytes, + )?; + assert_eq!(actual, expected); + Ok(()) + } + + #[rstest] + #[case::unsliced(3000, None)] + #[case::mid_slice(5000, Some(500..3500))] + #[crate::test] + fn test_for_bitpacked_with_patches( + #[case] len: usize, + #[case] slice_range: Option>, + ) -> VortexResult<()> { + let bit_width: u8 = 6; + let reference = 42u32; + let max_val = (1u32 << bit_width) - 1; + let residuals: Vec = (0..len) + .map(|i| { + if i % 200 == 0 { + 500 + } else { + (i as u32) % (max_val + 1) + } + }) + .collect(); + let all_values: Vec = residuals.iter().map(|&v| v + reference).collect(); + + let prim = PrimitiveArray::new(Buffer::from(residuals), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + let for_arr = FoR::try_new(bp.into_array(), Scalar::from(reference))?; + + let (array, expected) = if let Some(range) = slice_range { + let sliced = for_arr.into_array().slice(range.clone())?; + (sliced, all_values[range].to_vec()) + } else { + (for_arr.into_array(), all_values) + }; + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&array, &mut cuda_ctx)?; + let actual = run_dynamic_dispatch_plan( + &cuda_ctx, + expected.len(), + &plan.dispatch_plan, + plan.shared_mem_bytes, + )?; + assert_eq!(actual, expected); + Ok(()) + } + + #[rstest] + #[case::unsliced(2000, None)] + #[case::mid_slice(5000, Some(100..4000))] + #[case::large_offset(5000, Some(1500..4500))] + #[crate::test] + fn test_alp_with_patches( + #[case] len: usize, + #[case] slice_range: Option>, + ) -> VortexResult<()> { + let mut values: Vec = (0..len).map(|i| (i as f32) * 1.1).collect(); + // Insert exception values that ALP can't encode. + values[0] = 99.9; + values[500] = std::f32::consts::PI; + values[1024] = std::f32::consts::E; + if len > 2048 { + values[2048] = std::f32::consts::LN_2; + } + if len > 3333 { + values[3333] = std::f32::consts::SQRT_2; + } + + let float_prim = PrimitiveArray::new(Buffer::from(values), NonNullable); + let encoded = alp_encode( + float_prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(); + + let (array, base_offset) = if let Some(ref range) = slice_range { + (encoded.slice(range.clone())?, range.start) + } else { + (encoded, 0) + }; + + // Decode on CPU as ground truth (accounts for ALP precision loss + patches). + let cpu_decoded = array + .clone() + .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; + let expected: Vec = cpu_decoded.as_slice::().to_vec(); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&array, &mut cuda_ctx)?; + let actual = run_dispatch_plan_f32( + &cuda_ctx, + expected.len(), + &plan.dispatch_plan, + plan.shared_mem_bytes, + )?; + for (i, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + a.to_bits() == e.to_bits(), + "mismatch at index {i} (original index {}): gpu={a} cpu={e} (bits: {:#010x} vs {:#010x})", + i + base_offset, + a.to_bits(), + e.to_bits(), + ); + } + Ok(()) + } + + // --------------------------------------------------------------- + // Additional patch tests — typed widths, edge cases, composites + // --------------------------------------------------------------- + + /// u8 BitPacked with patches (bit_width=3, patch values > 7). + #[crate::test] + async fn test_bitpacked_with_patches_u8() -> VortexResult<()> { + let bit_width: u8 = 3; + let len = 3000usize; + let max_val = (1u8 << bit_width) - 1; + let values: Vec = (0..len) + .map(|i| { + if i % 100 == 0 { + 200u8 + } else { + (i as u8) % (max_val + 1) + } + }) + .collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let expected_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + Ok(()) + } + + /// u16 BitPacked with patches (bit_width=6, patch values > 63). + #[crate::test] + async fn test_bitpacked_with_patches_u16() -> VortexResult<()> { + let bit_width: u8 = 6; + let len = 3000usize; + let max_val = (1u16 << bit_width) - 1; + let values: Vec = (0..len) + .map(|i| { + if i % 150 == 0 { + 5000u16 + } else { + (i as u16) % (max_val + 1) + } + }) + .collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let expected_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + Ok(()) + } + + /// u64 BitPacked with patches (bit_width=4, patch values > 15). + #[crate::test] + async fn test_bitpacked_with_patches_u64() -> VortexResult<()> { + let bit_width: u8 = 4; + let len = 3000usize; + let max_val = (1u64 << bit_width) - 1; + let values: Vec = (0..len) + .map(|i| { + if i % 200 == 0 { + 1_000_000u64 + } else { + (i as u64) % (max_val + 1) + } + }) + .collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let expected_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + Ok(()) + } + + /// Dict where codes are BitPacked u32 with patches exceeding the bit width. + #[crate::test] + fn test_dict_bitpacked_codes_with_patches() -> VortexResult<()> { + let dict_values: Vec = (0..256).map(|i| i * 1000 + 42).collect(); + let len = 3000; + let bit_width: u8 = 4; + let max_code = (1u32 << bit_width) - 1; + // Some codes exceed max_code (15), creating patches in the BitPacked codes. + let codes: Vec = (0..len) + .map(|i| { + if i % 100 == 0 { + 100u32 // exceeds max_code=15, becomes a patch + } else { + (i as u32) % (max_code + 1) + } + }) + .collect(); + let expected: Vec = codes.iter().map(|&c| dict_values[c as usize]).collect(); + + let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable); + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(codes_bp.patches().is_some(), "expected patches on codes"); + + let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); + let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx)?; + let actual = + run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; + assert_eq!(actual, expected); + Ok(()) + } + + /// Patches placed exactly at FastLanes chunk boundaries (1024-element chunks). + #[crate::test] + fn test_bitpacked_patches_at_chunk_boundaries() -> VortexResult<()> { + let len = 4096usize; + let bit_width: u8 = 4; + let max_val = (1u32 << bit_width) - 1; + let mut values: Vec = (0..len).map(|i| (i as u32) % (max_val + 1)).collect(); + // Place patches at FL chunk boundaries (1024 elements per chunk). + values[1023] = 1000; // end of chunk 0 + values[1024] = 2000; // start of chunk 1 + values[2047] = 3000; // end of chunk 1 + values[2048] = 4000; // start of chunk 2 + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx)?; + let actual = + run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; + assert_eq!(actual, values); + Ok(()) + } + + /// Large array (100k elements) spanning many blocks with sparse patches. + #[crate::test] + fn test_bitpacked_large_array_with_patches() -> VortexResult<()> { + let len = 100_000usize; + let bit_width: u8 = 6; + let max_val = (1u32 << bit_width) - 1; + let values: Vec = (0..len) + .map(|i| { + if i % 500 == 0 { + 1000 + } else { + (i as u32) % (max_val + 1) + } + }) + .collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx)?; + let actual = + run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; + assert_eq!(actual, values); + Ok(()) + } + + /// Nullable BitPacked with patches — validity must survive through fused + /// dispatch alongside patch application. + #[crate::test] + async fn test_nullable_bitpacked_with_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + + let len = 3000usize; + let bit_width: u8 = 4; + let max_val = (1u32 << bit_width) - 1; + + // Every 7th element is null; every 100th (non-null) element is a patch. + let values: Vec> = (0..len) + .map(|i| { + if i % 7 == 0 { + None + } else if i % 100 == 0 { + Some(1000u32) // exceeds max_val=15, becomes a patch + } else { + Some((i as u32) % (max_val + 1)) + } + }) + .collect(); + + let prim = PrimitiveArray::from_option_iter(values.iter().copied()); + let cpu = crate::canonicalize_cpu(prim.clone())?.into_array(); + + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let gpu = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + vortex::array::assert_arrays_eq!(cpu, gpu); + Ok(()) + } + + /// Extreme case: ALL values are patches (bit_width=1, every value > 1). + #[crate::test] + fn test_bitpacked_all_patches() -> VortexResult<()> { + let bit_width: u8 = 1; + let len = 2000usize; + // All values >= 2, so every single element exceeds max storable (1) and + // becomes a patch. + let values: Vec = (0..len).map(|i| (i as u32) + 2).collect(); + + let prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let bp = BitPacked::encode( + &prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!(bp.patches().is_some(), "expected patches"); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx)?; + let actual = + run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; + assert_eq!(actual, values); + Ok(()) + } } diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index bec93d4336f..1b63630906c 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -16,12 +16,14 @@ use vortex::array::arrays::Slice; use vortex::array::arrays::dict::DictArraySlotsExt; use vortex::array::arrays::slice::SliceArrayExt; use vortex::array::buffer::BufferHandle; +use vortex::array::patches::Patches; use vortex::array::validity::Validity; use vortex::dtype::PType; use vortex::encodings::alp::ALP; use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPArraySlotsExt; use vortex::encodings::alp::ALPFloat; +use vortex::encodings::alp::Exponents; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::encodings::fastlanes::FoR; @@ -45,6 +47,7 @@ use super::ptype_to_tag; use super::tag_to_ptype; use crate::CudaBufferExt; use crate::CudaExecutionCtx; +use crate::kernel::load_patches_sync; /// A plan whose source buffers have been copied to the device, ready for kernel launch. pub struct MaterializedPlan { @@ -69,10 +72,10 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { let id = array.encoding_id(); if id == ALP.id() { let arr = array.as_::(); - return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; + return arr.dtype().as_ptype() == PType::F32; } if id == BitPacked.id() { - return array.as_::().patches().is_none(); + return true; } if id == Dict.id() { let arr = array.as_::(); @@ -119,9 +122,17 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { } /// An unmaterialized stage: a source op, scalar ops, and optional source buffer reference. +/// +/// Patches are tied to their owning ops, mirroring the CUDA side where +/// `patches_ptr` lives on `BitunpackParams` / `AlpParams`: +/// - `source_patches` for the source op (BitPacked exceptions) +/// - Each scalar op carries its own `Option` (ALP exceptions) struct Stage { source: SourceOp, - scalar_ops: Vec, + /// Patches from the source op (e.g. BitPacked overflow exceptions). + source_patches: Option, + /// Scalar ops with optional per-op patches (e.g. ALP float exceptions). + scalar_ops: Vec<(ScalarOp, Option)>, /// Index into `FusedPlan::source_buffers`, or `None` /// for sources that don't read from a device buffer. source_buffer_index: Option, @@ -133,6 +144,7 @@ impl Stage { fn new(source: SourceOp, source_buffer_index: Option, source_ptype: PTypeTag) -> Self { Self { source, + source_patches: None, scalar_ops: vec![], source_buffer_index, source_ptype, @@ -240,7 +252,7 @@ impl DispatchPlan { /// - Validity is propagated from the root array to the output. Nullable /// arrays are supported, but Dict with nullable codes and RunEnd with /// nullable ends are rejected to guard against out-of-bounds access. - /// - `BitPackedArray` and `ALPArray` with patches are not supported. + /// - `BitPackedArray` and `ALPArray` with patches are supported. /// - Only f32 ALP is supported (kernel stores multipliers as `float`). pub fn new(array: &ArrayRef) -> VortexResult { if PType::try_from(array.dtype()).is_err() || !is_dyn_dispatch_compatible(array) { @@ -340,7 +352,7 @@ impl FusedPlan { } /// Copy source buffers to the device, producing a [`MaterializedPlan`]. - pub fn materialize(self, ctx: &CudaExecutionCtx) -> VortexResult { + pub fn materialize(self, ctx: &mut CudaExecutionCtx) -> VortexResult { let shared_mem_bytes = self.dynamic_shared_mem_bytes(); let mut device_buffers = Vec::new(); @@ -367,20 +379,37 @@ impl FusedPlan { // Byte offsets are passed directly to the C ABI — the kernel now // indexes shared memory by byte offset and casts to the correct type // using source_ptype / output_ptype. - let stages: Vec = self - .stages - .iter() - .map(|(stage, smem_byte_offset, len)| { - MaterializedStage::new( - resolve_ptr(stage), - *smem_byte_offset, - *len, - stage.source_ptype, - stage.source, - &stage.scalar_ops, - ) - }) - .collect(); + let mut stages: Vec = Vec::new(); + for (stage, smem_byte_offset, len) in &self.stages { + let mut source = stage.source; + + // Upload source patches (e.g. BitPacked exceptions). + if let Some(patches) = &stage.source_patches { + let (ptr, bufs) = load_patches_sync(patches, ctx)?; + source.params.bitunpack.patches_ptr = ptr; + device_buffers.extend(bufs); + } + + // Upload patches for each scalar op that carries them. + let mut scalar_ops: Vec = Vec::with_capacity(stage.scalar_ops.len()); + for (mut op, patches) in stage.scalar_ops.clone() { + if let Some(patches) = &patches { + let (ptr, bufs) = load_patches_sync(patches, ctx)?; + op.params.alp.patches_ptr = ptr; + device_buffers.extend(bufs); + } + scalar_ops.push(op); + } + + stages.push(MaterializedStage::new( + resolve_ptr(stage), + *smem_byte_offset, + *len, + stage.source_ptype, + source, + &scalar_ops, + )); + } Ok(MaterializedPlan { dispatch_plan: CudaDispatchPlan::new(stages, self.output_ptype), @@ -398,7 +427,7 @@ impl FusedPlan { pub fn materialize_with_subtrees( mut self, subtree_buffers: Vec, - ctx: &CudaExecutionCtx, + ctx: &mut CudaExecutionCtx, ) -> VortexResult { for (slot, buf) in zip_eq( self.source_buffers.iter_mut().filter(|s| s.is_none()), @@ -465,7 +494,8 @@ impl FusedPlan { /// SliceArray → resolve the slice via reduce/execute rules. /// /// When the plan builder encounters a `SliceArray`, it resolves the slice - /// by invoking the child's `reduce_parent`, `execute_parent`. + /// by invoking the child's `reduce_parent`. If that fails (e.g. ALP + /// doesn't implement it), we manually slice the child's sub-arrays. fn walk_slice( &mut self, array: ArrayRef, @@ -478,6 +508,26 @@ impl FusedPlan { return self.walk(reduced, pending_subtrees); } + // ALP doesn't implement reduce_parent — slice encoded child and + // patches manually (Patches::slice adjusts offsets for the range). + if child.encoding_id() == ALP.id() { + let alp = child.as_::(); + let offset = slice_arr.data().slice_range().start; + let len = array.len(); + let sliced_encoded = alp.encoded().clone().slice(offset..offset + len)?; + let sliced_patches = alp + .patches() + .map(|p| p.slice(offset..offset + len)) + .transpose()? + .flatten(); + return self.walk_alp_inner( + sliced_encoded, + sliced_patches, + alp.exponents(), + pending_subtrees, + ); + } + vortex_bail!( "Cannot resolve SliceArray wrapping {:?} in dynamic dispatch plan builder", child.encoding_id() @@ -498,20 +548,18 @@ impl FusedPlan { fn walk_bitpacked(&mut self, array: ArrayRef) -> VortexResult { let bp = array.as_::(); - if bp.patches().is_some() { - vortex_bail!("Dynamic dispatch does not support BitPackedArray with patches"); - } - let source_ptype = ptype_to_tag(PType::try_from(bp.dtype()).map_err(|_| { vortex_err!("BitPacked must have primitive dtype, got {:?}", bp.dtype()) })?); let buf_index = self.source_buffers.len(); self.source_buffers.push(Some(bp.packed().clone())); - Ok(Stage::new( + let mut stage = Stage::new( SourceOp::bitunpack(bp.bit_width(), bp.offset()), Some(buf_index), source_ptype, - )) + ); + stage.source_patches = bp.patches(); + Ok(stage) } fn walk_for( @@ -537,7 +585,7 @@ impl FusedPlan { .cast::()?; pipeline .scalar_ops - .push(ScalarOp::frame_of_ref(ref_u64, output_ptype)); + .push((ScalarOp::frame_of_ref(ref_u64, output_ptype), None)); Ok(pipeline) } @@ -553,7 +601,9 @@ impl FusedPlan { })?); let mut pipeline = self.walk(encoded, pending_subtrees)?; - pipeline.scalar_ops.push(ScalarOp::zigzag(output_ptype)); + pipeline + .scalar_ops + .push((ScalarOp::zigzag(output_ptype), None)); Ok(pipeline) } @@ -563,26 +613,29 @@ impl FusedPlan { pending_subtrees: &mut Vec, ) -> VortexResult { let alp = array.as_::(); + self.walk_alp_inner( + alp.encoded().clone(), + alp.patches(), + alp.exponents(), + pending_subtrees, + ) + } - if alp.patches().is_some() { - vortex_bail!("Dynamic dispatch does not support ALPArray with patches"); - } - - let ptype = alp.dtype().as_ptype(); - if ptype != PType::F32 { - vortex_bail!( - "Dynamic dispatch only supports f32 ALP, got {:?}", - alp.dtype() - ); - } - - let exponents = alp.exponents(); + /// Shared ALP logic for both `walk_alp` and `walk_slice` (Slice(ALP)). + fn walk_alp_inner( + &mut self, + encoded: ArrayRef, + patches: Option, + exponents: Exponents, + pending_subtrees: &mut Vec, + ) -> VortexResult { let alp_f = ::F10[exponents.f as usize]; let alp_e = ::IF10[exponents.e as usize]; - let encoded = alp.encoded().clone(); let mut pipeline = self.walk(encoded, pending_subtrees)?; - pipeline.scalar_ops.push(ScalarOp::alp(alp_f, alp_e)); + pipeline + .scalar_ops + .push((ScalarOp::alp(alp_f, alp_e), patches)); Ok(pipeline) } @@ -645,9 +698,9 @@ impl FusedPlan { }; // DICT scalar op: pass byte offset directly (C ABI uses byte offsets). // output_ptype is the values' ptype — DICT transforms codes → values. - pipeline.scalar_ops.push(ScalarOp::dict( - values_smem_byte_offset, - ptype_to_tag(values_ptype), + pipeline.scalar_ops.push(( + ScalarOp::dict(values_smem_byte_offset, ptype_to_tag(values_ptype)), + None, )); Ok(pipeline) } @@ -727,7 +780,7 @@ impl FusedPlan { let final_ptype = spec .scalar_ops .last() - .map(|op| op.output_ptype) + .map(|(op, _)| op.output_ptype) .unwrap_or(spec.source_ptype); let final_elem_bytes = tag_to_ptype(final_ptype).byte_width() as u32; let elem_bytes = final_elem_bytes.max(self.output_elem_bytes); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 1853557e809..0059d74fd5e 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -21,7 +21,6 @@ use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedDataParts; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; -use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use vortex::error::vortex_err; @@ -29,13 +28,10 @@ use crate::CudaBufferExt; use crate::CudaDeviceBuffer; use crate::executor::CudaExecute; use crate::executor::CudaExecutionCtx; -use crate::kernel::patches::gpu::ChunkOffsetType; -use crate::kernel::patches::gpu::ChunkOffsetType_CO_U8; -use crate::kernel::patches::gpu::ChunkOffsetType_CO_U16; use crate::kernel::patches::gpu::ChunkOffsetType_CO_U32; -use crate::kernel::patches::gpu::ChunkOffsetType_CO_U64; use crate::kernel::patches::gpu::GPUPatches; use crate::kernel::patches::types::load_patches; +use crate::kernel::patches::types::ptype_to_chunk_offset_type; /// CUDA decoder for bit-packed arrays. #[derive(Debug)] @@ -92,17 +88,6 @@ pub fn bitpacked_cuda_launch_config(output_width: usize, len: usize) -> VortexRe unsafe impl DeviceRepr for GPUPatches {} -/// Convert a PType to the corresponding ChunkOffsetType for GPU patches. -fn ptype_to_chunk_offset_type(ptype: vortex::dtype::PType) -> VortexResult { - match ptype { - vortex::dtype::PType::U8 => Ok(ChunkOffsetType_CO_U8), - vortex::dtype::PType::U16 => Ok(ChunkOffsetType_CO_U16), - vortex::dtype::PType::U32 => Ok(ChunkOffsetType_CO_U32), - vortex::dtype::PType::U64 => Ok(ChunkOffsetType_CO_U64), - _ => vortex_bail!("Invalid PType for chunk_offsets: {:?}", ptype), - } -} - #[instrument(skip_all)] pub(crate) async fn decode_bitpacked( array: BitPackedArray, diff --git a/vortex-cuda/src/kernel/mod.rs b/vortex-cuda/src/kernel/mod.rs index 68f40a15005..2c9dfe0d033 100644 --- a/vortex-cuda/src/kernel/mod.rs +++ b/vortex-cuda/src/kernel/mod.rs @@ -34,6 +34,7 @@ pub use encodings::ZstdKernelPrep; pub use encodings::zstd_kernel_prepare; pub(crate) use encodings::*; pub(crate) use filter::FilterExecutor; +pub(crate) use patches::types::load_patches_sync; pub(crate) use slice::SliceExecutor; use crate::CudaKernelEvents; diff --git a/vortex-cuda/src/kernel/patches/types.rs b/vortex-cuda/src/kernel/patches/types.rs index 350c5210e8a..e8e3ce5bfc6 100644 --- a/vortex-cuda/src/kernel/patches/types.rs +++ b/vortex-cuda/src/kernel/patches/types.rs @@ -3,18 +3,29 @@ //! GPU patches loading for fused exception patching during bit-unpacking. +use std::mem::size_of; + use num_traits::ToPrimitive; use vortex::array::buffer::BufferHandle; +use vortex::buffer::Alignment; use vortex::buffer::Buffer; use vortex::buffer::BufferMut; +use vortex::buffer::ByteBufferMut; use vortex::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::patches::Patches; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use crate::CudaBufferExt; use crate::CudaExecutionCtx; use crate::executor::CudaArrayExt; +use crate::kernel::patches::gpu::ChunkOffsetType; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U8; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U16; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U32; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U64; +use crate::kernel::patches::gpu::GPUPatches; /// A set of device-resident patches. pub struct DevicePatches { @@ -28,7 +39,7 @@ pub struct DevicePatches { pub(crate) n_chunks: usize, } -/// Load patches for GPU use. +/// Load patches for GPU use (async). /// /// # Errors /// @@ -40,17 +51,16 @@ pub(crate) async fn load_patches( ) -> VortexResult { let offset = patches.offset(); let offset_within_chunk = patches.offset_within_chunk().unwrap_or_default(); - let array_len = patches.array_len(); - // Get or compute chunk_offsets let Some(co) = patches.chunk_offsets() else { vortex_bail!("cannot execute_cuda for patched BitPacked array without chunk_offsets") }; - let (chunk_offsets, chunk_offset_ptype) = { + let (chunk_offsets, chunk_offset_ptype, n_chunks) = { let co_canonical = co.clone().execute_cuda(ctx).await?.into_primitive(); let ptype = co_canonical.ptype(); - (co_canonical.buffer_handle().clone(), ptype) + let len = co_canonical.len(); + (co_canonical.buffer_handle().clone(), ptype, len) }; // Load indices - must be converted to u32 for GPU use @@ -93,7 +103,6 @@ pub(crate) async fn load_patches( let values = ctx.ensure_on_device(values.buffer_handle().clone()).await?; let num_patches = patches.num_patches(); - let n_chunks = array_len.div_ceil(1024); Ok(DevicePatches { chunk_offsets, @@ -107,6 +116,76 @@ pub(crate) async fn load_patches( }) } +/// Convert a PType to the corresponding `ChunkOffsetType` for GPU patches. +pub(crate) fn ptype_to_chunk_offset_type(ptype: PType) -> VortexResult { + match ptype { + PType::U8 => Ok(ChunkOffsetType_CO_U8), + PType::U16 => Ok(ChunkOffsetType_CO_U16), + PType::U32 => Ok(ChunkOffsetType_CO_U32), + PType::U64 => Ok(ChunkOffsetType_CO_U64), + _ => vortex_bail!("Invalid PType for chunk_offsets: {:?}", ptype), + } +} + +/// Build a [`GPUPatches`] struct from [`DevicePatches`], serialize it to +/// bytes, and upload to the device. Returns the device pointer and a buffer +/// handle that must be kept alive for the kernel launch. +fn build_gpu_patches( + dp: &DevicePatches, + ctx: &CudaExecutionCtx, +) -> VortexResult<(BufferHandle, u64)> { + // Zero-initialize to avoid uninitialized padding bytes (e.g. between + // chunk_offset_type and indices) which would be UB when serialized. + let mut gpu_patches: GPUPatches = unsafe { std::mem::zeroed() }; + gpu_patches.chunk_offsets = dp.chunk_offsets.cuda_device_ptr()? as _; + gpu_patches.chunk_offset_type = ptype_to_chunk_offset_type(dp.chunk_offset_ptype)?; + gpu_patches.indices = dp.indices.cuda_device_ptr()? as _; + gpu_patches.values = dp.values.cuda_device_ptr()? as _; + #[expect(clippy::cast_possible_truncation)] + { + gpu_patches.offset = dp.offset as u32; + gpu_patches.offset_within_chunk = dp.offset_within_chunk as u32; + gpu_patches.num_patches = dp.num_patches as u32; + // n_chunks must match the chunk_offsets array length, not array_len / 1024. + // When patches are sliced, chunk_offsets is sliced to only include chunks + // overlapping the slice range — matching the CPU's patch_chunk which uses + // chunk_offsets_slice.len(). + gpu_patches.n_chunks = dp.n_chunks as u32; + } + + let bytes = unsafe { + std::slice::from_raw_parts( + std::ptr::from_ref(&gpu_patches).cast::(), + size_of::(), + ) + }; + let mut buf = + ByteBufferMut::with_capacity_aligned(size_of::(), Alignment::of::()); + buf.extend_from_slice(bytes); + let gpu_buf = ctx.ensure_on_device_sync(BufferHandle::new_host(buf.freeze()))?; + let ptr = gpu_buf.cuda_device_ptr()?; + Ok((gpu_buf, ptr)) +} + +/// Sync wrapper: load patches via [`load_patches`] (blocking), then build and +/// upload a [`GPUPatches`] struct. Returns the device pointer and all buffer +/// handles that must be kept alive for the kernel launch. +pub(crate) fn load_patches_sync( + patches: &Patches, + ctx: &mut CudaExecutionCtx, +) -> VortexResult<(u64, Vec)> { + let device_patches = futures::executor::block_on(load_patches(patches, ctx))?; + let (gpu_buf, ptr) = build_gpu_patches(&device_patches, ctx)?; + + let DevicePatches { + chunk_offsets, + indices, + values, + .. + } = device_patches; + Ok((ptr, vec![chunk_offsets, indices, values, gpu_buf])) +} + #[cfg(test)] mod tests { use vortex_array::IntoArray; From 2fe2a69a25f1b7b46f0875ba96cd64bb31c90ca3 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 22 Apr 2026 15:39:08 +0100 Subject: [PATCH 159/250] Use layout file splits when DF re-partitions individual files (#7591) ## Summary Instead of just splitting files arbitrarily, align it with split layouts to make better use of Vortex's internal pruning and other behaviors. --------- Signed-off-by: Adam Gutglick --- vortex-datafusion/src/persistent/opener.rs | 197 +++++++++++++++------ vortex-datafusion/src/persistent/source.rs | 5 + vortex-datafusion/src/persistent/tests.rs | 177 ++++++++++++++++++ 3 files changed, 321 insertions(+), 58 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index d93a849e75a..0719b023881 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -10,7 +10,6 @@ use datafusion_common::DataFusionError; use datafusion_common::Result as DFResult; use datafusion_common::ScalarValue; use datafusion_common::exec_datafusion_err; -use datafusion_datasource::FileRange; use datafusion_datasource::PartitionedFile; use datafusion_datasource::TableSchema; use datafusion_datasource::file_stream::FileOpenFuture; @@ -30,16 +29,19 @@ use futures::FutureExt; use futures::StreamExt; use futures::TryStreamExt; use futures::stream; +use itertools::Itertools; use object_store::path::Path; use tracing::Instrument; -use vortex::array::ArrayRef; use vortex::array::VortexSessionExecute; use vortex::array::arrow::ArrowArrayExecutor; +use vortex::dtype::FieldMask; use vortex::error::VortexError; +use vortex::error::VortexExpect; use vortex::file::OpenOptionsSessionExt; use vortex::io::InstrumentedReadAt; use vortex::layout::LayoutReader; use vortex::layout::scan::scan_builder::ScanBuilder; +use vortex::layout::scan::split_by::SplitBy; use vortex::metrics::Label; use vortex::metrics::MetricsRegistry; use vortex::session::VortexSession; @@ -88,6 +90,8 @@ pub(crate) struct VortexOpener { /// To save on the overhead of reparsing FlatBuffers and rebuilding the layout tree, we cache /// a file reader the first time we read a file. pub layout_readers: Arc>>, + /// Shared full-file natural split ranges keyed by file path. + pub natural_split_ranges: Arc]>>>, /// Whether the query has output ordering specified pub has_output_ordering: bool, @@ -123,6 +127,7 @@ impl FileOpener for VortexOpener { let batch_size = self.batch_size; let limit = self.limit; let layout_reader = Arc::clone(&self.layout_readers); + let natural_split_ranges = Arc::clone(&self.natural_split_ranges); let has_output_ordering = self.has_output_ordering; let scan_concurrency = self.scan_concurrency; @@ -302,6 +307,12 @@ impl FileOpener for VortexOpener { } }; + let natural_split_ranges = natural_split_ranges_for_file( + natural_split_ranges.as_ref(), + &file.object_meta.location, + &layout_reader, + )?; + let mut scan_builder = ScanBuilder::new(session.clone(), layout_reader); if let Some(extensions) = file.extensions @@ -311,12 +322,22 @@ impl FileOpener for VortexOpener { } if let Some(file_range) = file.range { - scan_builder = apply_byte_range( - file_range, + let byte_range = Range { + start: u64::try_from(file_range.start) + .map_err(|_| exec_datafusion_err!("Vortex file range start is negative"))?, + end: u64::try_from(file_range.end) + .map_err(|_| exec_datafusion_err!("Vortex file range end is negative"))?, + }; + + let Some(row_range) = split_aligned_row_range( + byte_range, file.object_meta.size, - vxf.row_count(), - scan_builder, - ); + natural_split_ranges.as_ref(), + ) else { + return Ok(stream::empty().boxed()); + }; + + scan_builder = scan_builder.with_row_range(row_range); } let filter = filter @@ -421,33 +442,74 @@ impl FileOpener for VortexOpener { } } -/// If the file has a [`FileRange`], we translate it into a row range in the file for the scan. -fn apply_byte_range( - file_range: FileRange, - total_size: u64, - row_count: u64, - scan_builder: ScanBuilder, -) -> ScanBuilder { - let row_range = byte_range_to_row_range( - file_range.start as u64..file_range.end as u64, - row_count, - total_size, - ); - - scan_builder.with_row_range(row_range) +fn natural_split_ranges_for_file( + natural_split_ranges: &DashMap]>>, + path: &Path, + layout_reader: &Arc, +) -> DFResult]>> { + if let Some(split_ranges) = natural_split_ranges.get(path) { + return Ok(Arc::clone(split_ranges.value())); + } + + let split_ranges = compute_natural_split_ranges(layout_reader.as_ref())?; + + match natural_split_ranges.entry(path.clone()) { + Entry::Occupied(entry) => Ok(Arc::clone(entry.get())), + Entry::Vacant(entry) => { + entry.insert(Arc::clone(&split_ranges)); + Ok(split_ranges) + } + } } -fn byte_range_to_row_range(byte_range: Range, row_count: u64, total_size: u64) -> Range { - debug_assert!(row_count > 0); // Asserted by an early exit check in VortexOpener::open +fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult]>> { + let row_count = layout_reader.row_count(); + let row_range = 0..row_count; + let split_points: Vec<_> = SplitBy::Layout + .splits(layout_reader, &row_range, &[FieldMask::All]) + .map_err(|e| exec_datafusion_err!("Failed to compute Vortex natural splits: {e}"))? + .into_iter() + .tuple_windows() + .map(|(s, e)| s..e) + .collect::>(); + + Ok(split_points.into()) +} - let average_row = total_size / row_count; - assert!(average_row > 0, "A row must always have at least one byte"); +/// Translate a DataFusion byte range to the contiguous natural split ranges it owns. +fn split_aligned_row_range( + byte_range: Range, + total_size: u64, + split_ranges: &[Range], +) -> Option> { + if byte_range.start >= byte_range.end { + return None; + } - let start_row = byte_range.start / average_row; - let end_row = byte_range.end / average_row; + let row_count = split_ranges.last().map(|split| split.end)?; + if row_count == 0 { + return None; + } + + let mut owned_splits = split_ranges.iter().filter(|split_range| { + let midpoint_byte = split_midpoint_to_byte(split_range, row_count, total_size); + byte_range.contains(&midpoint_byte) + }); - // We take the min here as `end_row` might overshoot - start_row..u64::min(row_count, end_row) + let first_split = owned_splits.next()?; + let mut row_range = first_split.start..first_split.end; + for split_range in owned_splits { + row_range.end = split_range.end; + } + + Some(row_range) +} + +fn split_midpoint_to_byte(split_range: &Range, row_count: u64, total_size: u64) -> u64 { + let midpoint_row = split_range.start + (split_range.end - split_range.start) / 2; + let midpoint_byte = (u128::from(midpoint_row) * u128::from(total_size)) / u128::from(row_count); + + u64::try_from(midpoint_byte).vortex_expect("midpoint byte projection should fit into u64") } #[cfg(test)] @@ -500,43 +562,56 @@ mod tests { static SESSION: LazyLock = LazyLock::new(VortexSession::default); #[rstest] - #[case(0..100, 100, 100, 0..100)] - #[case(0..105, 100, 105, 0..100)] - #[case(0..50, 100, 105, 0..50)] - #[case(50..105, 100, 105, 50..100)] - #[case(0..1, 4, 8, 0..0)] - #[case(1..8, 4, 8, 0..4)] - fn test_range_translation( + #[case(0..3, 10, vec![0..2, 2..5, 5..10], Some(0..2))] + #[case(3..7, 10, vec![0..2, 2..5, 5..10], Some(2..5))] + #[case(1..8, 10, vec![0..1, 1..9, 9..10], Some(1..9))] + #[case(1..4, 16, vec![0..1, 1..2, 2..3, 3..4], None)] + fn test_split_aligned_row_range( #[case] byte_range: Range, - #[case] row_count: u64, #[case] total_size: u64, - #[case] expected: Range, + #[case] split_ranges: Vec>, + #[case] expected: Option>, ) { assert_eq!( - byte_range_to_row_range(byte_range, row_count, total_size), + split_aligned_row_range(byte_range, total_size, &split_ranges), expected ); } #[test] - fn test_consecutive_ranges() { - let row_count = 100; - let total_size = 429; - let bytes_a = 0..143; - let bytes_b = 143..286; - let bytes_c = 286..429; - - let rows_a = byte_range_to_row_range(bytes_a, row_count, total_size); - let rows_b = byte_range_to_row_range(bytes_b, row_count, total_size); - let rows_c = byte_range_to_row_range(bytes_c, row_count, total_size); - - assert_eq!(rows_a.end - rows_a.start, 35); - assert_eq!(rows_b.end - rows_b.start, 36); - assert_eq!(rows_c.end - rows_c.start, 29); - - assert_eq!(rows_a.start, 0); - assert_eq!(rows_c.end, 100); - for (left, right) in [rows_a, rows_b, rows_c].iter().tuple_windows() { + fn test_split_aligned_ranges_cover_splits_exactly_once() { + let split_ranges = vec![0..1, 1..4, 4..10, 10..13]; + let byte_ranges = [0..4, 4..8, 8..12, 12..16]; + + let assigned = byte_ranges + .into_iter() + .filter_map(|byte_range| split_aligned_row_range(byte_range, 16, &split_ranges)) + .collect::>(); + + assert_eq!(assigned, vec![0..4, 4..10, 10..13]); + assert_eq!( + assigned + .iter() + .map(|range| range.end - range.start) + .sum::(), + 13 + ); + + let split_starts = split_ranges + .iter() + .map(|range| range.start) + .collect::>(); + let split_ends = split_ranges + .iter() + .map(|range| range.end) + .collect::>(); + + for range in &assigned { + assert!(split_starts.contains(&range.start)); + assert!(split_ends.contains(&range.end)); + } + + for (left, right) in assigned.iter().tuple_windows() { assert_eq!(left.end, right.start); } } @@ -577,6 +652,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, @@ -650,7 +726,7 @@ mod tests { let file_schema = data_batch.schema(); // Parallel scans may attach a byte range even for empty files; the - // opener must not call byte_range_to_row_range when the row_count is 0. + // opener must return early before attempting split-aligned translation. let file = PartitionedFile::new_with_range(file_path.to_string(), file_size, 0, file_size as i64); @@ -708,6 +784,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, @@ -794,6 +871,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, @@ -950,6 +1028,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, @@ -1009,6 +1088,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, @@ -1212,6 +1292,7 @@ mod tests { limit: None, metrics_registry: Arc::new(DefaultMetricsRegistry::default()), layout_readers: Default::default(), + natural_split_ranges: Default::default(), has_output_ordering: false, expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: None, diff --git a/vortex-datafusion/src/persistent/source.rs b/vortex-datafusion/src/persistent/source.rs index fdd6244eaf5..211170c6607 100644 --- a/vortex-datafusion/src/persistent/source.rs +++ b/vortex-datafusion/src/persistent/source.rs @@ -3,6 +3,7 @@ use std::any::Any; use std::fmt::Formatter; +use std::ops::Range; use std::sync::Arc; use std::sync::Weak; @@ -185,6 +186,8 @@ pub struct VortexSource { /// /// Sharing the readers allows us to only read every layout once from the file, even across partitions. layout_readers: Arc>>, + /// Shared full-file natural split ranges keyed by path. + natural_split_ranges: Arc]>>>, expression_convertor: Arc, pub(crate) vortex_reader_factory: Option>, vx_metrics_registry: Arc, @@ -216,6 +219,7 @@ impl VortexSource { batch_size: None, _unused_df_metrics: Default::default(), layout_readers: Arc::new(DashMap::default()), + natural_split_ranges: Arc::new(DashMap::default()), expression_convertor: Arc::new(DefaultExpressionConvertor::default()), vortex_reader_factory: None, vx_metrics_registry: Arc::new(DefaultMetricsRegistry::default()), @@ -333,6 +337,7 @@ impl FileSource for VortexSource { limit: base_config.limit.map(|l| l as u64), metrics_registry: Arc::clone(&self.vx_metrics_registry), layout_readers: Arc::clone(&self.layout_readers), + natural_split_ranges: Arc::clone(&self.natural_split_ranges), has_output_ordering: !base_config.output_ordering.is_empty(), expression_convertor: Arc::new(DefaultExpressionConvertor::default()), file_metadata_cache: self.file_metadata_cache.clone(), diff --git a/vortex-datafusion/src/persistent/tests.rs b/vortex-datafusion/src/persistent/tests.rs index 09c56d4fbe4..194184e904e 100644 --- a/vortex-datafusion/src/persistent/tests.rs +++ b/vortex-datafusion/src/persistent/tests.rs @@ -3,9 +3,18 @@ use std::sync::Arc; +use anyhow::anyhow; +use datafusion::arrow::array::Int32Array; use datafusion::arrow::util::pretty::pretty_format_batches; +use datafusion::datasource::provider::DefaultTableFactory; +use datafusion::execution::SessionStateBuilder; +use datafusion::prelude::SessionConfig; +use datafusion::prelude::SessionContext; +use datafusion_common::GetExt; use datafusion_physical_plan::display::DisplayableExecutionPlan; use insta::assert_snapshot; +use object_store::ObjectStore; +use object_store::memory::InMemory; use rstest::rstest; use vortex::VortexSessionDefault; use vortex::array::IntoArray; @@ -13,14 +22,79 @@ use vortex::array::arrays::ChunkedArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinArray; use vortex::array::validity::Validity; +use vortex::buffer::Buffer; use vortex::buffer::buffer; +use vortex::file::OpenOptionsSessionExt; use vortex::file::WriteOptionsSessionExt; use vortex::io::VortexWrite; +use vortex::io::object_store::ObjectStoreReadAt; use vortex::io::object_store::ObjectStoreWrite; +use vortex::layout::LayoutStrategy; +use vortex::layout::layouts::chunked::writer::ChunkedLayoutStrategy; +use vortex::layout::layouts::flat::writer::FlatLayoutStrategy; +use vortex::layout::layouts::table::TableStrategy; use vortex::session::VortexSession; +use crate::VortexFormatFactory; use crate::common_tests::TestSessionContext; +fn make_session( + object_store: Arc, + repartition_file_scans: bool, +) -> SessionContext { + let factory = Arc::new(VortexFormatFactory::new()); + + let config = SessionConfig::new() + .with_target_partitions(4) + .with_repartition_file_scans(repartition_file_scans) + .with_repartition_file_min_size(0); + let mut state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_table_factory( + factory.get_ext().to_uppercase(), + Arc::new(DefaultTableFactory::new()), + ) + .with_object_store(&url::Url::try_from("file://").unwrap(), object_store); + + if let Some(file_formats) = state.file_formats() { + file_formats.push(factory as _); + } + + SessionContext::new_with_state(state.build()).enable_url_table() +} + +async fn count_query_partitions(ctx: &SessionContext, sql: &str) -> anyhow::Result { + let explain = ctx.sql(&format!("EXPLAIN {sql}")).await?.collect().await?; + let plan = pretty_format_batches(&explain)?.to_string(); + let marker = "DataSourceExec: file_groups={"; + let start = plan + .find(marker) + .ok_or_else(|| anyhow!("EXPLAIN plan did not contain a DataSourceExec"))? + + marker.len(); + let partitions = plan[start..] + .chars() + .take_while(|ch| ch.is_ascii_digit()) + .collect::(); + + Ok(partitions.parse()?) +} + +fn batch_values(batches: &[datafusion::arrow::array::RecordBatch]) -> Vec { + let mut values = Vec::with_capacity(batches.iter().map(|batch| batch.num_rows()).sum()); + + for batch in batches { + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("value column should be Int32"); + values.extend(array.values().iter().copied()); + } + + values +} + #[rstest] #[tokio::test] async fn test_query_file(#[values(Some(1), None)] limit: Option) -> anyhow::Result<()> { @@ -255,3 +329,106 @@ async fn doc_example() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_repartitioned_scan_matches_non_repartitioned_for_uneven_splits() -> anyhow::Result<()> +{ + let store = Arc::new(InMemory::new()) as _; + let session = VortexSession::default(); + let path = object_store::path::Path::parse("/split-aligned-repartition.vortex")?; + + let chunk_1_len = 2_000; + let chunk_2_len = 5_000; + let chunk_3_len = 6_000; + let row_count = chunk_1_len + chunk_2_len + chunk_3_len; + + let chunk_1 = StructArray::try_new( + ["value"].into(), + vec![Buffer::from_iter(0_i32..chunk_1_len).into_array()], + usize::try_from(chunk_1_len)?, + Validity::NonNullable, + )?; + let chunk_2 = StructArray::try_new( + ["value"].into(), + vec![Buffer::from_iter(chunk_1_len..(chunk_1_len + chunk_2_len)).into_array()], + usize::try_from(chunk_2_len)?, + Validity::NonNullable, + )?; + let chunk_3 = StructArray::try_new( + ["value"].into(), + vec![Buffer::from_iter((chunk_1_len + chunk_2_len)..row_count).into_array()], + usize::try_from(chunk_3_len)?, + Validity::NonNullable, + )?; + let table = ChunkedArray::from_iter([ + chunk_1.into_array(), + chunk_2.into_array(), + chunk_3.into_array(), + ]) + .into_array(); + let flat: Arc = Arc::new(FlatLayoutStrategy::default()); + let strategy: Arc = Arc::new(TableStrategy::new( + Arc::clone(&flat), + Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default())), + )); + + let mut writer = ObjectStoreWrite::new(Arc::clone(&store), &path).await?; + let summary = session + .write_options() + .with_strategy(strategy) + .write(&mut writer, table.into_array().to_array_stream()) + .await?; + writer.shutdown().await?; + + let reader = Arc::new(ObjectStoreReadAt::new( + Arc::clone(&store), + path.clone(), + vortex::io::runtime::Handle::find().expect("tokio runtime should be available in tests"), + )); + let vxf = session + .open_options() + .with_file_size(summary.size()) + .open_read(reader) + .await?; + let split_ranges = vxf.splits()?; + let split_lengths = split_ranges + .iter() + .map(|range| range.end - range.start) + .collect::>(); + + assert!(split_ranges.len() > 1); + assert!( + split_lengths + .windows(2) + .any(|window| window[0] != window[1]) + ); + + let serial_ctx = make_session(Arc::clone(&store), false); + let repartitioned_ctx = make_session(Arc::clone(&store), true); + let repartitioned_partitions = count_query_partitions( + &repartitioned_ctx, + "SELECT value FROM '/split-aligned-repartition.vortex'", + ) + .await?; + + assert!(repartitioned_partitions > 1); + + let serial = serial_ctx + .sql("SELECT value FROM '/split-aligned-repartition.vortex' ORDER BY value") + .await? + .collect() + .await?; + let repartitioned = repartitioned_ctx + .sql("SELECT value FROM '/split-aligned-repartition.vortex' ORDER BY value") + .await? + .collect() + .await?; + let serial_values = batch_values(&serial); + let repartitioned_values = batch_values(&repartitioned); + let expected = (0_i32..row_count).collect::>(); + + assert_eq!(serial_values, expected); + assert_eq!(repartitioned_values, serial_values); + + Ok(()) +} From 2e6e1633d527a9815ad9994fbee7639d46e5509a Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 22 Apr 2026 16:13:56 +0100 Subject: [PATCH 160/250] Skip nightly lance benchmark (#7594) It seems like I've added `lance` accidentally, and it break the nightly TPC-H run on NVMe. From looking at the data I don't think we ever ran Lance in that configuration. Signed-off-by: Adam Gutglick --- .github/workflows/nightly-bench.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 9fcb7e45a87..f2689bc70a4 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -39,7 +39,6 @@ jobs: "develop_targets": [ {"engine": "datafusion", "format": "parquet"}, {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, {"engine": "duckdb", "format": "parquet"}, {"engine": "duckdb", "format": "vortex"} ], From 9e261cca7b37d01e1c093660dd074adfba8f1687 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 22 Apr 2026 16:24:25 +0100 Subject: [PATCH 161/250] Try and make the bot-review-action not hang and/or just fail (#7593) 1. Only enforce the action for bots 2. Trigger action on any PR event, so the status won't get "lost" Signed-off-by: Adam Gutglick --- .github/workflows/approvals.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/approvals.yml b/.github/workflows/approvals.yml index 74ee32d9b42..7448b02ecd2 100644 --- a/.github/workflows/approvals.yml +++ b/.github/workflows/approvals.yml @@ -3,6 +3,9 @@ name: PR Approval Check on: pull_request_review: types: [submitted, dismissed] + pull_request: + branches: + - "develop" jobs: check-approvals: @@ -41,7 +44,7 @@ jobs: console.log(`One-approval bot allowlist: ${oneApprovalBotAuthors.has(authorLogin)}`); console.log(`Approvals: ${approvalCount} / ${required} required`); - if (approvalCount < required) { + if (isBot && (approvalCount < required)) { core.setFailed( `This PR needs ${required} human approval(s) but has ${approvalCount}. ` + `(Author is ${isBot ? 'a bot' : 'human'})` From 30042ee26d64d85c75c4be21d87218ba3130435e Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Wed, 22 Apr 2026 12:20:59 -0400 Subject: [PATCH 162/250] data-parallel patched ALP standalone kernel (#7576) ## Summary Follow up to #7440 This changes ALP execution on CUDA. Previously, we'd execute two kernel passes: one to perform ALP decoding to global memory, and a second to apply patches. This PR works similarly to prior work to push patching into the decoding kernel for unpacking. We assign a FastLanes 1024-element block to each warp (32 threads), and then perform decoding and patching in a single kernel pass. ## Testing Unit tests were added to check for simple and edge cases (multi-chunk, mix of chunks with/without patches) Signed-off-by: Andrew Duffy --- vortex-cuda/kernels/src/alp.cu | 112 +++++-- vortex-cuda/src/kernel/encodings/alp.rs | 279 +++++++++++++++--- vortex-cuda/src/kernel/encodings/bitpacked.rs | 32 +- vortex-cuda/src/kernel/patches/mod.rs | 65 ++++ 4 files changed, 390 insertions(+), 98 deletions(-) diff --git a/vortex-cuda/kernels/src/alp.cu b/vortex-cuda/kernels/src/alp.cu index e34285d9d9b..95502131257 100644 --- a/vortex-cuda/kernels/src/alp.cu +++ b/vortex-cuda/kernels/src/alp.cu @@ -1,36 +1,92 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "scalar_kernel.cuh" - -// ALP (Adaptive Lossless floating-Point) decode operation. -// Converts integers to floats by multiplying by precomputed exponent factors. -// Formula: decoded = (float)encoded * f * e -// Where f = F10[exponents.f] and e = IF10[exponents.e] are passed directly. -template -struct AlpOp { - FloatT f; // F10[exponents.f] - power of 10 - FloatT e; // IF10[exponents.e] - inverse power of 10 - - __device__ inline FloatT operator()(EncodedT value) const { - return static_cast(value) * f * e; +#include "patches.cuh" + +// ALP (Adaptive Lossless floating-Point) decode: out[i] = (FloatT)in[i] * f * e. +// +// Each block processes one 1024-element chunk cooperatively and applies patches +// into shared memory before writing to global memory, mirroring the strategy +// used by bit_unpack. f = F10[exponents.f], e = IF10[exponents.e]. +// +// The cast from EncT to FloatT must preserve ALP's lossless contract: f32 is +// only encoded as i32, and f64 is only encoded as i64. The i64 → double cast +// is lossless for all values ALP can produce. +template +__device__ void alp_device(const EncT *__restrict in, + FloatT *__restrict out, + FloatT f, + FloatT e, + uint64_t array_len, + int thread_idx, + GPUPatches &patches) { + constexpr int ThreadCount = 32; + // ThreadCount == 32 (one warp) is baked into this kernel: + // - __syncwarp() below is only sufficient because all threads live in one warp. + // - per_thread must evenly divide 1024 so the unrolled loops cover the chunk. + static_assert(ThreadCount == 32, "alp kernel requires exactly one warp per block"); + static_assert(1024 % ThreadCount == 0, "ThreadCount must evenly divide 1024"); + __shared__ FloatT shared_out[1024]; + + constexpr int per_thread = 1024 / ThreadCount; + uint64_t chunk_base = static_cast(blockIdx.x) * 1024; + + // Step 1: decode the chunk into shared memory. The tail block is bounds-checked; + // all interior blocks take the fast path with no per-element branch. + if (chunk_base + 1024 <= array_len) { +#pragma unroll + for (int i = 0; i < per_thread; i++) { + int idx = i * ThreadCount + thread_idx; + shared_out[idx] = static_cast(in[idx]) * f * e; + } + } else { +#pragma unroll + for (int i = 0; i < per_thread; i++) { + int idx = i * ThreadCount + thread_idx; + uint64_t global_idx = chunk_base + static_cast(idx); + if (global_idx < array_len) { + shared_out[idx] = static_cast(in[idx]) * f * e; + } else { + shared_out[idx] = FloatT {}; + } + } } -}; - -// Macro to generate ALP kernel for each type combination. -// Input is integer (encoded), output is float (decoded). -#define GENERATE_ALP_KERNEL(enc_suffix, float_suffix, EncType, FloatType) \ - extern "C" __global__ void alp_##enc_suffix##_##float_suffix(const EncType *__restrict encoded, \ - FloatType *__restrict decoded, \ - FloatType f, \ - FloatType e, \ - uint64_t array_len) { \ - scalar_kernel(encoded, decoded, array_len, AlpOp {f, e}); \ + __syncwarp(); + + // Step 2: apply patches in parallel across the warp. + PatchesCursor cursor(patches, blockIdx.x, thread_idx, static_cast(ThreadCount)); + auto patch = cursor.next(); + while (patch.index != 1024) { + shared_out[patch.index] = patch.value; + patch = cursor.next(); } + __syncwarp(); -// f32 variants (ALP for f32 encodes as i32 or i64) -GENERATE_ALP_KERNEL(i32, f32, int32_t, float) -GENERATE_ALP_KERNEL(i64, f32, int64_t, float) +// Step 3: coalesced write-out of the full 1024-element chunk. The caller +// allocates `full_out` rounded up to a multiple of 1024, so every block +// writes entirely within bounds. Positions in `[array_len, rounded_len)` +// of the tail chunk hold don't-care values; the caller slices them off. +#pragma unroll + for (int i = 0; i < per_thread; i++) { + int idx = i * ThreadCount + thread_idx; + out[idx] = shared_out[idx]; + } +} + +#define GENERATE_ALP_KERNEL(enc_suffix, float_suffix, EncT, FloatT) \ + extern "C" __global__ void alp_##enc_suffix##_##float_suffix##_32t(const EncT *__restrict full_in, \ + FloatT *__restrict full_out, \ + FloatT f, \ + FloatT e, \ + uint64_t array_len, \ + GPUPatches patches) { \ + int thread_idx = threadIdx.x; \ + auto in = full_in + (blockIdx.x * 1024); \ + auto out = full_out + (blockIdx.x * 1024); \ + alp_device(in, out, f, e, array_len, thread_idx, patches); \ + } -// f64 variants (ALP for f64 encodes as i64) +// The only ALPInt bindings produced by the encoder are (f32, i32) and (f64, i64). +// i64 → double is lossless; i32 → float is lossless for all values ALP emits. +GENERATE_ALP_KERNEL(i32, f32, int32_t, float) GENERATE_ALP_KERNEL(i64, f64, int64_t, double) diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 068b07e7978..77dd4dc73b4 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -2,10 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::fmt::Debug; -use std::sync::Arc; use async_trait::async_trait; use cudarc::driver::DeviceRepr; +use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use tracing::instrument; use vortex::array::ArrayRef; @@ -13,7 +13,7 @@ use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::primitive::PrimitiveDataParts; use vortex::array::buffer::BufferHandle; -use vortex::array::match_each_unsigned_integer_ptype; +use vortex::array::buffer::DeviceBufferExt; use vortex::dtype::NativePType; use vortex::encodings::alp::ALP; use vortex::encodings::alp::ALPArray; @@ -30,7 +30,8 @@ use crate::CudaDeviceBuffer; use crate::executor::CudaArrayExt; use crate::executor::CudaExecute; use crate::executor::CudaExecutionCtx; -use crate::kernel::patches::execute_patches; +use crate::kernel::patches::build_gpu_patches; +use crate::kernel::patches::types::load_patches; /// CUDA decoder for ALP (Adaptive Lossless floating-Point) decompression. #[derive(Debug)] @@ -54,6 +55,11 @@ impl CudaExecute for ALPExecutor { } } +/// Threads per block. 32 threads × 32 elements = 1024 element chunks for both +/// (f32, i32) and (f64, i64). f64 uses 8 KB of shared memory per block. +const ALP_THREADS_PER_BLOCK: u32 = 32; + +#[instrument(skip_all)] async fn decode_alp(array: ALPArray, ctx: &mut CudaExecutionCtx) -> VortexResult where A: ALPFloat + NativePType + DeviceRepr + Send + Sync + 'static, @@ -67,7 +73,7 @@ where let f: A = A::F10[exponents.f as usize]; let e: A = A::IF10[exponents.e as usize]; - // Execute child and copy to device + // Execute child and copy to device. let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); let PrimitiveDataParts { @@ -75,42 +81,53 @@ where } = primitive.into_data_parts(); let device_input = ctx.ensure_on_device(buffer).await?; - - // Get CUDA view of input let input_view = device_input.cuda_view::()?; - // Allocate output buffer - let output_slice = ctx.device_alloc::(array_len)?; + // Allocate output rounded up to a full chunk: the fused kernel writes a + // whole 1024-element chunk per block, and we slice off any padding below. + let output_slice = ctx.device_alloc::(array_len.next_multiple_of(1024))?; let output_buf = CudaDeviceBuffer::new(output_slice); let output_view = output_buf.as_view::(); - let array_len_u64 = array_len as u64; - - // Load kernel function - let kernel_ptypes = [A::ALPInt::PTYPE, A::PTYPE]; - let cuda_function = ctx.load_function("alp", &kernel_ptypes)?; + // Patch validity does not need to be scattered: the ALP encoder strips null + // positions from the exception list, so patches only exist at valid + // positions. load_patches additionally rejects patches without + // chunk_offsets (required by the fused kernel's PatchesCursor). + let device_patches = if let Some(patches) = array.patches() { + Some(load_patches(&patches, ctx).await?) + } else { + None + }; + let patches_arg = build_gpu_patches(device_patches.as_ref())?; + + // Load the kernel: alp_{enc}_{float}_32t + let enc_suffix = A::ALPInt::PTYPE.to_string(); + let float_suffix = A::PTYPE.to_string(); + let cuda_function = ctx + .load_function_with_suffixes("alp", &[enc_suffix.as_str(), float_suffix.as_str(), "32t"])?; + + let num_blocks = u32::try_from(array_len.div_ceil(1024))?; + let config = LaunchConfig { + grid_dim: (num_blocks, 1, 1), + block_dim: (ALP_THREADS_PER_BLOCK, 1, 1), + shared_mem_bytes: 0, + }; - ctx.launch_kernel(&cuda_function, array_len, |args| { + let array_len_u64 = array_len as u64; + ctx.launch_kernel_config(&cuda_function, config, array_len, |args| { args.arg(&input_view) .arg(&output_view) .arg(&f) .arg(&e) - .arg(&array_len_u64); + .arg(&array_len_u64) + .arg(&patches_arg); })?; - // Check if there are any patches to decode here. Patch validity does not - // need to be scattered: the ALP encoder strips null positions from the - // exception list, so patches only exist at valid positions. execute_patches - // additionally guards against nullable patch values at runtime. - let output_buf = if let Some(patches) = array.patches() { - match_each_unsigned_integer_ptype!(patches.indices_ptype()?, |I| { - execute_patches::(patches.clone(), output_buf, ctx).await? - }) - } else { - output_buf - }; + // Synchronize so the device patches buffers remain alive for the kernel. + ctx.synchronize_stream()?; + drop(device_patches); - let output_handle = BufferHandle::new_device(Arc::new(output_buf)); + let output_handle = BufferHandle::new_device(output_buf.slice_typed::(0..array_len)); Ok(Canonical::Primitive(PrimitiveArray::from_buffer_handle( output_handle, A::PTYPE, @@ -137,30 +154,38 @@ mod tests { use super::*; use crate::CanonicalCudaExt; + use crate::canonicalize_cpu; use crate::executor::CudaArrayExt; use crate::session::CudaSession; + /// Irrational values ALP cannot encode losslessly, guaranteed to land + /// in the exception list on round-trip through `alp_encode`. + const UNENCODABLE: f64 = std::f64::consts::PI; + const UNENCODABLE_F32: f32 = std::f32::consts::PI; + + /// Small manually-constructed ALP array with patches. Exercises the + /// custom-construction path (as opposed to going through `alp_encode`). + /// Patches must carry `chunk_offsets` — the fused kernel requires them. #[crate::test] async fn test_cuda_alp_decompression_f32() -> VortexResult<()> { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context"); - // Create encoded values (what ALP would produce) // For f32 with exponents (e=0, f=2): decoded = encoded * F10[2] * IF10[0] // = encoded * 100.0 * 1.0 - // So encoded value of 100 -> decoded 10000.0 + // Encoded value of 100 -> decoded 10000.0. let encoded_data: Vec = vec![100, 200, 300, 400, 500]; - let exponents = Exponents { e: 0, f: 2 }; // multiply by 100 + let exponents = Exponents { e: 0, f: 2 }; - // Patches + // One chunk holds all 5 elements. chunk_offsets[0] = 0: chunk 0's + // patches begin at patch index 0. let patches = Patches::new( 5, 0, PrimitiveArray::new(buffer![0u32, 4u32], Validity::NonNullable).into_array(), PrimitiveArray::new(buffer![0.0f32, 999f32], Validity::NonNullable).into_array(), - None, - ) - .unwrap(); + Some(PrimitiveArray::new(buffer![0u32], Validity::NonNullable).into_array()), + )?; let alp_array = ALP::try_new( PrimitiveArray::new(Buffer::from(encoded_data.clone()), Validity::NonNullable) @@ -169,7 +194,7 @@ mod tests { Some(patches), )?; - let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = ALPExecutor .execute(alp_array.into_array(), &mut cuda_ctx) @@ -194,14 +219,14 @@ mod tests { .vortex_expect("failed to create execution context"); // Values that will produce ALP exceptions at non-null positions. - // Nulls at positions 1 and 3; the exception at position 4 (1.23456) - // can't be encoded losslessly by ALP. + // Nulls at positions 1 and 3; the exception at position 4 can't be + // encoded losslessly by ALP. let values: Vec> = vec![ Some(1.0), None, Some(2.0), None, - Some(1.23456), + Some(UNENCODABLE_F32), Some(3.0), Some(4.0), Some(5.0), @@ -213,7 +238,7 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), )?; - let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = alp_array .into_array() @@ -244,7 +269,181 @@ mod tests { &mut LEGACY_SESSION.create_execution_ctx(), )?; - let cpu_result = crate::canonicalize_cpu(alp_array.clone())?.into_array(); + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); + + let gpu_result = alp_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } + + /// Multi-chunk ALP (> 1024 elements) with patches in chunks 0 and 2 but + /// none in chunk 1. Exercises the `PatchesCursor` branch where a + /// non-trailing chunk has `chunk_offsets[c] == chunk_offsets[c+1]` + /// (zero patches) via the offset math rather than the NULL sentinel. + #[crate::test] + async fn test_cuda_alp_multi_chunk_sparse_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // 3072 values (3 chunks). Inject exceptions (values ALP can't encode + // losslessly) only in chunks 0 and 2; chunk 1 stays exception-free so + // its cursor slice is empty despite patches existing in the array. + let values: Buffer = (0u32..3072) + .map(|i| { + if matches!(i, 0 | 100 | 1023 | 3071) { + UNENCODABLE_F32 + } else { + i as f32 + } + }) + .collect(); + let prim = PrimitiveArray::new(values, Validity::NonNullable); + let alp_array = alp_encode( + prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!( + alp_array.patches().is_some(), + "expected patches from ALP exceptions" + ); + + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); + + let gpu_result = alp_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } + + /// Multi-chunk f64 decode with patches distributed across chunks. The f64 + /// path (i64 → double) is otherwise only covered by the partial-tail case, + /// so this guards the fast-path for the (i64, f64) kernel variant. + #[crate::test] + async fn test_cuda_alp_f64_multi_chunk_with_patches() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // 3072 values (3 chunks). Sprinkle exceptions into each chunk. + let values: Buffer = (0u32..3072) + .map(|i| { + if matches!(i, 0 | 500 | 1024 | 1500 | 2048 | 3071) { + UNENCODABLE + } else { + i as f64 + } + }) + .collect(); + let prim = PrimitiveArray::new(values, Validity::NonNullable); + let alp_array = alp_encode( + prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!( + alp_array.patches().is_some(), + "expected patches from ALP exceptions" + ); + + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); + + let gpu_result = alp_array + .into_array() + .execute_cuda(&mut cuda_ctx) + .await? + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } + + /// Single chunk with more patches than threads per block (32). Forces + /// `PatchesCursor` to split patches across multiple threads, exercising + /// the per-thread ceil-division and clamping math that no other test hits + /// (existing tests have ≤ 6 patches per chunk). + #[crate::test] + async fn test_cuda_alp_dense_patches_single_chunk() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + // Build a 1024-element ALP array manually with exactly 40 patches + // all in the single chunk. 40 > 32 forces cursor division (each of + // the first 20 threads handles 2 patches; remaining threads idle). + const LEN: i32 = 1024; + const NUM_PATCHES: u32 = 40; + + let exponents = Exponents { e: 0, f: 2 }; + let encoded: Buffer = (0i32..LEN).map(|i| i * 100).collect(); + + let patch_indices: Buffer = (0u32..NUM_PATCHES).collect(); + let patch_values: Buffer = (0..NUM_PATCHES).map(|i| i as f32 * 0.125 + 0.5).collect(); + + let patches = Patches::new( + LEN as usize, + 0, + PrimitiveArray::new(patch_indices, Validity::NonNullable).into_array(), + PrimitiveArray::new(patch_values, Validity::NonNullable).into_array(), + Some(PrimitiveArray::new(buffer![0u32], Validity::NonNullable).into_array()), + )?; + + let alp_array = ALP::try_new( + PrimitiveArray::new(encoded, Validity::NonNullable).into_array(), + exponents, + Some(patches), + )?; + + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); + + let gpu_result = ALPExecutor + .execute(alp_array.into_array(), &mut cuda_ctx) + .await + .vortex_expect("GPU decompression failed") + .into_host() + .await? + .into_array(); + + assert_arrays_eq!(cpu_result, gpu_result); + Ok(()) + } + + /// Tail-chunk bounds check: an array whose length is not a multiple of + /// 1024 forces the kernel's tail-block path to bounds-check its decode + /// loop. Includes a patch in the tail. + #[crate::test] + async fn test_cuda_alp_partial_tail_chunk() -> VortexResult<()> { + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + .vortex_expect("failed to create execution context"); + + let values: Buffer = (0u32..1500) + .map(|i| if i == 1400 { UNENCODABLE } else { i as f64 }) + .collect(); + let prim = PrimitiveArray::new(values, Validity::NonNullable); + let alp_array = alp_encode( + prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; + assert!( + alp_array.patches().is_some(), + "expected patches from ALP exceptions" + ); + + let cpu_result = canonicalize_cpu(alp_array.clone())?.into_array(); let gpu_result = alp_array .into_array() diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 0059d74fd5e..29b56436feb 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -28,10 +28,8 @@ use crate::CudaBufferExt; use crate::CudaDeviceBuffer; use crate::executor::CudaExecute; use crate::executor::CudaExecutionCtx; -use crate::kernel::patches::gpu::ChunkOffsetType_CO_U32; -use crate::kernel::patches::gpu::GPUPatches; +use crate::kernel::patches::build_gpu_patches; use crate::kernel::patches::types::load_patches; -use crate::kernel::patches::types::ptype_to_chunk_offset_type; /// CUDA decoder for bit-packed arrays. #[derive(Debug)] @@ -86,8 +84,6 @@ pub fn bitpacked_cuda_launch_config(output_width: usize, len: usize) -> VortexRe }) } -unsafe impl DeviceRepr for GPUPatches {} - #[instrument(skip_all)] pub(crate) async fn decode_bitpacked( array: BitPackedArray, @@ -131,31 +127,7 @@ where None }; - #[expect(clippy::cast_possible_truncation)] - let patches_arg = if let Some(p) = &device_patches { - GPUPatches { - chunk_offsets: p.chunk_offsets.cuda_device_ptr()? as _, - chunk_offset_type: ptype_to_chunk_offset_type(p.chunk_offset_ptype)?, - indices: p.indices.cuda_device_ptr()? as _, - values: p.values.cuda_device_ptr()? as _, - offset: p.offset as u32, - offset_within_chunk: p.offset_within_chunk as u32, - num_patches: p.num_patches as u32, - n_chunks: p.n_chunks as u32, - } - } else { - // NULL chunk_offsets signals no patches to the kernel - GPUPatches { - chunk_offsets: std::ptr::null_mut(), - chunk_offset_type: ChunkOffsetType_CO_U32, - indices: std::ptr::null_mut(), - values: std::ptr::null_mut(), - offset: 0, - offset_within_chunk: 0, - num_patches: 0, - n_chunks: 0, - } - }; + let patches_arg = build_gpu_patches(device_patches.as_ref())?; ctx.launch_kernel_config(&cuda_function, config, len, |args| { args.arg(&input_view) diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 07cd51c3f01..6075b664f72 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -17,14 +17,79 @@ use vortex::array::patches::Patches; use vortex::array::validity::Validity; use vortex::dtype::NativePType; use vortex::error::VortexResult; +use vortex::error::vortex_bail; use vortex::error::vortex_ensure; use crate::CudaBufferExt; use crate::CudaDeviceBuffer; use crate::CudaExecutionCtx; use crate::executor::CudaArrayExt; +use crate::kernel::patches::gpu::ChunkOffsetType; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U8; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U16; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U32; +use crate::kernel::patches::gpu::ChunkOffsetType_CO_U64; +use crate::kernel::patches::gpu::GPUPatches; +use crate::kernel::patches::types::DevicePatches; + +// Safe because `GPUPatches` contains only raw pointers, POD integers, and an enum. +unsafe impl DeviceRepr for GPUPatches {} + +impl GPUPatches { + /// Sentinel value passed to kernels when no patches are present. A NULL + /// `chunk_offsets` pointer is the signal `PatchesCursor` checks for. + pub(crate) const NULL_PATCHES: Self = Self { + chunk_offsets: std::ptr::null_mut(), + chunk_offset_type: ChunkOffsetType_CO_U32, + indices: std::ptr::null_mut(), + values: std::ptr::null_mut(), + offset: 0, + offset_within_chunk: 0, + num_patches: 0, + n_chunks: 0, + }; +} + +/// Convert a [`PType`] to the corresponding [`ChunkOffsetType`] for GPU patches. +fn ptype_to_chunk_offset_type(ptype: vortex::dtype::PType) -> VortexResult { + match ptype { + vortex::dtype::PType::U8 => Ok(ChunkOffsetType_CO_U8), + vortex::dtype::PType::U16 => Ok(ChunkOffsetType_CO_U16), + vortex::dtype::PType::U32 => Ok(ChunkOffsetType_CO_U32), + vortex::dtype::PType::U64 => Ok(ChunkOffsetType_CO_U64), + _ => vortex_bail!("Invalid PType for chunk_offsets: {:?}", ptype), + } +} + +/// Build a [`GPUPatches`] kernel argument from optional device-resident patches. +/// +/// When `device_patches` is `None`, returns a sentinel value whose NULL +/// `chunk_offsets` signals "no patches" to the kernel. +pub(crate) fn build_gpu_patches( + device_patches: Option<&DevicePatches>, +) -> VortexResult { + #[expect(clippy::cast_possible_truncation)] + match device_patches { + Some(p) => Ok(GPUPatches { + chunk_offsets: p.chunk_offsets.cuda_device_ptr()? as _, + chunk_offset_type: ptype_to_chunk_offset_type(p.chunk_offset_ptype)?, + indices: p.indices.cuda_device_ptr()? as _, + values: p.values.cuda_device_ptr()? as _, + offset: p.offset as u32, + offset_within_chunk: p.offset_within_chunk as u32, + num_patches: p.num_patches as u32, + n_chunks: p.n_chunks as u32, + }), + None => Ok(GPUPatches::NULL_PATCHES), + } +} /// Apply a set of patches in-place onto a [`CudaDeviceBuffer`] holding `ValuesT`. +/// +/// Naive scatter kernel. Kept as a reusable fallback for encoders that cannot +/// use the chunk-based fused patching path (e.g., where `chunk_offsets` are +/// unavailable); no production caller uses it today. +#[allow(dead_code)] #[instrument(skip_all)] pub(crate) async fn execute_patches< ValuesT: NativePType + DeviceRepr, From 8f7f6d3874cdfd9086b0b6a1fdb4e88fd6ffddd8 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 22 Apr 2026 18:26:32 +0100 Subject: [PATCH 163/250] Bump msrv and rust-toolchain to 1.91 (and lance for benchmarks) (#7595) We're two major version back on our lance comparison (which is about 2 months of changes), but it also seems like more and more dependencies break because we're on 1.90. libs.rs says 68% of crates.io requests ask for this stable version or newer, not sure what was our bar in the past but I figured its worth at least checking it out. I've reviewed the changes and none of them seem substantial, they are all tests and some lints that were improved and don't warn anymore. --------- Signed-off-by: Adam Gutglick --- Cargo.lock | 910 ++++++------------ Cargo.toml | 2 +- benchmarks/lance-bench/Cargo.toml | 4 +- benchmarks/vector-search-bench/src/ingest.rs | 9 +- encodings/datetime-parts/src/canonical.rs | 7 +- .../src/bitpacking/array/bitpack_compress.rs | 33 +- .../bitpacking/array/bitpack_decompress.rs | 39 +- .../src/delta/array/delta_compress.rs | 4 +- .../fastlanes/src/for/array/for_compress.rs | 4 +- encodings/fastlanes/src/rle/array/mod.rs | 54 +- .../fastlanes/src/rle/array/rle_compress.rs | 26 +- encodings/parquet-variant/src/array.rs | 34 +- encodings/parquet-variant/src/kernel.rs | 10 +- encodings/parquet-variant/src/operations.rs | 88 +- encodings/runend/src/arrow.rs | 12 +- encodings/runend/src/compute/filter.rs | 2 +- encodings/runend/src/decompress_bool.rs | 9 +- encodings/sparse/src/canonical.rs | 91 +- encodings/zigzag/src/array.rs | 7 +- encodings/zigzag/src/compute/mod.rs | 4 +- rust-toolchain.toml | 2 +- .../src/aggregate_fn/fns/is_sorted/mod.rs | 15 +- vortex-array/src/arrays/chunked/array.rs | 27 +- .../src/arrays/constant/vtable/canonical.rs | 11 +- .../src/arrays/dict/compute/min_max.rs | 91 +- vortex-array/src/arrays/listview/rebuild.rs | 54 +- .../src/arrays/listview/tests/basic.rs | 22 +- vortex-array/src/arrays/masked/tests.rs | 2 +- .../src/arrays/masked/vtable/canonical.rs | 64 +- .../src/arrays/patched/compute/compare.rs | 9 +- vortex-array/src/arrow/executor/dictionary.rs | 28 +- vortex-array/src/arrow/executor/run_end.rs | 39 +- vortex-array/src/extension/uuid/vtable.rs | 6 +- vortex-array/src/scalar/typed_view/utf8.rs | 2 +- vortex-array/src/scalar_fn/fns/case_when.rs | 8 +- vortex-bench/src/polarsignals/benchmark.rs | 2 +- .../src/statpopgen/statpopgen_benchmark.rs | 2 +- vortex-bench/src/tpcds/tpcds_benchmark.rs | 2 +- vortex-bench/src/tpch/benchmark.rs | 2 +- vortex-cuda/src/dynamic_dispatch/mod.rs | 36 +- vortex-duckdb/src/exporter/dict.rs | 65 +- vortex-duckdb/src/exporter/varbinview.rs | 6 +- vortex-duckdb/src/multi_file.rs | 37 +- vortex-layout/src/scan/scan_builder.rs | 2 +- vortex-metrics/src/histogram.rs | 1 - vortex-metrics/src/timer.rs | 1 - .../encodings/turboquant/tests/roundtrip.rs | 12 +- .../src/scalar_fns/cosine_similarity.rs | 28 +- vortex-tensor/src/scalar_fns/inner_product.rs | 28 +- vortex-tensor/src/scalar_fns/l2_denorm.rs | 13 +- vortex-tensor/src/scalar_fns/l2_norm.rs | 12 +- vortex-tensor/src/utils.rs | 8 +- vortex-web/crate/Cargo.toml | 2 +- 53 files changed, 841 insertions(+), 1147 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 945d35e3e08..87d6ae125f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1880,6 +1880,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -2094,39 +2104,39 @@ dependencies = [ [[package]] name = "datafusion" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" dependencies = [ "arrow 57.3.0", "arrow-schema 57.3.0", "async-trait", "bytes", "chrono", - "datafusion-catalog 51.0.0", - "datafusion-catalog-listing 51.0.0", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-datasource-arrow 51.0.0", - "datafusion-datasource-csv 51.0.0", - "datafusion-datasource-json 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-functions-aggregate 51.0.0", - "datafusion-functions-nested 51.0.0", - "datafusion-functions-table 51.0.0", - "datafusion-functions-window 51.0.0", - "datafusion-optimizer 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-optimizer 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", - "datafusion-sql 51.0.0", + "datafusion-catalog 52.5.0", + "datafusion-catalog-listing 52.5.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-datasource-arrow 52.5.0", + "datafusion-datasource-csv 52.5.0", + "datafusion-datasource-json 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-functions-aggregate 52.5.0", + "datafusion-functions-nested 52.5.0", + "datafusion-functions-table 52.5.0", + "datafusion-functions-window 52.5.0", + "datafusion-optimizer 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-adapter 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-optimizer 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", + "datafusion-sql 52.5.0", "futures", "itertools 0.14.0", "log", @@ -2134,7 +2144,6 @@ dependencies = [ "parking_lot", "rand 0.9.2", "regex", - "rstest", "sqlparser 0.59.0", "tempfile", "tokio", @@ -2226,21 +2235,21 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" dependencies = [ "arrow 57.3.0", "async-trait", "dashmap", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", "futures", "itertools 0.14.0", "log", @@ -2276,26 +2285,25 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" dependencies = [ "arrow 57.3.0", "async-trait", - "datafusion-catalog 51.0.0", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-catalog 52.5.0", + "datafusion-common 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-adapter 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", "futures", "itertools 0.14.0", "log", "object_store 0.12.5", - "tokio", ] [[package]] @@ -2323,16 +2331,16 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", "arrow-ipc 57.3.0", "chrono", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", "libc", "log", @@ -2371,9 +2379,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f" dependencies = [ "futures", "log", @@ -2393,23 +2401,23 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" dependencies = [ "arrow 57.3.0", "async-trait", "bytes", "chrono", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-adapter 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", "futures", "glob", "itertools 0.14.0", @@ -2457,22 +2465,22 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" dependencies = [ "arrow 57.3.0", "arrow-ipc 57.3.0", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", "futures", "itertools 0.14.0", "object_store 0.12.5", @@ -2525,21 +2533,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" dependencies = [ "arrow 57.3.0", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", "futures", "object_store 0.12.5", "regex", @@ -2571,21 +2579,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" dependencies = [ "arrow 57.3.0", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-session 52.5.0", "futures", "object_store 0.12.5", "tokio", @@ -2647,9 +2655,9 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a" [[package]] name = "datafusion-doc" @@ -2659,15 +2667,16 @@ checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" [[package]] name = "datafusion-execution" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" dependencies = [ "arrow 57.3.0", "async-trait", + "chrono", "dashmap", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", "futures", "log", "object_store 0.12.5", @@ -2702,19 +2711,19 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" dependencies = [ "arrow 57.3.0", "async-trait", "chrono", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-doc 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-functions-aggregate-common 52.5.0", + "datafusion-functions-window-common 52.5.0", + "datafusion-physical-expr-common 52.5.0", "indexmap", "itertools 0.14.0", "paste", @@ -2747,12 +2756,12 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" dependencies = [ "arrow 57.3.0", - "datafusion-common 51.0.0", + "datafusion-common 52.5.0", "indexmap", "itertools 0.14.0", "paste", @@ -2773,9 +2782,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" dependencies = [ "arrow 57.3.0", "arrow-buffer 57.3.0", @@ -2783,12 +2792,13 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-macros 51.0.0", + "chrono-tz", + "datafusion-common 52.5.0", + "datafusion-doc 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-macros 52.5.0", "hex", "itertools 0.14.0", "log", @@ -2835,20 +2845,20 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-doc 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-functions-aggregate-common 52.5.0", + "datafusion-macros 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", "half", "log", "paste", @@ -2878,15 +2888,15 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-physical-expr-common 52.5.0", ] [[package]] @@ -2904,22 +2914,22 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" dependencies = [ "arrow 57.3.0", "arrow-ord 57.3.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-functions-aggregate 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-doc 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-functions-aggregate 52.5.0", + "datafusion-functions-aggregate-common 52.5.0", + "datafusion-macros 52.5.0", + "datafusion-physical-expr-common 52.5.0", "itertools 0.14.0", "log", "paste", @@ -2952,16 +2962,16 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" dependencies = [ "arrow 57.3.0", "async-trait", - "datafusion-catalog 51.0.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-catalog 52.5.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-plan 52.5.0", "parking_lot", "paste", ] @@ -2984,18 +2994,18 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" dependencies = [ "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-doc 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-functions-window-common 52.5.0", + "datafusion-macros 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", "log", "paste", ] @@ -3020,12 +3030,12 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93" dependencies = [ - "datafusion-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-physical-expr-common 52.5.0", ] [[package]] @@ -3040,11 +3050,11 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5" dependencies = [ - "datafusion-doc 51.0.0", + "datafusion-doc 52.5.0", "quote", "syn 2.0.117", ] @@ -3062,16 +3072,16 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" dependencies = [ "arrow 57.3.0", "chrono", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-physical-expr 52.5.0", "indexmap", "itertools 0.14.0", "log", @@ -3101,24 +3111,25 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-functions-aggregate-common 52.5.0", + "datafusion-physical-expr-common 52.5.0", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "parking_lot", "paste", "petgraph", + "tokio", ] [[package]] @@ -3147,16 +3158,16 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" dependencies = [ "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", "itertools 0.14.0", ] @@ -3177,16 +3188,19 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-expr-common 51.0.0", - "hashbrown 0.14.5", + "chrono", + "datafusion-common 52.5.0", + "datafusion-expr-common 52.5.0", + "hashbrown 0.16.1", + "indexmap", "itertools 0.14.0", + "parking_lot", ] [[package]] @@ -3208,19 +3222,19 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" dependencies = [ "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-pruning 51.0.0", + "datafusion-common 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", + "datafusion-pruning 52.5.0", "itertools 0.14.0", ] @@ -3245,27 +3259,27 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" dependencies = [ "ahash 0.8.12", "arrow 57.3.0", "arrow-ord 57.3.0", "arrow-schema 57.3.0", "async-trait", - "chrono", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common 52.5.0", + "datafusion-common-runtime 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-functions-aggregate-common 52.5.0", + "datafusion-functions-window-common 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", "itertools 0.14.0", "log", @@ -3308,17 +3322,17 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" dependencies = [ "arrow 57.3.0", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-common 52.5.0", + "datafusion-datasource 52.5.0", + "datafusion-expr-common 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-expr-common 52.5.0", + "datafusion-physical-plan 52.5.0", "itertools 0.14.0", "log", ] @@ -3342,15 +3356,15 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d" dependencies = [ "async-trait", - "datafusion-common 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-common 52.5.0", + "datafusion-execution 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-plan 52.5.0", "parking_lot", ] @@ -3397,15 +3411,15 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "51.0.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" dependencies = [ "arrow 57.3.0", "bigdecimal", "chrono", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", "indexmap", "log", "regex", @@ -3648,16 +3662,6 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "earcutr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79127ed59a85d7687c409e9978547cffb7dc79675355ed22da6b66fd5f6ead01" -dependencies = [ - "itertools 0.11.0", - "num-traits", -] - [[package]] name = "educe" version = "0.6.0" @@ -3920,12 +3924,6 @@ dependencies = [ "zlib-rs", ] -[[package]] -name = "float_next_after" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" - [[package]] name = "fnv" version = "1.0.7" @@ -3980,9 +3978,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9e5c0b1c67a38cb92b41535d44623483beb9511592ae23a3bf42ddec758690" +checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" dependencies = [ "arrow-array 57.3.0", "rand 0.9.2", @@ -4153,128 +4151,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "geo" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc1a1678e54befc9b4bcab6cd43b8e7f834ae8ea121118b0fd8c42747675b4a" -dependencies = [ - "earcutr", - "float_next_after", - "geo-types", - "geographiclib-rs", - "i_overlay", - "log", - "num-traits", - "robust", - "rstar", - "spade", -] - -[[package]] -name = "geo-traits" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" -dependencies = [ - "geo-types", -] - -[[package]] -name = "geo-types" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c" -dependencies = [ - "approx", - "num-traits", - "rayon", - "rstar", - "serde", -] - -[[package]] -name = "geoarrow-array" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1cc4106ac0a0a512c398961ce95d8150475c84a84e17c4511c3643fa120a17" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", - "geo-traits", - "geoarrow-schema", - "num-traits", - "wkb", - "wkt", -] - -[[package]] -name = "geoarrow-expr-geo" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84300361ce57fb875bcaa6e32b95b0aff5c6b1af692b936bdd58ff343f4394" -dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "geo", - "geo-traits", - "geoarrow-array", - "geoarrow-schema", -] - -[[package]] -name = "geoarrow-schema" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97be4e9f523f92bd6a0e0458323f4b783d073d011664decd8dbf05651704f34" -dependencies = [ - "arrow-schema 57.3.0", - "geo-traits", - "serde", - "serde_json", - "thiserror 1.0.69", -] - -[[package]] -name = "geodatafusion" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773cfa1fb0d7f7661b76b3fde00f3ffd8e0ff7b3635096f0ff6294fe5ca62a2b" -dependencies = [ - "arrow-arith 57.3.0", - "arrow-array 57.3.0", - "arrow-schema 57.3.0", - "datafusion 51.0.0", - "geo", - "geo-traits", - "geoarrow-array", - "geoarrow-expr-geo", - "geoarrow-schema", - "geohash", - "thiserror 1.0.69", - "wkt", -] - -[[package]] -name = "geographiclib-rs" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7" -dependencies = [ - "libm", -] - -[[package]] -name = "geohash" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fb94b1a65401d6cbf22958a9040aa364812c26674f841bee538b12c135db1e6" -dependencies = [ - "geo-types", - "libm", -] - [[package]] name = "get_dir" version = "0.5.0" @@ -4403,15 +4279,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ed72152641d513a6084a3907bfcba3f35ae2d3335c22ce2242969c25ff8e46" -[[package]] -name = "hash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" -dependencies = [ - "byteorder", -] - [[package]] name = "hashbag" version = "0.1.13" @@ -4470,16 +4337,6 @@ dependencies = [ "foldhash 0.2.0", ] -[[package]] -name = "heapless" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" -dependencies = [ - "hash32", - "stable_deref_trait", -] - [[package]] name = "heck" version = "0.5.0" @@ -4649,49 +4506,6 @@ dependencies = [ "serde", ] -[[package]] -name = "i_float" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "010025c2c532c8d82e42d0b8bb5184afa449fa6f06c709ea9adcb16c49ae405b" -dependencies = [ - "libm", -] - -[[package]] -name = "i_key_sort" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27" - -[[package]] -name = "i_overlay" -version = "4.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3" -dependencies = [ - "i_float", - "i_key_sort", - "i_shape", - "i_tree", - "rayon", -] - -[[package]] -name = "i_shape" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ea154b742f7d43dae2897fcd5ead86bc7b5eefcedd305a7ebf9f69d44d61082" -dependencies = [ - "i_float", -] - -[[package]] -name = "i_tree" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915" - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -4976,15 +4790,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -5179,9 +4984,9 @@ dependencies = [ [[package]] name = "lance" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7f07b905df393a5554eba19055c620f9ea25a3e40a013bda4bd8dc4ca66f01" +checksum = "efe6c3ddd79cdfd2b7e1c23cafae52806906bc40fbd97de9e8cf2f8c7a75fc04" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5198,12 +5003,13 @@ dependencies = [ "byteorder", "bytes", "chrono", + "crossbeam-skiplist", "dashmap", - "datafusion 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-physical-plan 52.5.0", "deepsize", "either", "futures", @@ -5215,7 +5021,6 @@ dependencies = [ "lance-datafusion", "lance-encoding", "lance-file", - "lance-geo", "lance-index", "lance-io", "lance-linalg", @@ -5229,7 +5034,7 @@ dependencies = [ "prost 0.14.3", "prost-types", "rand 0.9.2", - "roaring 0.10.12", + "roaring", "semver", "serde", "serde_json", @@ -5237,6 +5042,7 @@ dependencies = [ "tantivy", "tokio", "tokio-stream", + "tokio-util", "tracing", "url", "uuid", @@ -5244,9 +5050,9 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "100e076cb81c8f0c24cd2881c706fc53e037c7d6e81eb320e929e265d157effb" +checksum = "5d9f5d95bdda2a2b790f1fb8028b5b6dcf661abeb3133a8bca0f3d24b054af87" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5256,6 +5062,7 @@ dependencies = [ "arrow-schema 57.3.0", "arrow-select 57.3.0", "bytes", + "futures", "getrandom 0.2.17", "half", "jsonb", @@ -5283,9 +5090,9 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588318d3d1ba0f97162fab39a323a0a49866bb35b32af42572c6b6a12296fa27" +checksum = "f827d6ab9f8f337a9509d5ad66a12f3314db8713868260521c344ef6135eb4e4" dependencies = [ "arrayref", "paste", @@ -5294,9 +5101,9 @@ dependencies = [ [[package]] name = "lance-core" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa01d1cf490ccfd3b8eaeee2781415d0419e6be8366040e57e43677abf2644e" +checksum = "0f1e25df6a79bf72ee6bcde0851f19b1cd36c5848c1b7db83340882d3c9fdecb" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5305,8 +5112,8 @@ dependencies = [ "byteorder", "bytes", "chrono", - "datafusion-common 51.0.0", - "datafusion-sql 51.0.0", + "datafusion-common 52.5.0", + "datafusion-sql 52.5.0", "deepsize", "futures", "itertools 0.13.0", @@ -5320,7 +5127,7 @@ dependencies = [ "pin-project", "prost 0.14.3", "rand 0.9.2", - "roaring 0.10.12", + "roaring", "serde_json", "snafu", "tempfile", @@ -5333,9 +5140,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef89a39e3284eef76f79e63f23de8881a0583ad6feb20ed39f47eadd847a2b88" +checksum = "93146de8ae720cb90edef81c2f2d0a1b065fc2f23ecff2419546f389b0fa70a4" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -5345,19 +5152,19 @@ dependencies = [ "arrow-select 57.3.0", "async-trait", "chrono", - "datafusion 51.0.0", - "datafusion-common 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-physical-expr 51.0.0", + "datafusion 52.5.0", + "datafusion-common 52.5.0", + "datafusion-functions 52.5.0", + "datafusion-physical-expr 52.5.0", "futures", "jsonb", "lance-arrow", "lance-core", "lance-datagen", - "lance-geo", "log", "pin-project", "prost 0.14.3", + "prost-build", "snafu", "tokio", "tracing", @@ -5365,9 +5172,9 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2a60eef5c47e65d91e2ffa8e7e1629c52e7190c8b88a371a1a60601dc49371" +checksum = "ccec8ce4d8e0a87a99c431dab2364398029f2ffb649c1a693c60c79e05ed30dd" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -5385,9 +5192,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce4a6631308aa681b2671af8f2a845ff781f8d4e755a2a7ccd012379467094" +checksum = "5c1aec0bbbac6bce829bc10f1ba066258126100596c375fb71908ecf11c2c2a5" dependencies = [ "arrow-arith 57.3.0", "arrow-array 57.3.0", @@ -5424,9 +5231,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d4d82357cbfaa1a18494226c15b1cb3c8ed0b6c84b91146323c82047ede419" +checksum = "14a8c548804f5b17486dc2d3282356ed1957095a852780283bc401fdd69e9075" dependencies = [ "arrow-arith 57.3.0", "arrow-array 57.3.0", @@ -5438,7 +5245,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "datafusion-common 51.0.0", + "datafusion-common 52.5.0", "deepsize", "futures", "lance-arrow", @@ -5456,27 +5263,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "lance-geo" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7183fc870da62826f0f97df8007b634da053eb310157856efe1dc74f446951c" -dependencies = [ - "datafusion 51.0.0", - "geo-traits", - "geo-types", - "geoarrow-array", - "geoarrow-schema", - "geodatafusion", - "lance-core", - "serde", -] - [[package]] name = "lance-index" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20e9c5aa7024a63af9ae89ee8c0f23c8421b7896742e5cd4a271a60f9956cb80" +checksum = "2da212f0090ea59f79ac3686660f596520c167fe1cb5f408900cf71d215f0e03" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5490,19 +5281,17 @@ dependencies = [ "bitpacking", "bitvec", "bytes", + "chrono", "crossbeam-queue", - "datafusion 51.0.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-sql 51.0.0", + "datafusion 52.5.0", + "datafusion-common 52.5.0", + "datafusion-expr 52.5.0", + "datafusion-physical-expr 52.5.0", + "datafusion-sql 52.5.0", "deepsize", "dirs", "fst", "futures", - "geo-types", - "geoarrow-array", - "geoarrow-schema", "half", "itertools 0.13.0", "jsonb", @@ -5512,7 +5301,6 @@ dependencies = [ "lance-datagen", "lance-encoding", "lance-file", - "lance-geo", "lance-io", "lance-linalg", "lance-table", @@ -5528,7 +5316,7 @@ dependencies = [ "rand_distr 0.5.1", "rangemap", "rayon", - "roaring 0.10.12", + "roaring", "serde", "serde_json", "smallvec", @@ -5543,9 +5331,9 @@ dependencies = [ [[package]] name = "lance-io" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d2af0b17fb374a8181bcf1a10bce5703ae3ee4373c1587ce4bba23e15e45c8" +checksum = "41d958eb4b56f03bbe0f5f85eb2b4e9657882812297b6f711f201ffc995f259f" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5562,6 +5350,7 @@ dependencies = [ "chrono", "deepsize", "futures", + "http", "lance-arrow", "lance-core", "lance-namespace", @@ -5572,8 +5361,8 @@ dependencies = [ "prost 0.14.3", "rand 0.9.2", "serde", - "shellexpand", "snafu", + "tempfile", "tokio", "tracing", "url", @@ -5581,9 +5370,9 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5125aa62696e75a7475807564b4921f252d8815be606b84bc00e6def0f5c24bb" +checksum = "0285b70da35def7ed95e150fae1d5308089554e1290470403ed3c50cb235bc5e" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5599,23 +5388,24 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70545c2676ce954dfd801da5c6a631a70bba967826cd3a8f31b47d1f04bbfed3" +checksum = "5f78e2a828b654e062a495462c6e3eb4fcf0e7e907d761b8f217fc09ccd3ceac" dependencies = [ "arrow 57.3.0", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", "snafu", ] [[package]] name = "lance-namespace-reqwest-client" -version = "0.4.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2acdba67f84190067532fce07b51a435dd390d7cdc1129a05003e5cb3274cf0" +checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc" dependencies = [ "reqwest 0.12.28", "serde", @@ -5626,9 +5416,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "2.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06ad37bd90045de8ef533df170c6098e6ff6ecb427aade47d7db8e2c86f2678" +checksum = "3df9c4adca3eb2074b3850432a9fb34248a3d90c3d6427d158b13ff9355664ee" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -5652,7 +5442,7 @@ dependencies = [ "prost-types", "rand 0.9.2", "rangemap", - "roaring 0.10.12", + "roaring", "semver", "serde", "serde_json", @@ -6439,7 +6229,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ - "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -8075,16 +7864,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "roaring" -version = "0.10.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" -dependencies = [ - "bytemuck", - "byteorder", -] - [[package]] name = "roaring" version = "0.11.3" @@ -8095,23 +7874,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "robust" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" - -[[package]] -name = "rstar" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" -dependencies = [ - "heapless", - "num-traits", - "smallvec", -] - [[package]] name = "rstest" version = "0.26.1" @@ -8654,15 +8416,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shellexpand" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" -dependencies = [ - "dirs", -] - [[package]] name = "shlex" version = "1.3.0" @@ -8793,18 +8546,18 @@ dependencies = [ [[package]] name = "snafu" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ "heck", "proc-macro2", @@ -8838,18 +8591,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spade" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990" -dependencies = [ - "hashbrown 0.15.5", - "num-traits", - "robust", - "smallvec", -] - [[package]] name = "sqllogictest" version = "0.29.1" @@ -11015,7 +10756,7 @@ version = "0.1.0" dependencies = [ "async-trait", "futures", - "roaring 0.11.3", + "roaring", "tracing", "vortex-array", "vortex-buffer", @@ -11820,31 +11561,6 @@ dependencies = [ "wasmparser", ] -[[package]] -name = "wkb" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" -dependencies = [ - "byteorder", - "geo-traits", - "num_enum", - "thiserror 1.0.69", -] - -[[package]] -name = "wkt" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7" -dependencies = [ - "geo-traits", - "geo-types", - "log", - "num-traits", - "thiserror 1.0.69", -] - [[package]] name = "writeable" version = "0.6.2" diff --git a/Cargo.toml b/Cargo.toml index 12e0acb882d..c9b7814af34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ keywords = ["vortex"] license = "Apache-2.0" readme = "README.md" repository = "https://github.com/spiraldb/vortex" -rust-version = "1.90.0" +rust-version = "1.91.0" version = "0.1.0" [workspace.dependencies] diff --git a/benchmarks/lance-bench/Cargo.toml b/benchmarks/lance-bench/Cargo.toml index e5c6044e4f5..775b051dd9c 100644 --- a/benchmarks/lance-bench/Cargo.toml +++ b/benchmarks/lance-bench/Cargo.toml @@ -15,8 +15,8 @@ version.workspace = true publish = false [dependencies] -lance = { version = "2.0.0", default-features = false } -lance-encoding = { version = "2.0.0" } +lance = { version = "4", default-features = false } +lance-encoding = { version = "4" } anyhow = { workspace = true } arrow-cast = { version = "57" } diff --git a/benchmarks/vector-search-bench/src/ingest.rs b/benchmarks/vector-search-bench/src/ingest.rs index f3fe04f4422..bab176b0482 100644 --- a/benchmarks/vector-search-bench/src/ingest.rs +++ b/benchmarks/vector-search-bench/src/ingest.rs @@ -169,8 +169,13 @@ mod tests { let chunk = StructArray::from_fields(&[("id", id_array(&[0, 1])), ("emb", emb)])?.into_array(); let out = transform_chunk(chunk, &mut ctx)?; - let out_struct = out.as_opt::().expect("returns Struct"); - let out_emb = out_struct.unmasked_field_by_name("emb").unwrap().clone(); + let out_struct = out + .as_opt::() + .context("transform_chunk should return a Struct array")?; + let out_emb = out_struct + .unmasked_field_by_name("emb") + .context("transform_chunk output should contain an emb field")? + .clone(); let DType::Extension(ext) = out_emb.dtype() else { panic!("expected extension dtype, got {}", out_emb.dtype()); }; diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index cb5ea0358ba..de1313339dd 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -158,12 +158,7 @@ mod test { .execute::(&mut ctx)?; assert_arrays_eq!(primitive_values, milliseconds); - assert!( - primitive_values - .validity() - .unwrap() - .mask_eq(&validity, &mut ctx)? - ); + assert!(primitive_values.validity()?.mask_eq(&validity, &mut ctx)?); Ok(()) } } diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index bfde0671f31..2e517819059 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -442,6 +442,7 @@ mod test { use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_error::VortexError; + use vortex_error::vortex_err; use vortex_session::VortexSession; use super::*; @@ -540,13 +541,15 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx)?; - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked + .patches() + .ok_or_else(|| vortex_err!("expected patches"))?; let chunk_offsets = patches .chunk_offsets() .as_ref() - .unwrap() + .ok_or_else(|| vortex_err!("expected chunk offsets"))? .clone() .execute::(&mut ctx)?; @@ -570,13 +573,15 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx)?; - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked + .patches() + .ok_or_else(|| vortex_err!("expected patches"))?; let chunk_offsets = patches .chunk_offsets() .as_ref() - .unwrap() + .ok_or_else(|| vortex_err!("expected chunk offsets"))? .clone() .execute::(&mut ctx)?; @@ -596,13 +601,15 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx)?; - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked + .patches() + .ok_or_else(|| vortex_err!("expected patches"))?; let chunk_offsets = patches .chunk_offsets() .as_ref() - .unwrap() + .ok_or_else(|| vortex_err!("expected chunk offsets"))? .clone() .execute::(&mut ctx)?; @@ -627,13 +634,15 @@ mod test { .for_each(|&idx| values[idx] = patch_value); let array = PrimitiveArray::from_iter(values); - let bitpacked = bitpack_encode(&array, 4, None, &mut ctx).unwrap(); + let bitpacked = bitpack_encode(&array, 4, None, &mut ctx)?; - let patches = bitpacked.patches().unwrap(); + let patches = bitpacked + .patches() + .ok_or_else(|| vortex_err!("expected patches"))?; let chunk_offsets = patches .chunk_offsets() .as_ref() - .unwrap() + .ok_or_else(|| vortex_err!("expected chunk offsets"))? .clone() .execute::(&mut ctx)?; diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index d7eb030ef7a..2f66123fae9 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -305,11 +305,8 @@ mod tests { let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); assert!(bitpacked.patches().is_some()); - let slice_ref = bitpacked.into_array().slice(1023..1025).unwrap(); - let actual = slice_ref - .execute::(&mut ctx) - .unwrap() - .into_primitive(); + let slice_ref = bitpacked.into_array().slice(1023..1025)?; + let actual = slice_ref.execute::(&mut ctx)?.into_primitive(); assert_arrays_eq!(actual, PrimitiveArray::from_iter([1535u16, 1536])); Ok(()) } @@ -323,11 +320,8 @@ mod tests { let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 2229); assert!(bitpacked.patches().is_some()); - let slice_ref = bitpacked.into_array().slice(1023..2049).unwrap(); - let actual = slice_ref - .execute::(&mut ctx) - .unwrap() - .into_primitive(); + let slice_ref = bitpacked.into_array().slice(1023..2049)?; + let actual = slice_ref.execute::(&mut ctx)?.into_primitive(); assert_arrays_eq!( actual, PrimitiveArray::from_iter((1023u16..2049).map(|x| x + 512)) @@ -380,11 +374,11 @@ mod tests { // Verify the validity mask was correctly applied. assert_eq!(result.len(), 5); let mut ctx = SESSION.create_execution_ctx(); - assert!(!result.execute_scalar(0, &mut ctx).unwrap().is_null()); - assert!(result.execute_scalar(1, &mut ctx).unwrap().is_null()); - assert!(!result.execute_scalar(2, &mut ctx).unwrap().is_null()); - assert!(!result.execute_scalar(3, &mut ctx).unwrap().is_null()); - assert!(result.execute_scalar(4, &mut ctx).unwrap().is_null()); + assert!(!result.execute_scalar(0, &mut ctx)?.is_null()); + assert!(result.execute_scalar(1, &mut ctx)?.is_null()); + assert!(!result.execute_scalar(2, &mut ctx)?.is_null()); + assert!(!result.execute_scalar(3, &mut ctx)?.is_null()); + assert!(result.execute_scalar(4, &mut ctx)?.is_null()); Ok(()) } @@ -504,10 +498,7 @@ mod tests { let executed = { let mut ctx = SESSION.create_execution_ctx(); - bitpacked - .into_array() - .execute::(&mut ctx) - .unwrap() + bitpacked.into_array().execute::(&mut ctx)? }; assert_eq!( @@ -530,8 +521,7 @@ mod tests { let mut ctx = SESSION.create_execution_ctx(); unpacked_array .into_array() - .execute::(&mut ctx) - .unwrap() + .execute::(&mut ctx)? .into_primitive() }; assert_eq!( @@ -560,13 +550,12 @@ mod tests { // Test with sliced array (offset > 0). let values = PrimitiveArray::from_iter(0u32..2048); let bitpacked = encode(&values, 11); - let slice_ref = bitpacked.into_array().slice(500..1500).unwrap(); + let slice_ref = bitpacked.into_array().slice(500..1500)?; let sliced = { let mut ctx = SESSION.create_execution_ctx(); slice_ref .clone() - .execute::(&mut ctx) - .unwrap() + .execute::(&mut ctx)? .into_primitive() }; @@ -575,7 +564,7 @@ mod tests { let unpacked_array = sliced; let executed = { let mut ctx = SESSION.create_execution_ctx(); - slice_ref.execute::(&mut ctx).unwrap() + slice_ref.execute::(&mut ctx)? }; assert_eq!( diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 65f8b853aa6..d51cef72b49 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -135,8 +135,8 @@ mod tests { let array = PrimitiveArray::from_option_iter( (0u8..200).map(|i| (!(50..100).contains(&i)).then_some(i)), ); - let (bases, deltas) = delta_compress(&array, &mut ctx).unwrap(); - let bitpacked_deltas = bitpack_encode(&deltas, 1, None, &mut ctx).unwrap(); + let (bases, deltas) = delta_compress(&array, &mut ctx)?; + let bitpacked_deltas = bitpack_encode(&deltas, 1, None, &mut ctx)?; let packed_delta = Delta::try_new( bases.into_array(), bitpacked_deltas.into_array(), diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 358eff2a4e9..ad9566027d2 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -143,7 +143,7 @@ mod test { // Create a range offset by a million. let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); - let bp = BitPackedData::encode(&array.into_array(), 2, &mut ctx).unwrap(); + let bp = BitPackedData::encode(&array.into_array(), 2, &mut ctx)?; let compressed = FoR::try_new(bp.clone().into_array(), 10u32.into())?; let decompressed = fused_decompress::(&compressed, bp.as_view(), &mut ctx)?; assert_arrays_eq!(decompressed, expect); @@ -154,7 +154,7 @@ mod test { fn test_overflow() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter(i8::MIN..=i8::MAX); - let compressed = FoRData::encode(array.clone()).unwrap(); + let compressed = FoRData::encode(array.clone())?; assert_eq!( i8::MIN, compressed diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 186d7f37c3c..aeaba0ef42e 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -365,7 +365,7 @@ mod tests { fn test_rle_serialization() -> VortexResult<()> { let mut exec_ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx).unwrap(); + let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx)?; assert_eq!(rle_array.len(), 2048); let original_data = rle_array @@ -374,10 +374,10 @@ mod tests { .execute::(&mut exec_ctx)?; let ctx = ArrayContext::empty(); - let serialized = rle_array - .into_array() - .serialize(&ctx, &SESSION, &SerializeOptions::default()) - .unwrap(); + let serialized = + rle_array + .into_array() + .serialize(&ctx, &SESSION, &SerializeOptions::default())?; let mut concat = ByteBufferMut::empty(); for buf in serialized { @@ -385,15 +385,13 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); - let decoded = parts - .decode( - &DType::Primitive(PType::U32, Nullability::NonNullable), - 2048, - &ReadContext::new(ctx.to_ids()), - &SESSION, - ) - .unwrap(); + let parts = SerializedArray::try_from(concat)?; + let decoded = parts.decode( + &DType::Primitive(PType::U32, Nullability::NonNullable), + 2048, + &ReadContext::new(ctx.to_ids()), + &SESSION, + )?; let decoded_data = decoded.execute::(&mut exec_ctx)?; @@ -405,7 +403,7 @@ mod tests { fn test_rle_serialization_slice() -> VortexResult<()> { let mut exec_ctx = SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); - let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx).unwrap(); + let rle_array = RLEData::encode(primitive.as_view(), &mut exec_ctx)?; let sliced = RLE::try_new( rle_array.values().clone(), @@ -418,11 +416,11 @@ mod tests { assert_eq!(sliced.len(), 100); let ctx = ArrayContext::empty(); - let serialized = sliced - .clone() - .into_array() - .serialize(&ctx, &SESSION, &SerializeOptions::default()) - .unwrap(); + let serialized = + sliced + .clone() + .into_array() + .serialize(&ctx, &SESSION, &SerializeOptions::default())?; let mut concat = ByteBufferMut::empty(); for buf in serialized { @@ -430,15 +428,13 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); - let decoded = parts - .decode( - sliced.dtype(), - sliced.len(), - &ReadContext::new(ctx.to_ids()), - &SESSION, - ) - .unwrap(); + let parts = SerializedArray::try_from(concat)?; + let decoded = parts.decode( + sliced.dtype(), + sliced.len(), + &ReadContext::new(ctx.to_ids()), + &SESSION, + )?; let original_data = sliced .as_array() diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index f566e1f101b..e48f31f51da 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -178,8 +178,7 @@ mod tests { let encoded_u8 = RLEData::encode( PrimitiveArray::new(array_u8, Validity::NonNullable).as_view(), &mut ctx, - ) - .unwrap(); + )?; let decoded_u8 = encoded_u8 .as_array() .clone() @@ -192,8 +191,7 @@ mod tests { let encoded_u16 = RLEData::encode( PrimitiveArray::new(array_u16, Validity::NonNullable).as_view(), &mut ctx, - ) - .unwrap(); + )?; let decoded_u16 = encoded_u16 .as_array() .clone() @@ -206,8 +204,7 @@ mod tests { let encoded_u64 = RLEData::encode( PrimitiveArray::new(array_u64, Validity::NonNullable).as_view(), &mut ctx, - ) - .unwrap(); + )?; let decoded_u64 = encoded_u64 .as_array() .clone() @@ -251,8 +248,7 @@ mod tests { let encoded = RLEData::encode( PrimitiveArray::new(values, Validity::NonNullable).as_view(), &mut ctx, - ) - .unwrap(); + )?; assert_eq!(encoded.values().len(), 2); // 2 chunks, each storing value 42 let decoded = encoded @@ -272,8 +268,7 @@ mod tests { let encoded = RLEData::encode( PrimitiveArray::new(values, Validity::NonNullable).as_view(), &mut ctx, - ) - .unwrap(); + )?; assert_eq!(encoded.values().len(), 256); let decoded = encoded @@ -334,17 +329,12 @@ mod tests { .clone() .into_array() .execute::(&mut ctx)?; - let result = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); + let result = RLEData::encode(primitive.as_view(), &mut ctx)?; let decoded = result .as_array() .clone() .execute::(&mut ctx)?; - let expected = PrimitiveArray::new( - values, - primitive - .validity() - .vortex_expect("primitive validity should be derivable"), - ); + let expected = PrimitiveArray::new(values, primitive.validity()?); assert_arrays_eq!(decoded, expected); Ok(()) } @@ -555,7 +545,7 @@ mod tests { ) -> VortexResult<()> { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let primitive = PrimitiveArray::from_iter(values); - let rle = RLEData::encode(primitive.as_view(), &mut ctx).unwrap(); + let rle = RLEData::encode(primitive.as_view(), &mut ctx)?; let decoded = rle.as_array().clone().execute::(&mut ctx)?; assert_arrays_eq!(primitive, decoded); Ok(()) diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index ba13eb8e808..1f549e6e5bf 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -304,17 +304,22 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_error::vortex_err; use crate::ParquetVariant; use crate::ParquetVariantData; use crate::array::ParquetVariantArrayExt; fn assert_arrow_variant_storage_roundtrip(struct_array: StructArray) -> VortexResult<()> { - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; - let variant_view = vortex_arr.as_opt::().unwrap(); + let variant_view = vortex_arr + .as_opt::() + .ok_or_else(|| vortex_err!("expected variant array"))?; let child = variant_view.child(); - let inner = child.as_opt::().unwrap(); + let inner = child + .as_opt::() + .ok_or_else(|| vortex_err!("expected parquet variant child"))?; let mut ctx = LEGACY_SESSION.create_execution_ctx(); let roundtripped = inner.to_arrow(&mut ctx)?; @@ -390,10 +395,9 @@ mod tests { ] .into(); let struct_array = - StructArray::try_new(struct_fields, vec![Arc::new(metadata), typed_value], None) - .unwrap(); + StructArray::try_new(struct_fields, vec![Arc::new(metadata), typed_value], None)?; - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; assert_eq!(vortex_arr.len(), 3); @@ -402,8 +406,13 @@ mod tests { &DType::Variant(Nullability::NonNullable) ); - let variant_arr = vortex_arr.as_opt::().unwrap(); - let inner = variant_arr.child().as_opt::().unwrap(); + let variant_arr = vortex_arr + .as_opt::() + .ok_or_else(|| vortex_err!("expected variant array"))?; + let inner = variant_arr + .child() + .as_opt::() + .ok_or_else(|| vortex_err!("expected parquet variant child"))?; assert!(inner.typed_value_array().is_some()); Ok(()) @@ -473,8 +482,7 @@ mod tests { .into(), vec![metadata, typed_value], None, - ) - .unwrap(); + )?; assert_arrow_variant_storage_roundtrip(struct_array) } @@ -494,8 +502,7 @@ mod tests { .into(), vec![metadata, value, typed_value], None, - ) - .unwrap(); + )?; assert_arrow_variant_storage_roundtrip(struct_array) } @@ -512,8 +519,7 @@ mod tests { .into(), vec![metadata, value], Some(NullBuffer::from(vec![true, false, true])), - ) - .unwrap(); + )?; assert_arrow_variant_storage_roundtrip(struct_array) } diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index 4743b9fc1a7..48ef4ec1d64 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -135,9 +135,8 @@ mod tests { inner.fields().clone(), inner.columns().to_vec(), Some(NullBuffer::from(vec![true, false, true, false])), - ) - .unwrap(); - let arrow_variant = ArrowVariantArray::try_new(&null_struct).unwrap(); + )?; + let arrow_variant = ArrowVariantArray::try_new(&null_struct)?; ParquetVariantData::from_arrow_variant(&arrow_variant) } @@ -306,9 +305,8 @@ mod tests { .into(), vec![metadata, typed_value], None, - ) - .unwrap(); - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + )?; + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; let arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; let sliced = arr.slice(1..3)?; diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index e059b640912..8a323cce76f 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -384,13 +384,11 @@ mod tests { let vortex_arr = ParquetVariantData::from_arrow_variant(arrow_variant)?; for index in rows { - let expected_inner = - parquet_variant_to_scalar(arrow_variant.try_value(index).unwrap())?; + let expected_inner = parquet_variant_to_scalar(arrow_variant.try_value(index)?)?; let expected = Scalar::try_new( vortex_arr.dtype().clone(), Some(ScalarValue::Variant(Box::new(expected_inner))), - ) - .unwrap(); + )?; assert_eq!( vortex_arr.execute_scalar(index, &mut LEGACY_SESSION.create_execution_ctx())?, expected @@ -412,10 +410,9 @@ mod tests { inner.fields().clone(), inner.columns().to_vec(), Some(NullBuffer::from(vec![true, false, true])), - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&null_struct).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&null_struct)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; assert_eq!(vortex_arr.dtype(), &DType::Variant(Nullability::Nullable)); @@ -453,10 +450,9 @@ mod tests { inner.fields().clone(), inner.columns().to_vec(), Some(NullBuffer::from(vec![false, false])), - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&null_struct).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&null_struct)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; assert_eq!(vortex_arr.dtype(), &DType::Variant(Nullability::Nullable)); @@ -533,10 +529,9 @@ mod tests { Arc::new(Int32Array::from(vec![10, 20, 30])), ], None, - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; assert_scalar_at_matches_arrow_try_value(&arrow_variant, [0, 1, 2]) } @@ -567,10 +562,9 @@ mod tests { .into(), vec![metadata, value, typed_value], None, - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; assert_scalar_at_matches_arrow_try_value(&arrow_variant, [0, 1, 2]) } @@ -583,22 +577,18 @@ mod tests { vec![Arc::new(Field::new("typed_value", DataType::Int32, false))].into(), vec![Arc::new(Int32Array::from(vec![10, 20, 30]))], None, - ) - .unwrap(); + )?; - let typed_value: ArrowArrayRef = Arc::new( - ListArray::try_new( - Arc::new(Field::new( - "element", - element_struct.data_type().clone(), - false, - )), - OffsetBuffer::from_lengths([2, 1]), - Arc::new(element_struct), - None, - ) - .unwrap(), - ); + let typed_value: ArrowArrayRef = Arc::new(ListArray::try_new( + Arc::new(Field::new( + "element", + element_struct.data_type().clone(), + false, + )), + OffsetBuffer::from_lengths([2, 1]), + Arc::new(element_struct), + None, + )?); let struct_array = StructArray::try_new( vec![ @@ -612,10 +602,9 @@ mod tests { .into(), vec![binary_view_array(&[b"\x01\x00", b"\x01\x00"]), typed_value], None, - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; let row0 = vortex_arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; @@ -676,21 +665,17 @@ mod tests { vec![Arc::new(Field::new("typed_value", DataType::Int32, false))].into(), vec![Arc::new(Int32Array::from(vec![7]))], None, - ) - .unwrap(); - let typed_value: ArrowArrayRef = Arc::new( - StructArray::try_new( - vec![Arc::new(Field::new( - "a", - shredded_a.data_type().clone(), - false, - ))] - .into(), - vec![Arc::new(shredded_a)], - None, - ) - .unwrap(), - ); + )?; + let typed_value: ArrowArrayRef = Arc::new(StructArray::try_new( + vec![Arc::new(Field::new( + "a", + shredded_a.data_type().clone(), + false, + ))] + .into(), + vec![Arc::new(shredded_a)], + None, + )?); let struct_array = StructArray::try_new( vec![ @@ -709,10 +694,9 @@ mod tests { typed_value, ], None, - ) - .unwrap(); + )?; - let arrow_variant = ArrowVariantArray::try_new(&struct_array).unwrap(); + let arrow_variant = ArrowVariantArray::try_new(&struct_array)?; let vortex_arr = ParquetVariantData::from_arrow_variant(&arrow_variant)?; let object = vortex_arr.execute_scalar(0, &mut LEGACY_SESSION.create_execution_ctx())?; let object = object.as_variant().value().unwrap().as_struct(); diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 34f8387ff19..f423dcaa60e 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -157,7 +157,7 @@ mod tests { // Values: [10, 20, 30] means values 10, 10, 10, 20, 20, 30, 30, 30 let run_ends = Int32Array::from(vec![3i32, 5, 8]); let values = Int32Array::from(vec![10, 20, 30]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Convert to Vortex let vortex_array = decode_run_array(&arrow_run_array, false)?; @@ -174,7 +174,7 @@ mod tests { // Create an Arrow RunArray with nullable values let run_ends = Int32Array::from(vec![2i32, 4, 6]); let values = Int32Array::from(vec![Some(100), None, Some(300)]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Convert to Vortex with nullable=true let vortex_array = decode_run_array(&arrow_run_array, true)?; @@ -198,7 +198,7 @@ mod tests { // Test with UInt64 run ends and Float64 values let run_ends = Int64Array::from(vec![1i64, 3, 4]); let values = Float64Array::from(vec![1.5, 2.5, 3.5]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Convert to Vortex let vortex_array = decode_run_array(&arrow_run_array, false)?; @@ -214,7 +214,7 @@ mod tests { // Values: [100, 200, 300, 400] means: 100, 100, 200, 200, 200, 300, 300, 300, 400, 400 let run_ends = Int32Array::from(vec![2i32, 5, 8, 10]); let values = Int32Array::from(vec![100, 200, 300, 400]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Slice the array from index 1 to 7 (length 6) // This should give us: 100, 200, 200, 200, 300, 300 @@ -236,7 +236,7 @@ mod tests { // Values: [Some(10), None, Some(30), Some(40)] let run_ends = Int64Array::from(vec![3i64, 6, 9, 12]); let values = Int64Array::from(vec![Some(10), None, Some(30), Some(40)]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Slice from index 4 to 10 (length 6) // Original: 10, 10, 10, null, null, null, 30, 30, 30, 40, 40, 40 @@ -267,7 +267,7 @@ mod tests { // Values: [Some(10), None, Some(30), Some(40)] let run_ends = Int64Array::from(vec![3i64, 6, 9, 12]); let values = Int64Array::from(vec![Some(10), None, Some(30), Some(40)]); - let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + let arrow_run_array = RunArray::::try_new(&run_ends, &values)?; // Slice from index 4 to 4 (length 0) // Original: 10, 10, 10, null, null, null, 30, 30, 30, 40, 40, 40 diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index 7e0a2af29b0..e4c7a298430 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -137,7 +137,7 @@ mod tests { #[test] fn filter_sliced_run_end() -> VortexResult<()> { - let arr = ree_array().slice(2..7).unwrap(); + let arr = ree_array().slice(2..7)?; let filtered = arr.filter(Mask::from_iter([true, false, false, true, true]))?; let mut ctx = LEGACY_SESSION.create_execution_ctx(); diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index 9e03f166e4b..68b6d3d5981 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -390,8 +390,7 @@ mod tests { decoded .as_ref() .validity()? - .execute_mask(decoded.as_ref().len(), &mut ctx) - .unwrap() + .execute_mask(decoded.as_ref().len(), &mut ctx)? .value(0) ); assert!(decoded.to_bit_buffer().value(0)); @@ -400,8 +399,7 @@ mod tests { !decoded .as_ref() .validity()? - .execute_mask(decoded.as_ref().len(), &mut ctx) - .unwrap() + .execute_mask(decoded.as_ref().len(), &mut ctx)? .value(2000) ); // Third run: valid true @@ -409,8 +407,7 @@ mod tests { decoded .as_ref() .validity()? - .execute_mask(decoded.as_ref().len(), &mut ctx) - .unwrap() + .execute_mask(decoded.as_ref().len(), &mut ctx)? .value(4000) ); assert!(decoded.to_bit_buffer().value(4000)); diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 359b9403801..0dcd8d533b1 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -1082,9 +1082,7 @@ mod test { let indices = buffer![0u8, 3u8, 4u8, 5u8].into_array(); let fill_value = Scalar::null(lists.dtype().clone()); - let sparse = Sparse::try_new(indices, lists, 6, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, lists, 6, fill_value)?.into_array(); let actual = sparse.execute::(&mut ctx)?.into_array(); let result_listview = actual.execute::(&mut ctx)?; @@ -1193,9 +1191,7 @@ mod test { let indices = buffer![0u8, 3u8, 4u8, 5u8].into_array(); let fill_value = Scalar::from(Some(vec![5i32, 6, 7, 8])); - let sparse = Sparse::try_new(indices, lists, 6, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, lists, 6, fill_value)?.into_array(); let actual = sparse.execute::(&mut ctx)?.into_array(); let result_listview = actual.execute::(&mut ctx)?; @@ -1303,9 +1299,7 @@ mod test { fn test_sparse_fixed_size_list_null_fill() -> VortexResult<()> { // Create a FixedSizeListArray with 3 lists of size 3. let elements = buffer![1i32, 2, 3, 4, 5, 6, 7, 8, 9].into_array(); - let fsl = FixedSizeListArray::try_new(elements, 3, Validity::AllValid, 3) - .unwrap() - .into_array(); + let fsl = FixedSizeListArray::try_new(elements, 3, Validity::AllValid, 3)?.into_array(); let indices = buffer![0u8, 2u8, 3u8].into_array(); let fill_value = Scalar::null(DType::FixedSizeList( @@ -1313,9 +1307,7 @@ mod test { 3, Nullable, )); - let sparse = Sparse::try_new(indices, fsl, 5, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, fsl, 5, fill_value)?.into_array(); let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? @@ -1329,8 +1321,7 @@ mod test { 3, Validity::Array(BoolArray::from_iter([true, false, true, true, false]).into_array()), 5, - ) - .unwrap() + )? .into_array(); assert_arrays_eq!(actual, expected); @@ -1340,9 +1331,7 @@ mod test { #[test] fn test_sparse_fixed_size_list_non_null_fill() -> VortexResult<()> { let elements = buffer![1i32, 2, 3, 4, 5, 6].into_array(); - let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 3) - .unwrap() - .into_array(); + let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 3)?.into_array(); let indices = buffer![0u8, 2u8, 4u8].into_array(); let fill_value = Scalar::fixed_size_list( @@ -1353,9 +1342,7 @@ mod test { ], NonNullable, ); - let sparse = Sparse::try_new(indices, fsl, 6, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, fsl, 6, fill_value)?.into_array(); let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? @@ -1363,8 +1350,7 @@ mod test { // Expected: [1,2], [99,88], [3,4], [99,88], [5,6], [99,88]. let expected_elements = buffer![1i32, 2, 99, 88, 3, 4, 99, 88, 5, 6, 99, 88].into_array(); - let expected = FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 6) - .unwrap() + let expected = FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 6)? .into_array(); assert_arrays_eq!(actual, expected); @@ -1380,8 +1366,7 @@ mod test { 2, Validity::Array(BoolArray::from_iter([true, false, true]).into_array()), 3, - ) - .unwrap() + )? .into_array(); let indices = buffer![1u16, 3u16, 4u16].into_array(); @@ -1393,9 +1378,7 @@ mod test { ], Nullable, ); - let sparse = Sparse::try_new(indices, fsl, 6, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, fsl, 6, fill_value)?.into_array(); let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? @@ -1411,8 +1394,7 @@ mod test { BoolArray::from_iter([true, true, true, false, true, true]).into_array(), ), 6, - ) - .unwrap() + )? .into_array(); assert_arrays_eq!(actual, expected); @@ -1426,9 +1408,7 @@ mod test { // Create patch values: only 3 distinct lists out of 100 total positions. let elements = buffer![10i32, 11, 20, 21, 30, 31].into_array(); - let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 3) - .unwrap() - .into_array(); + let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 3)?.into_array(); // Patches at positions 5, 50, and 95 out of 100. let indices = buffer![5u32, 50, 95].into_array(); @@ -1443,9 +1423,7 @@ mod test { NonNullable, ); - let sparse = Sparse::try_new(indices, fsl, 100, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, fsl, 100, fill_value)?.into_array(); let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? @@ -1477,8 +1455,7 @@ mod test { } let expected_elements = PrimitiveArray::from_iter(expected_elements_vec).into_array(); let expected = - FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 100) - .unwrap() + FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 100)? .into_array(); assert_arrays_eq!(actual, expected); @@ -1489,9 +1466,7 @@ mod test { fn test_sparse_fixed_size_list_single_element() -> VortexResult<()> { // Test with a single element FSL array. let elements = buffer![42i32, 43].into_array(); - let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 1) - .unwrap() - .into_array(); + let fsl = FixedSizeListArray::try_new(elements, 2, Validity::AllValid, 1)?.into_array(); let indices = buffer![0u32].into_array(); let fill_value = Scalar::fixed_size_list( @@ -1502,9 +1477,7 @@ mod test { ], NonNullable, ); - let sparse = Sparse::try_new(indices, fsl, 1, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, fsl, 1, fill_value)?.into_array(); let actual = sparse .execute::(&mut LEGACY_SESSION.create_execution_ctx())? @@ -1512,8 +1485,7 @@ mod test { // Expected: just [42, 43]. let expected_elements = buffer![42i32, 43].into_array(); - let expected = FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 1) - .unwrap() + let expected = FixedSizeListArray::try_new(expected_elements, 2, Validity::NonNullable, 1)? .into_array(); assert_arrays_eq!(actual, expected); @@ -1525,15 +1497,11 @@ mod test { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let elements = buffer![1i32, 2, 1, 2].into_array(); let offsets = buffer![0u8, 1, 2, 3, 4].into_array(); - let lists = ListArray::try_new(elements, offsets, Validity::AllValid) - .unwrap() - .into_array(); + let lists = ListArray::try_new(elements, offsets, Validity::AllValid)?.into_array(); let indices = buffer![0u8, 1u8, 2u8, 3u8].into_array(); let fill_value = Scalar::from(Some(vec![42i32; 252])); // 252 + 4 elements = 256 > u8::MAX - let sparse = Sparse::try_new(indices, lists, 5, fill_value) - .unwrap() - .into_array(); + let sparse = Sparse::try_new(indices, lists, 5, fill_value)?.into_array(); let actual = sparse.execute::(&mut ctx)?.into_array(); let mut expected_elements = buffer_mut![1, 2, 1, 2]; @@ -1542,8 +1510,7 @@ mod test { expected_elements.freeze().into_array(), buffer![0u16, 1, 2, 3, 4, 256].into_array(), Validity::AllValid, - ) - .unwrap() + )? .into_array(); let actual_listview = actual.clone().execute::(&mut ctx)?; @@ -1554,11 +1521,9 @@ mod test { assert_arrays_eq!(&actual, &expected); // Note that the preferred arrow list representation is `List` (not `ListView`). - let arrow_dtype = expected.dtype().to_arrow_dtype().unwrap(); - let actual = actual.execute_arrow(Some(&arrow_dtype), &mut ctx).unwrap(); - let expected = expected - .execute_arrow(Some(&arrow_dtype), &mut ctx) - .unwrap(); + let arrow_dtype = expected.dtype().to_arrow_dtype()?; + let actual = actual.execute_arrow(Some(&arrow_dtype), &mut ctx)?; + let expected = expected.execute_arrow(Some(&arrow_dtype), &mut ctx)?; assert_eq!(actual.data_type(), expected.data_type()); Ok(()) @@ -1602,8 +1567,7 @@ mod test { list_view.into_array(), 10, Scalar::null(list_dtype), - ) - .unwrap(); + )?; // Convert to canonical form - this triggers the function we're testing let canonical = sparse @@ -1677,7 +1641,7 @@ mod test { // - Index 0: [2, 3, 4] (original list 1) // - Index 1: [5] (original list 2) // - Index 2: [6, 7] (original list 3) - let sliced = full_listview.slice(1..4).unwrap(); + let sliced = full_listview.slice(1..4)?; // Create sparse array with indices [0, 1] and length 5 // Expected result: @@ -1688,9 +1652,8 @@ mod test { // - Index 4: null let indices = buffer![0u8, 1].into_array(); // Extract only the values we need from the sliced array - let values = sliced.slice(0..2).unwrap(); - let sparse = - Sparse::try_new(indices, values, 5, Scalar::null(sliced.dtype().clone())).unwrap(); + let values = sliced.slice(0..2)?; + let sparse = Sparse::try_new(indices, values, 5, Scalar::null(sliced.dtype().clone()))?; let canonical = sparse .into_array() diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 27bf37a7bde..52658e53961 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -300,13 +300,10 @@ mod test { array.statistics().compute_is_constant(&mut ctx) ); - let sliced = zigzag.slice(0..2).unwrap(); + let sliced = zigzag.slice(0..2)?; let sliced = sliced.as_::(); assert_eq!( - sliced - .array() - .execute_scalar(sliced.len() - 1, &mut ctx,) - .unwrap(), + sliced.array().execute_scalar(sliced.len() - 1, &mut ctx,)?, Scalar::from(-5i32) ); diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 4e105c4aa61..a90db7eacc7 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -108,7 +108,7 @@ mod tests { )?; let indices = buffer![0, 2].into_array(); - let actual = zigzag.take(indices).unwrap(); + let actual = zigzag.take(indices)?; let expected = zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid).as_view())? .into_array(); @@ -123,7 +123,7 @@ mod tests { )?; let filter_mask = BitBuffer::from(vec![true, false, true]).into(); - let actual = zigzag.filter(filter_mask).unwrap(); + let actual = zigzag.filter(filter_mask)?; let expected = zigzag_encode(PrimitiveArray::new(buffer![-189, 1], Validity::AllValid).as_view())? .into_array(); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e2ce7163390..cb6bd203288 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.90.0" +channel = "1.91.0" components = ["rust-src", "rustfmt", "clippy", "rust-analyzer"] profile = "minimal" \ No newline at end of file diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index d39a700fc02..1e6dcd7014b 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -662,10 +662,8 @@ mod tests { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let dtype = DecimalDType::new(19, 2); - let i100 = - parse_decimal::("100.00", dtype.precision(), dtype.scale()).unwrap(); - let i200 = - parse_decimal::("200.00", dtype.precision(), dtype.scale()).unwrap(); + let i100 = parse_decimal::("100.00", dtype.precision(), dtype.scale())?; + let i200 = parse_decimal::("200.00", dtype.precision(), dtype.scale())?; let sorted = buffer![i100, i200, i200]; let unsorted = buffer![i200, i100, i200]; @@ -689,12 +687,9 @@ mod tests { let mut ctx = LEGACY_SESSION.create_execution_ctx(); let dtype = DecimalDType::new(19, 2); - let i100 = - parse_decimal::("100.00", dtype.precision(), dtype.scale()).unwrap(); - let i200 = - parse_decimal::("200.00", dtype.precision(), dtype.scale()).unwrap(); - let i300 = - parse_decimal::("300.00", dtype.precision(), dtype.scale()).unwrap(); + let i100 = parse_decimal::("100.00", dtype.precision(), dtype.scale())?; + let i200 = parse_decimal::("200.00", dtype.precision(), dtype.scale())?; + let i300 = parse_decimal::("300.00", dtype.precision(), dtype.scale())?; let strict_sorted = buffer![i100, i200, i300]; let sorted = buffer![i100, i200, i200]; diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 78d0c4caa52..e66d46a071c 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -361,16 +361,11 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_valid since all non-empty chunks are all_valid - assert!( - chunked - .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(chunked.all_valid(&mut LEGACY_SESSION.create_execution_ctx())?); assert!( !chunked .into_array() - .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx())? ); Ok(()) @@ -390,16 +385,11 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be all_invalid since all non-empty chunks are all_invalid - assert!( - !chunked - .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(!chunked.all_valid(&mut LEGACY_SESSION.create_execution_ctx())?); assert!( chunked .into_array() - .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx())? ); Ok(()) @@ -419,16 +409,11 @@ mod test { ChunkedArray::try_new(chunks, DType::Primitive(PType::U64, Nullability::Nullable))?; // Should be neither all_valid nor all_invalid - assert!( - !chunked - .all_valid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(!chunked.all_valid(&mut LEGACY_SESSION.create_execution_ctx())?); assert!( !chunked .into_array() - .all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() + .all_invalid(&mut LEGACY_SESSION.create_execution_ctx())? ); Ok(()) diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 7d004ceb765..2ca23ae9a66 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -383,13 +383,10 @@ mod tests { fn test_canonicalize_propagates_stats() -> VortexResult<()> { let scalar = Scalar::bool(true, Nullability::NonNullable); let const_array = ConstantArray::new(scalar, 4).into_array(); - let stats = const_array - .statistics() - .compute_all( - &all::().collect_vec(), - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .unwrap(); + let stats = const_array.statistics().compute_all( + &all::().collect_vec(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; #[expect(deprecated)] let canonical = const_array.to_canonical()?.into_array(); let canonical_stats = canonical.statistics(); diff --git a/vortex-array/src/arrays/dict/compute/min_max.rs b/vortex-array/src/arrays/dict/compute/min_max.rs index 8abd34dbc0a..b390005adcf 100644 --- a/vortex-array/src/arrays/dict/compute/min_max.rs +++ b/vortex-array/src/arrays/dict/compute/min_max.rs @@ -92,42 +92,73 @@ mod tests { Ok(()) } - #[rstest] - #[case::covering( + fn dict_covering() -> DictArray { DictArray::try_new( buffer![0u32, 1, 2, 3, 0, 1].into_array(), buffer![10i32, 20, 30, 40].into_array(), - ).unwrap(), - (10, 40) - )] - #[case::non_covering_duplicates( + ) + .expect("valid test dictionary") + } + + fn dict_non_covering_duplicates() -> DictArray { DictArray::try_new( buffer![1u32, 1, 1, 3, 3].into_array(), buffer![1i32, 2, 3, 4, 5].into_array(), - ).unwrap(), - (2, 4) - )] - #[case::non_covering_gaps( + ) + .expect("valid test dictionary") + } + + fn dict_non_covering_gaps() -> DictArray { DictArray::try_new( buffer![0u32, 2, 4].into_array(), buffer![1i32, 2, 3, 4, 5].into_array(), - ).unwrap(), - (1, 5) - )] - #[case::single(dict_encode(&buffer![42i32].into_array()).unwrap(), (42, 42))] - #[case::nullable_codes( + ) + .expect("valid test dictionary") + } + + fn dict_single() -> DictArray { + dict_encode(&buffer![42i32].into_array()).expect("valid single-value dictionary") + } + + fn dict_nullable_codes() -> DictArray { DictArray::try_new( PrimitiveArray::from_option_iter([Some(0u32), None, Some(1), Some(2)]).into_array(), buffer![10i32, 20, 30].into_array(), - ).unwrap(), - (10, 30) - )] - #[case::nullable_values( + ) + .expect("valid nullable-code dictionary") + } + + fn dict_nullable_values() -> DictArray { dict_encode( - &PrimitiveArray::from_option_iter([Some(1i32), None, Some(2), Some(1), None]).into_array() - ).unwrap(), - (1, 2) - )] + &PrimitiveArray::from_option_iter([Some(1i32), None, Some(2), Some(1), None]) + .into_array(), + ) + .expect("valid nullable-value dictionary") + } + + fn dict_empty() -> DictArray { + DictArray::try_new( + PrimitiveArray::from_iter(Vec::::new()).into_array(), + buffer![10i32, 20, 30].into_array(), + ) + .expect("valid empty dictionary") + } + + fn dict_all_null_codes() -> DictArray { + DictArray::try_new( + PrimitiveArray::from_option_iter([Option::::None, None, None]).into_array(), + buffer![10i32, 20, 30].into_array(), + ) + .expect("valid all-null-code dictionary") + } + + #[rstest] + #[case::covering(dict_covering(), (10, 40))] + #[case::non_covering_duplicates(dict_non_covering_duplicates(), (2, 4))] + #[case::non_covering_gaps(dict_non_covering_gaps(), (1, 5))] + #[case::single(dict_single(), (42, 42))] + #[case::nullable_codes(dict_nullable_codes(), (10, 30))] + #[case::nullable_values(dict_nullable_values(), (1, 2))] fn test_min_max(#[case] dict: DictArray, #[case] expected: (i32, i32)) -> VortexResult<()> { assert_min_max(&dict.into_array(), Some(expected)) } @@ -141,18 +172,8 @@ mod tests { } #[rstest] - #[case::empty( - DictArray::try_new( - PrimitiveArray::from_iter(Vec::::new()).into_array(), - buffer![10i32, 20, 30].into_array(), - ).unwrap() - )] - #[case::all_null_codes( - DictArray::try_new( - PrimitiveArray::from_option_iter([Option::::None, None, None]).into_array(), - buffer![10i32, 20, 30].into_array(), - ).unwrap() - )] + #[case::empty(dict_empty())] + #[case::all_null_codes(dict_all_null_codes())] fn test_min_max_none(#[case] dict: DictArray) -> VortexResult<()> { assert_min_max(&dict.into_array(), None) } diff --git a/vortex-array/src/arrays/listview/rebuild.rs b/vortex-array/src/arrays/listview/rebuild.rs index d2ab4d92df7..416d73bb92a 100644 --- a/vortex-array/src/arrays/listview/rebuild.rs +++ b/vortex-array/src/arrays/listview/rebuild.rs @@ -421,12 +421,12 @@ mod tests { // Verify the data is correct assert_arrays_eq!( - flattened.list_elements_at(0).unwrap(), + flattened.list_elements_at(0)?, PrimitiveArray::from_iter([1i32, 2, 3]) ); assert_arrays_eq!( - flattened.list_elements_at(1).unwrap(), + flattened.list_elements_at(1)?, PrimitiveArray::from_iter([2i32, 3]) ); Ok(()) @@ -454,18 +454,18 @@ mod tests { // Verify nullability is preserved assert_eq!(flattened.dtype().nullability(), Nullability::Nullable); - assert!(flattened.validity()?.is_valid(0).unwrap()); - assert!(!flattened.validity()?.is_valid(1).unwrap()); - assert!(flattened.validity()?.is_valid(2).unwrap()); + assert!(flattened.validity()?.is_valid(0)?); + assert!(!flattened.validity()?.is_valid(1)?); + assert!(flattened.validity()?.is_valid(2)?); // Verify valid lists contain correct data assert_arrays_eq!( - flattened.list_elements_at(0).unwrap(), + flattened.list_elements_at(0)?, PrimitiveArray::from_iter([1i32, 2]) ); assert_arrays_eq!( - flattened.list_elements_at(2).unwrap(), + flattened.list_elements_at(2)?, PrimitiveArray::from_iter([3i32]) ); Ok(()) @@ -500,12 +500,12 @@ mod tests { // Verify the data is correct. assert_arrays_eq!( - trimmed.list_elements_at(0).unwrap(), + trimmed.list_elements_at(0)?, PrimitiveArray::from_iter([1i32, 2]) ); assert_arrays_eq!( - trimmed.list_elements_at(1).unwrap(), + trimmed.list_elements_at(1)?, PrimitiveArray::from_iter([3i32, 4]) ); @@ -513,9 +513,7 @@ mod tests { #[expect(deprecated)] let all_elements = trimmed.elements().to_primitive(); assert_eq!( - all_elements - .execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(), + all_elements.execute_scalar(2, &mut LEGACY_SESSION.create_execution_ctx())?, 97i32.into() ); Ok(()) @@ -557,35 +555,19 @@ mod tests { let exact = rebuilt.rebuild(ListViewRebuildMode::MakeExact)?; // Verify the result is still valid - assert!( - exact - .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - exact - .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !exact - .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !exact - .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(exact.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(exact.is_valid(1, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(!exact.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(!exact.is_valid(3, &mut LEGACY_SESSION.create_execution_ctx())?); // Verify data is preserved assert_arrays_eq!( - exact.list_elements_at(0).unwrap(), + exact.list_elements_at(0)?, PrimitiveArray::from_iter([1i32, 2]) ); assert_arrays_eq!( - exact.list_elements_at(1).unwrap(), + exact.list_elements_at(1)?, PrimitiveArray::from_iter([3i32, 4]) ); Ok(()) @@ -607,7 +589,7 @@ mod tests { let listview = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); let trimmed = listview.rebuild(ListViewRebuildMode::TrimElements)?; assert_arrays_eq!( - trimmed.list_elements_at(1).unwrap(), + trimmed.list_elements_at(1)?, PrimitiveArray::from_iter([30i32, 40]) ); Ok(()) @@ -627,7 +609,7 @@ mod tests { let listview = ListViewArray::new(elements, offsets, sizes, Validity::NonNullable); let trimmed = listview.rebuild(ListViewRebuildMode::TrimElements)?; assert_arrays_eq!( - trimmed.list_elements_at(1).unwrap(), + trimmed.list_elements_at(1)?, PrimitiveArray::from_iter([30i32, 40]) ); Ok(()) diff --git a/vortex-array/src/arrays/listview/tests/basic.rs b/vortex-array/src/arrays/listview/tests/basic.rs index 8925d97567f..61664c2f608 100644 --- a/vortex-array/src/arrays/listview/tests/basic.rs +++ b/vortex-array/src/arrays/listview/tests/basic.rs @@ -127,7 +127,7 @@ fn test_from_list_array() -> VortexResult<()> { let elements = buffer![1i32, 2, 3, 4, 5, 6, 7].into_array(); let validity = Validity::from_iter([true, false, true]); - let list_array = ListArray::try_new(elements, offsets, validity).unwrap(); + let list_array = ListArray::try_new(elements, offsets, validity)?; let mut ctx = LEGACY_SESSION.create_execution_ctx(); let list_view = list_view_from_list(list_array, &mut ctx)?; @@ -135,26 +135,14 @@ fn test_from_list_array() -> VortexResult<()> { // Check first list. assert_arrays_eq!( - list_view.list_elements_at(0).unwrap(), + list_view.list_elements_at(0)?, PrimitiveArray::from_iter([1i32, 2]) ); // Check validity is preserved. - assert!( - list_view - .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - list_view - .is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - list_view - .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(list_view.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(list_view.is_invalid(1, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(list_view.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx())?); // Check third list. assert_arrays_eq!( diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index a2db8c07f1a..2721ba9b519 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -46,7 +46,7 @@ fn test_dtype_nullability_with_nullable_child() { fn test_canonical_dtype_matches_array_dtype() -> VortexResult<()> { // The canonical form should have the same nullability as the array's dtype. let child = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); - let array = MaskedArray::try_new(child, Validity::AllValid).unwrap(); + let array = MaskedArray::try_new(child, Validity::AllValid)?; let canonical = array .clone() diff --git a/vortex-array/src/arrays/masked/vtable/canonical.rs b/vortex-array/src/arrays/masked/vtable/canonical.rs index be94c1cfddb..90bd999b8ed 100644 --- a/vortex-array/src/arrays/masked/vtable/canonical.rs +++ b/vortex-array/src/arrays/masked/vtable/canonical.rs @@ -15,21 +15,25 @@ mod tests { use crate::dtype::Nullability; use crate::validity::Validity; - #[rstest] - #[case( + fn masked_all_valid() -> MaskedArray { MaskedArray::try_new( PrimitiveArray::from_iter([1i32, 2, 3]).into_array(), - Validity::AllValid - ).unwrap(), - Nullability::Nullable - )] - #[case( + Validity::AllValid, + ) + .expect("valid masked array") + } + + fn masked_with_nulls() -> MaskedArray { MaskedArray::try_new( PrimitiveArray::from_iter([1i32, 2, 3]).into_array(), - Validity::from_iter([true, false, true]) - ).unwrap(), - Nullability::Nullable - )] + Validity::from_iter([true, false, true]), + ) + .expect("valid masked array") + } + + #[rstest] + #[case(masked_all_valid(), Nullability::Nullable)] + #[case(masked_with_nulls(), Nullability::Nullable)] fn test_canonical_nullability( #[case] array: MaskedArray, #[case] expected_nullability: Nullability, @@ -48,8 +52,7 @@ mod tests { let array = MaskedArray::try_new( PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]).into_array(), Validity::from_iter([true, false, true, false, true]), - ) - .unwrap(); + )?; let canonical = array .into_array() @@ -58,29 +61,12 @@ mod tests { // Check that null positions match validity. let mut ctx = LEGACY_SESSION.create_execution_ctx(); - assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); - assert!( - prim.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !prim - .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - prim.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !prim - .is_valid(3, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - prim.is_valid(4, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert_eq!(prim.valid_count(&mut ctx)?, 3); + assert!(prim.is_valid(0, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(!prim.is_valid(1, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(prim.is_valid(2, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(!prim.is_valid(3, &mut LEGACY_SESSION.create_execution_ctx())?); + assert!(prim.is_valid(4, &mut LEGACY_SESSION.create_execution_ctx())?); Ok(()) } @@ -89,8 +75,7 @@ mod tests { let array = MaskedArray::try_new( PrimitiveArray::from_iter([10i32, 20, 30]).into_array(), Validity::AllValid, - ) - .unwrap(); + )?; let canonical = array .into_array() @@ -99,8 +84,7 @@ mod tests { assert_eq!( canonical .into_array() - .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) - .unwrap(), + .valid_count(&mut LEGACY_SESSION.create_execution_ctx())?, 3 ); Ok(()) diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index abd54574940..17f5333e52d 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -160,6 +160,7 @@ impl ApplyPatches<'_, V> { mod tests { use vortex_buffer::buffer; use vortex_error::VortexResult; + use vortex_error::vortex_err; use crate::ExecutionCtx; use crate::IntoArray; @@ -261,7 +262,7 @@ mod tests { let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx)? .into_array() .try_downcast::() - .unwrap(); + .map_err(|_| vortex_err!("expected patched array"))?; let rhs = ConstantArray::new(subnormal, 512).into_array(); @@ -271,7 +272,7 @@ mod tests { CompareOperator::Eq, &mut ctx, )? - .unwrap(); + .ok_or_else(|| vortex_err!("expected compare result"))?; let expected = BoolArray::from_indices(512, [510], Validity::NonNullable).into_array(); @@ -295,7 +296,7 @@ mod tests { let lhs = Patched::from_array_and_patches(lhs, &patches, &mut ctx)? .into_array() .try_downcast::() - .unwrap(); + .map_err(|_| vortex_err!("expected patched array"))?; let rhs = ConstantArray::new(0.0f32, 10).into_array(); @@ -305,7 +306,7 @@ mod tests { CompareOperator::Eq, &mut ctx, )? - .unwrap(); + .ok_or_else(|| vortex_err!("expected compare result"))?; let expected = BoolArray::from_indices(10, [7], Validity::NonNullable).into_array(); diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-array/src/arrow/executor/dictionary.rs index f0808d88818..99e315bd8f7 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-array/src/arrow/executor/dictionary.rs @@ -168,6 +168,24 @@ mod tests { array.execute_arrow(Some(dt), &mut LEGACY_SESSION.create_execution_ctx()) } + fn dict_basic_input() -> crate::ArrayRef { + DictArray::try_new( + buffer![0u8, 1, 0].into_array(), + VarBinViewArray::from_iter_str(["a", "b"]).into_array(), + ) + .expect("valid dictionary input") + .into_array() + } + + fn dict_with_null_codes_input() -> crate::ArrayRef { + DictArray::try_new( + PrimitiveArray::from_option_iter(vec![Some(0u8), None, Some(1)]).into_array(), + VarBinViewArray::from_iter_str(["a", "b"]).into_array(), + ) + .expect("valid dictionary input with null codes") + .into_array() + } + #[rstest] #[case::constant_null( ConstantArray::new(Scalar::null(DType::Utf8(Nullable)), 4).into_array(), @@ -180,18 +198,12 @@ mod tests { Arc::new(vec![Some("hello"); 5].into_iter().collect::>()) as arrow_array::ArrayRef, )] #[case::dict_basic( - DictArray::try_new( - buffer![0u8, 1, 0].into_array(), - VarBinViewArray::from_iter_str(["a", "b"]).into_array(), - ).unwrap().into_array(), + dict_basic_input(), dict_type(DataType::UInt8, DataType::Utf8), Arc::new(vec![Some("a"), Some("b"), Some("a")].into_iter().collect::>()) as arrow_array::ArrayRef, )] #[case::dict_with_null_codes( - DictArray::try_new( - PrimitiveArray::from_option_iter(vec![Some(0u8), None, Some(1)]).into_array(), - VarBinViewArray::from_iter_str(["a", "b"]).into_array(), - ).unwrap().into_array(), + dict_with_null_codes_input(), dict_type(DataType::UInt8, DataType::Utf8), Arc::new(vec![Some("a"), None, Some("b")].into_iter().collect::>()) as arrow_array::ArrayRef, )] diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-array/src/arrow/executor/run_end.rs index b9424bc866b..b579cab7e4d 100644 --- a/vortex-array/src/arrow/executor/run_end.rs +++ b/vortex-array/src/arrow/executor/run_end.rs @@ -205,6 +205,7 @@ mod tests { use arrow_schema::Field; use rstest::rstest; use vortex_error::VortexResult; + use vortex_error::vortex_err; use vortex_session::VortexSession; use crate::IntoArray; @@ -232,22 +233,36 @@ mod tests { array.execute_arrow(Some(dt), &mut SESSION.create_execution_ctx()) } + fn constant_i32_with_i16_ends() -> arrow_array::ArrayRef { + Arc::new( + RunArray::::try_new( + &Int16Array::from(vec![5i16]), + &Int32Array::from(vec![42]), + ) + .expect("valid run-end test array"), + ) as arrow_array::ArrayRef + } + + fn constant_f64_with_i64_ends() -> arrow_array::ArrayRef { + Arc::new( + RunArray::::try_new( + &Int64Array::from(vec![7i64]), + &arrow_array::Float64Array::from(vec![1.5]), + ) + .expect("valid run-end test array"), + ) as arrow_array::ArrayRef + } + #[rstest] #[case::i32_with_i16_ends( ConstantArray::new(Scalar::from(42i32), 5).into_array(), ree_type(DataType::Int16, DataType::Int32), - Arc::new(RunArray::::try_new( - &Int16Array::from(vec![5i16]), - &Int32Array::from(vec![42]), - ).unwrap()) as arrow_array::ArrayRef, + constant_i32_with_i16_ends(), )] #[case::f64_with_i64_ends( ConstantArray::new(Scalar::from(1.5f64), 7).into_array(), ree_type(DataType::Int64, DataType::Float64), - Arc::new(RunArray::::try_new( - &Int64Array::from(vec![7i64]), - &arrow_array::Float64Array::from(vec![1.5]), - ).unwrap()) as arrow_array::ArrayRef, + constant_f64_with_i64_ends(), )] #[case::null( ConstantArray::new(Scalar::null(DType::Primitive(PType::I32, Nullable)), 4).into_array(), @@ -312,9 +327,13 @@ mod tests { let ree = result .as_any() .downcast_ref::>() - .unwrap(); + .ok_or_else(|| vortex_err!("expected Int32 run-end array"))?; assert_eq!(ree.run_ends().values(), expected_ends); - let values = ree.values().as_any().downcast_ref::().unwrap(); + let values = ree + .values() + .as_any() + .downcast_ref::() + .ok_or_else(|| vortex_err!("expected Int64 values"))?; assert_eq!(values.values(), expected_values); Ok(()) } diff --git a/vortex-array/src/extension/uuid/vtable.rs b/vortex-array/src/extension/uuid/vtable.rs index 283de266684..cd4d4d971ae 100644 --- a/vortex-array/src/extension/uuid/vtable.rs +++ b/vortex-array/src/extension/uuid/vtable.rs @@ -312,8 +312,7 @@ mod tests { version: Some(Version::Random), }, uuid_storage_dtype(Nullability::NonNullable), - ) - .unwrap(); + )?; let storage_value = uuid_storage_scalar(&v4_uuid); let result = Uuid::unpack_native(&ext_dtype, &storage_value)?; @@ -330,8 +329,7 @@ mod tests { let ext_dtype = ExtDType::try_new( UuidMetadata::default(), uuid_storage_dtype(Nullability::NonNullable), - ) - .unwrap(); + )?; let storage_value = uuid_storage_scalar(&v4_uuid); let result = Uuid::unpack_native(&ext_dtype, &storage_value)?; diff --git a/vortex-array/src/scalar/typed_view/utf8.rs b/vortex-array/src/scalar/typed_view/utf8.rs index b07201a483d..acf0ab9f407 100644 --- a/vortex-array/src/scalar/typed_view/utf8.rs +++ b/vortex-array/src/scalar/typed_view/utf8.rs @@ -159,7 +159,7 @@ mod private { impl Sealed for BufferString {} impl StringLike for BufferString { - #[expect(clippy::unwrap_in_result, clippy::expect_used)] + #[expect(clippy::expect_used)] fn increment(self) -> Result { if self.is_empty() { return Err(self); diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index 4424bdd6c2a..577e3eb3d2f 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -858,9 +858,8 @@ mod tests { // WHEN value = 1 THEN nullable(20) -- Nullable // ELSE 0 -- NonNullable // → result must be Nullable(i32) - let test_array = StructArray::from_fields(&[("value", buffer![0i32, 1, 2].into_array())]) - .unwrap() - .into_array(); + let test_array = + StructArray::from_fields(&[("value", buffer![0i32, 1, 2].into_array())])?.into_array(); let nullable_20 = Scalar::from(20i32).cast(&DType::Primitive(PType::I32, Nullability::Nullable))?; @@ -1117,8 +1116,7 @@ mod tests { fn test_evaluate_nary_string_output() -> VortexResult<()> { // Exercises merge_case_branches with a non-primitive (Utf8) builder. let test_array = - StructArray::from_fields(&[("value", buffer![1i32, 2, 3, 4].into_array())]) - .unwrap() + StructArray::from_fields(&[("value", buffer![1i32, 2, 3, 4].into_array())])? .into_array(); // CASE WHEN value > 2 THEN 'high' WHEN value > 0 THEN 'low' ELSE 'none' END diff --git a/vortex-bench/src/polarsignals/benchmark.rs b/vortex-bench/src/polarsignals/benchmark.rs index 16ba6a9886b..0ef7b4246e6 100644 --- a/vortex-bench/src/polarsignals/benchmark.rs +++ b/vortex-bench/src/polarsignals/benchmark.rs @@ -119,7 +119,7 @@ impl Benchmark for PolarSignalsBenchmark { )] } - #[expect(clippy::expect_used, clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] fn pattern(&self, _table_name: &str, format: Format) -> Option { Some( format!("{FILE_NAME}.{}", format.ext()) diff --git a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs index 0fa7f488b25..58c4b6aed51 100644 --- a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs +++ b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs @@ -168,7 +168,7 @@ impl Benchmark for StatPopGenBenchmark { vec![TableSpec::new("statpopgen", None)] } - #[expect(clippy::expect_used, clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] fn pattern(&self, _table_name: &str, format: Format) -> Option { Some( format!("{}.{}", Self::FILE_NAME, format.ext()) diff --git a/vortex-bench/src/tpcds/tpcds_benchmark.rs b/vortex-bench/src/tpcds/tpcds_benchmark.rs index 3ad60876521..d44b3645c57 100644 --- a/vortex-bench/src/tpcds/tpcds_benchmark.rs +++ b/vortex-bench/src/tpcds/tpcds_benchmark.rs @@ -124,7 +124,7 @@ impl Benchmark for TpcDsBenchmark { ] } - #[expect(clippy::expect_used, clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] fn pattern(&self, table_name: &str, format: Format) -> Option { Some( format!("{}.{}", table_name, format.ext()) diff --git a/vortex-bench/src/tpch/benchmark.rs b/vortex-bench/src/tpch/benchmark.rs index 71290f860e0..c9d85f2be9b 100644 --- a/vortex-bench/src/tpch/benchmark.rs +++ b/vortex-bench/src/tpch/benchmark.rs @@ -139,7 +139,7 @@ impl Benchmark for TpcHBenchmark { ] } - #[expect(clippy::expect_used, clippy::unwrap_in_result)] + #[expect(clippy::expect_used)] fn pattern(&self, table_name: &str, format: Format) -> Option { Some( format!("{}_*.{}", table_name, format.ext()) diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 46aa35d0465..56df2f21d45 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -664,7 +664,12 @@ mod tests { cuda_ctx: &CudaExecutionCtx, data: &[u32], ) -> VortexResult<(u64, Arc>)> { - let device_buf = Arc::new(cuda_ctx.stream().clone_htod(data).expect("htod")); + let device_buf = Arc::new( + cuda_ctx + .stream() + .clone_htod(data) + .map_err(|e| vortex_err!("htod: {e}"))?, + ); let (ptr, _) = device_buf.device_ptr(cuda_ctx.stream()); Ok((ptr, device_buf)) } @@ -732,12 +737,15 @@ mod tests { cuda_ctx .stream() .clone_htod(plan.as_bytes()) - .expect("copy plan to device"), + .map_err(|e| vortex_err!("copy plan to device: {e}"))?, ); let (plan_ptr, record_plan) = device_plan.device_ptr(cuda_ctx.stream()); let array_len_u64 = output_len as u64; - cuda_ctx.stream().synchronize().expect("sync"); + cuda_ctx + .stream() + .synchronize() + .map_err(|e| vortex_err!("sync: {e}"))?; let cuda_function = cuda_ctx .load_function("dynamic_dispatch", &[PType::U32]) @@ -754,14 +762,16 @@ mod tests { shared_mem_bytes, }; unsafe { - launch_builder.launch(config).expect("kernel launch"); + launch_builder + .launch(config) + .map_err(|e| vortex_err!("kernel launch: {e}"))?; } drop((record_output, record_plan)); - Ok(cuda_ctx + cuda_ctx .stream() .clone_dtoh(&output_buf.as_view::()) - .expect("copy back")) + .map_err(|e| vortex_err!("copy back: {e}")) } fn run_dispatch_plan_f32( @@ -1869,7 +1879,12 @@ mod tests { let i8_values: Vec = vec![-1, -2, -3, 127, -128, 0, 1, 42]; let len = i8_values.len(); - let device_buf = Arc::new(cuda_ctx.stream().clone_htod(&i8_values).expect("htod")); + let device_buf = Arc::new( + cuda_ctx + .stream() + .clone_htod(&i8_values) + .map_err(|e| vortex_err!("htod: {e}"))?, + ); let (input_ptr, _) = device_buf.device_ptr(cuda_ctx.stream()); // Build a single-stage LOAD plan: source ptype = I8, output ptype = U32. @@ -1902,7 +1917,12 @@ mod tests { let i16_values: Vec = vec![-1, -256, -32768, 32767, 0, 1, -100, 12345]; let len = i16_values.len(); - let device_buf = Arc::new(cuda_ctx.stream().clone_htod(&i16_values).expect("htod")); + let device_buf = Arc::new( + cuda_ctx + .stream() + .clone_htod(&i16_values) + .map_err(|e| vortex_err!("htod: {e}"))?, + ); let (input_ptr, _) = device_buf.device_ptr(cuda_ctx.stream()); let plan = CudaDispatchPlan::new( diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index 58873a8d7eb..da03cd4254c 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -192,15 +192,12 @@ mod tests { let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - new_exporter(&arr, &ConversionCache::default()) - .unwrap() - .export( - 0, - 2, - chunk.get_vector_mut(0), - &mut SESSION.create_execution_ctx(), - ) - .unwrap(); + new_exporter(&arr, &ConversionCache::default())?.export( + 0, + 2, + chunk.get_vector_mut(0), + &mut SESSION.create_execution_ctx(), + )?; chunk.set_len(2); assert_eq!( @@ -223,10 +220,12 @@ mod tests { let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); let mut ctx = ExecutionCtx::new(VortexSession::default()); - new_exporter_with_flatten(&arr, &ConversionCache::default(), &mut ctx, false) - .unwrap() - .export(0, 2, chunk.get_vector_mut(0), &mut ctx) - .unwrap(); + new_exporter_with_flatten(&arr, &ConversionCache::default(), &mut ctx, false)?.export( + 0, + 2, + chunk.get_vector_mut(0), + &mut ctx, + )?; chunk.set_len(2); assert_eq!( @@ -248,15 +247,12 @@ mod tests { let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - new_exporter(&arr, &ConversionCache::default()) - .unwrap() - .export( - 0, - 3, - chunk.get_vector_mut(0), - &mut SESSION.create_execution_ctx(), - ) - .unwrap(); + new_exporter(&arr, &ConversionCache::default())?.export( + 0, + 3, + chunk.get_vector_mut(0), + &mut SESSION.create_execution_ctx(), + )?; chunk.set_len(3); // some-invalid codes cannot be exported as a dictionary. @@ -271,10 +267,12 @@ mod tests { DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); let mut ctx = SESSION.create_execution_ctx(); - new_array_exporter(arr.into_array(), &ConversionCache::default(), &mut ctx) - .unwrap() - .export(0, 3, flat_chunk.get_vector_mut(0), &mut ctx) - .unwrap(); + new_array_exporter(arr.into_array(), &ConversionCache::default(), &mut ctx)?.export( + 0, + 3, + flat_chunk.get_vector_mut(0), + &mut ctx, + )?; flat_chunk.set_len(3); assert_eq!( @@ -297,15 +295,12 @@ mod tests { let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); - new_exporter(&arr, &ConversionCache::default()) - .unwrap() - .export( - 0, - 0, - chunk.get_vector_mut(0), - &mut SESSION.create_execution_ctx(), - ) - .unwrap(); + new_exporter(&arr, &ConversionCache::default())?.export( + 0, + 0, + chunk.get_vector_mut(0), + &mut SESSION.create_execution_ctx(), + )?; chunk.set_len(0); assert_eq!( diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index e17f63e2e2a..557795f45f3 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -161,7 +161,7 @@ mod tests { chunk.set_len(3); assert_eq!( - format!("{}", String::try_from(&*chunk).unwrap()), + format!("{}", String::try_from(&*chunk)?), r#"Chunk - [1 Columns] - CONSTANT VARCHAR: 3 = [ NULL] "# @@ -181,7 +181,7 @@ mod tests { chunk.set_len(3); assert_eq!( - format!("{}", String::try_from(&*chunk).unwrap()), + format!("{}", String::try_from(&*chunk)?), r#"Chunk - [1 Columns] - CONSTANT VARCHAR: 3 = [ NULL] "# @@ -203,7 +203,7 @@ mod tests { chunk.set_len(3); assert_eq!( - format!("{}", String::try_from(&*chunk).unwrap()), + format!("{}", String::try_from(&*chunk)?), r#"Chunk - [1 Columns] - FLAT VARCHAR: 3 = [ NULL, NULL, Hi] "# diff --git a/vortex-duckdb/src/multi_file.rs b/vortex-duckdb/src/multi_file.rs index ae89c5ef08c..3f99a854a22 100644 --- a/vortex-duckdb/src/multi_file.rs +++ b/vortex-duckdb/src/multi_file.rs @@ -180,7 +180,7 @@ mod tests { #[test] fn test_parse_glob_url_absolute_glob_path() -> VortexResult<()> { - let tmpdir = tempfile::tempdir().unwrap(); + let tmpdir = tempfile::tempdir()?; let glob = format!("{}/*.vortex", tmpdir.path().display()); let url = parse_glob_url(&glob)?; assert_eq!(url.scheme(), "file"); @@ -190,9 +190,11 @@ mod tests { #[test] fn test_parse_glob_url_absolute_existing_path() -> VortexResult<()> { - let tmpfile = tempfile::NamedTempFile::new().unwrap(); - let canonical = std::fs::canonicalize(tmpfile.path()).unwrap(); - let path_str = canonical.to_str().unwrap(); + let tmpfile = tempfile::NamedTempFile::new()?; + let canonical = std::fs::canonicalize(tmpfile.path())?; + let path_str = canonical + .to_str() + .ok_or_else(|| vortex_err!("canonical path is not valid UTF-8"))?; let url = parse_glob_url(path_str)?; assert_eq!(url.scheme(), "file"); assert_eq!(url.path(), path_str); @@ -203,8 +205,13 @@ mod tests { fn test_parse_glob_url_relative_path() -> VortexResult<()> { // Create a tempfile in the current working directory so we can refer to it // by a relative name (just the filename, without any directory component). - let tmpfile = tempfile::NamedTempFile::new_in(".").unwrap(); - let filename = tmpfile.path().file_name().unwrap().to_str().unwrap(); + let tmpfile = tempfile::NamedTempFile::new_in(".")?; + let filename = tmpfile + .path() + .file_name() + .ok_or_else(|| vortex_err!("temp file missing file name"))? + .to_str() + .ok_or_else(|| vortex_err!("temp file name is not valid UTF-8"))?; let url = parse_glob_url(filename)?; assert_eq!(url.scheme(), "file"); @@ -218,8 +225,13 @@ mod tests { fn test_parse_glob_url_relative_glob_path() -> VortexResult<()> { // A relative path with a glob character (e.g. `./data/*.vortex`) must also resolve // correctly. - let tmpdir = tempfile::tempdir_in(".").unwrap(); - let dir_name = tmpdir.path().file_name().unwrap().to_str().unwrap(); + let tmpdir = tempfile::tempdir_in(".")?; + let dir_name = tmpdir + .path() + .file_name() + .ok_or_else(|| vortex_err!("temp dir missing file name"))? + .to_str() + .ok_or_else(|| vortex_err!("temp dir name is not valid UTF-8"))?; let glob = format!("./{dir_name}/*.vortex"); let url = parse_glob_url(&glob)?; assert_eq!(url.scheme(), "file"); @@ -241,8 +253,13 @@ mod tests { fn test_parse_glob_url_parent_relative_path() -> VortexResult<()> { // A path starting with `..` must be resolved to an absolute path without // percent-encoding the `..` component in the resulting URL. - let tmpfile = tempfile::NamedTempFile::new_in("..").unwrap(); - let filename = tmpfile.path().file_name().unwrap().to_str().unwrap(); + let tmpfile = tempfile::NamedTempFile::new_in("..")?; + let filename = tmpfile + .path() + .file_name() + .ok_or_else(|| vortex_err!("temp file missing file name"))? + .to_str() + .ok_or_else(|| vortex_err!("temp file name is not valid UTF-8"))?; let relative = format!("../{filename}"); let url = parse_glob_url(&relative)?; diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index 84dfbdd93ce..d6a372ca19e 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -669,7 +669,7 @@ mod test { let runtime = SingleThreadRuntime::default(); let session = crate::scan::test::session_with_handle(runtime.handle()); - let stream = ScanBuilder::new(session, reader).into_stream().unwrap(); + let stream = ScanBuilder::new(session, reader).into_stream()?; let mut iter = runtime.block_on_stream(stream); let mut values = Vec::new(); diff --git a/vortex-metrics/src/histogram.rs b/vortex-metrics/src/histogram.rs index d39e7816b11..ea4728ce8d7 100644 --- a/vortex-metrics/src/histogram.rs +++ b/vortex-metrics/src/histogram.rs @@ -32,7 +32,6 @@ impl Histogram { /// Returns the estimated quantile value, which must be in the [0.0, 1.0] range, will panic otherwise. /// Returns `None` if the histogram is empty. #[expect(clippy::expect_used)] - #[expect(clippy::unwrap_in_result)] pub fn quantile(&self, quantile: f64) -> Option { assert!( (0.0..=1.0).contains(&quantile), diff --git a/vortex-metrics/src/timer.rs b/vortex-metrics/src/timer.rs index 20be2f7d69c..619d7ceb178 100644 --- a/vortex-metrics/src/timer.rs +++ b/vortex-metrics/src/timer.rs @@ -43,7 +43,6 @@ impl Timer { /// Returns the estimated quantile value, which must be in the [0.0, 1.0] range, will panic otherwise. /// Returns `None` if the timer is empty. #[expect(clippy::expect_used)] - #[expect(clippy::unwrap_in_result)] pub fn quantile(&self, quantile: f64) -> Option { assert!( (0.0..=1.0).contains(&quantile), diff --git a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs index dbe04f2e606..db013cdfd37 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexResult; +use vortex_error::vortex_err; use super::*; use crate::encodings::turboquant::turboquant_encode_unchecked; @@ -195,8 +196,7 @@ fn all_zero_vectors_roundtrip() -> VortexResult<()> { let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); let fsl = FixedSizeListArray::try_new( elements.into_array(), - dim.try_into() - .expect("somehow got dimension greater than u32::MAX"), + dim.try_into().map_err(|e| vortex_err!("{e}"))?, Validity::NonNullable, num_rows, )?; @@ -245,7 +245,7 @@ fn f64_input_encodes_successfully() -> VortexResult<()> { let num_rows = 10; let dim = 128; let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f64, 1.0).unwrap(); + let normal = Normal::new(0.0f64, 1.0).map_err(|e| vortex_err!("{e}"))?; let mut buf = BufferMut::::with_capacity(num_rows * dim); for _ in 0..(num_rows * dim) { @@ -254,7 +254,7 @@ fn f64_input_encodes_successfully() -> VortexResult<()> { let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); let fsl = FixedSizeListArray::try_new( elements.into_array(), - dim.try_into().unwrap(), + dim.try_into().map_err(|e| vortex_err!("{e}"))?, Validity::NonNullable, num_rows, )?; @@ -278,7 +278,7 @@ fn f16_input_encodes_successfully() -> VortexResult<()> { let num_rows = 10; let dim = 128; let mut rng = StdRng::seed_from_u64(99); - let normal = Normal::new(0.0f32, 1.0).unwrap(); + let normal = Normal::new(0.0f32, 1.0).map_err(|e| vortex_err!("{e}"))?; let mut buf = BufferMut::::with_capacity(num_rows * dim); for _ in 0..(num_rows * dim) { @@ -287,7 +287,7 @@ fn f16_input_encodes_successfully() -> VortexResult<()> { let elements = PrimitiveArray::new::(buf.freeze(), Validity::NonNullable); let fsl = FixedSizeListArray::try_new( elements.into_array(), - dim.try_into().unwrap(), + dim.try_into().map_err(|e| vortex_err!("{e}"))?, Validity::NonNullable, num_rows, )?; diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index f325bf2a205..b39b010082c 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -704,16 +704,8 @@ mod tests { } #[rstest] - #[case::vector( - vector_array(3, &[1.0, 0.0, 0.0, 3.0, 4.0, 0.0]).unwrap(), - vector_array(3, &[0.0, 1.0, 0.0, 3.0, 4.0, 0.0]).unwrap(), - 2, - )] - #[case::fixed_shape_tensor( - tensor_array(&[2], &[1.0, 0.0, 3.0, 4.0]).unwrap(), - tensor_array(&[2], &[0.0, 1.0, 3.0, 4.0]).unwrap(), - 2, - )] + #[case::vector(cosine_vector_lhs(), cosine_vector_rhs(), 2)] + #[case::fixed_shape_tensor(cosine_tensor_lhs(), cosine_tensor_rhs(), 2)] fn serde_round_trip( #[case] lhs: ArrayRef, #[case] rhs: ArrayRef, @@ -741,4 +733,20 @@ mod tests { assert_eq!(recovered.encoding_id(), original.encoding_id()); Ok(()) } + + fn cosine_vector_lhs() -> ArrayRef { + vector_array(3, &[1.0, 0.0, 0.0, 3.0, 4.0, 0.0]).expect("valid vector array") + } + + fn cosine_vector_rhs() -> ArrayRef { + vector_array(3, &[0.0, 1.0, 0.0, 3.0, 4.0, 0.0]).expect("valid vector array") + } + + fn cosine_tensor_lhs() -> ArrayRef { + tensor_array(&[2], &[1.0, 0.0, 3.0, 4.0]).expect("valid tensor array") + } + + fn cosine_tensor_rhs() -> ArrayRef { + tensor_array(&[2], &[0.0, 1.0, 3.0, 4.0]).expect("valid tensor array") + } } diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 7bff5165edb..5b04f2b0bd6 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -839,16 +839,8 @@ mod tests { } #[rstest] - #[case::vector( - vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), - vector_array(3, &[7.0, 8.0, 9.0, 10.0, 11.0, 12.0]).unwrap(), - 2, - )] - #[case::fixed_shape_tensor( - tensor_array(&[2], &[1.0, 2.0, 3.0, 4.0]).unwrap(), - tensor_array(&[2], &[5.0, 6.0, 7.0, 8.0]).unwrap(), - 2, - )] + #[case::vector(inner_product_vector_lhs(), inner_product_vector_rhs(), 2)] + #[case::fixed_shape_tensor(inner_product_tensor_lhs(), inner_product_tensor_rhs(), 2)] fn serde_round_trip( #[case] lhs: ArrayRef, #[case] rhs: ArrayRef, @@ -877,6 +869,22 @@ mod tests { Ok(()) } + fn inner_product_vector_lhs() -> ArrayRef { + vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).expect("valid vector array") + } + + fn inner_product_vector_rhs() -> ArrayRef { + vector_array(3, &[7.0, 8.0, 9.0, 10.0, 11.0, 12.0]).expect("valid vector array") + } + + fn inner_product_tensor_lhs() -> ArrayRef { + tensor_array(&[2], &[1.0, 2.0, 3.0, 4.0]).expect("valid tensor array") + } + + fn inner_product_tensor_rhs() -> ArrayRef { + tensor_array(&[2], &[5.0, 6.0, 7.0, 8.0]).expect("valid tensor array") + } + // ---- Tests for the `SorfTransform + constant` and `Dict + constant` fast paths ---- #[allow( diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index fa6e787eeac..9da97959075 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -1113,8 +1113,8 @@ mod tests { /// inherits the input's nullability, giving us two different per-child nullabilities to /// round-trip. #[rstest] - #[case::vector(vector_array(3, &[3.0, 4.0, 0.0, 0.0, 0.0, 0.0]).unwrap())] - #[case::fixed_shape_tensor(tensor_array(&[2, 2], &[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0]).unwrap())] + #[case::vector(l2_denorm_vector_input())] + #[case::fixed_shape_tensor(l2_denorm_tensor_input())] fn serde_round_trip(#[case] input: ArrayRef) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let original = normalize_as_l2_denorm(input, &mut ctx)?.into_array(); @@ -1141,4 +1141,13 @@ mod tests { assert_eq!(recovered.encoding_id(), original.encoding_id()); Ok(()) } + + fn l2_denorm_vector_input() -> ArrayRef { + vector_array(3, &[3.0, 4.0, 0.0, 0.0, 0.0, 0.0]).expect("valid vector array") + } + + fn l2_denorm_tensor_input() -> ArrayRef { + tensor_array(&[2, 2], &[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0]) + .expect("valid tensor array") + } } diff --git a/vortex-tensor/src/scalar_fns/l2_norm.rs b/vortex-tensor/src/scalar_fns/l2_norm.rs index 5a8172939b7..5d741eef55e 100644 --- a/vortex-tensor/src/scalar_fns/l2_norm.rs +++ b/vortex-tensor/src/scalar_fns/l2_norm.rs @@ -418,8 +418,8 @@ mod tests { } #[rstest] - #[case::fixed_shape_tensor(tensor_array(&[3], &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] - #[case::vector(vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(), 2)] + #[case::fixed_shape_tensor(l2_norm_tensor_child(), 2)] + #[case::vector(l2_norm_vector_child(), 2)] fn serde_round_trip(#[case] child: ArrayRef, #[case] len: usize) -> VortexResult<()> { let original = L2Norm::try_new_array(child.clone(), len)?.into_array(); @@ -443,4 +443,12 @@ mod tests { assert_eq!(recovered.encoding_id(), original.encoding_id()); Ok(()) } + + fn l2_norm_tensor_child() -> ArrayRef { + tensor_array(&[3], &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).expect("valid tensor array") + } + + fn l2_norm_vector_child() -> ArrayRef { + vector_array(3, &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).expect("valid vector array") + } } diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 92ead274bcb..97bc91d3759 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -241,6 +241,7 @@ pub mod test_helpers { use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; + use vortex_error::vortex_err; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::types::fixed_shape::FixedShapeTensor; @@ -270,7 +271,12 @@ pub mod test_helpers { /// The number of rows is inferred from the total element count divided by the product of the /// shape dimensions. For 0-dimensional tensors (scalar), each element is one row. pub fn tensor_array(shape: &[usize], elements: &[T]) -> VortexResult { - let list_size: u32 = shape.iter().product::().max(1).try_into().unwrap(); + let list_size: u32 = shape + .iter() + .product::() + .max(1) + .try_into() + .map_err(|e| vortex_err!("{e}"))?; let storage = flat_fsl(elements, list_size); let metadata = FixedShapeTensorMetadata::new(shape.to_vec()); let ext_dtype = diff --git a/vortex-web/crate/Cargo.toml b/vortex-web/crate/Cargo.toml index 434932c1d25..1b65fe832eb 100644 --- a/vortex-web/crate/Cargo.toml +++ b/vortex-web/crate/Cargo.toml @@ -2,7 +2,7 @@ name = "vortex-web-wasm" version = "0.1.0" edition = "2024" -rust-version = "1.90.0" +rust-version = "1.91.0" license = "Apache-2.0" description = "WASM bindings for the Vortex web explorer" publish = false From bfb5dba1af6ebba2d3d598e63d8ff99510a42ed7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:55:11 +0000 Subject: [PATCH 164/250] Lock file maintenance (#7526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Update | Change | |---|---| | lockFileMaintenance | All locks refreshed | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. 🔧 This Pull Request updates lock files to use the latest dependency versions. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - "before 4am on monday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 513 ++++++---- benchmarks-website/package-lock.json | 305 +++--- java/testfiles/Cargo.lock | 172 ++-- vortex-web/package-lock.json | 1402 ++++++++++++++++++-------- 4 files changed, 1561 insertions(+), 831 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87d6ae125f8..c9b74f2ff69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,7 +161,7 @@ dependencies = [ "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -793,9 +793,9 @@ dependencies = [ [[package]] name = "async-signal" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ "async-io", "async-lock", @@ -880,9 +880,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "zeroize", @@ -890,9 +890,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -989,9 +989,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitpacking" @@ -1025,16 +1025,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.2.17", + "cpufeatures 0.3.0", ] [[package]] @@ -1239,9 +1239,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a0c0e6148f11f01f32650a2ea02d532b2ad4e81d8bd41e6e565b5adc5e6082" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" dependencies = [ "serde", "serde_core", @@ -1348,7 +1348,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1486,9 +1486,9 @@ dependencies = [ [[package]] name = "codspeed" -version = "4.4.1" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b684e94583e85a5ca7e1a6454a89d76a5121240f2fb67eb564129d9bafdb9db0" +checksum = "c83592369519f73d5731f9c91aa562e213a33cc14bb0f27ac2f5730fd1ecbcec" dependencies = [ "anyhow", "cc", @@ -1504,9 +1504,9 @@ dependencies = [ [[package]] name = "codspeed-criterion-compat-walltime" -version = "4.4.1" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96389aaa4bbb872ea4924dc0335b2bb181bcf28d6eedbe8fea29afcc5bde36a6" +checksum = "1601803d919d1bca7338b98948a319f0e40275c304516792aeabdd98db6dcd4f" dependencies = [ "anes", "cast", @@ -1531,9 +1531,9 @@ dependencies = [ [[package]] name = "codspeed-divan-compat" -version = "4.4.1" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e4bf8c7793c170fd0fcf3be97b9032b2ae39c2b9e8818aba3cc10ca0f0c6c0" +checksum = "a717deb83a7472e31b38244f836c92947ac17890970f61a083495b3a6653bf1b" dependencies = [ "clap", "codspeed", @@ -1544,9 +1544,9 @@ dependencies = [ [[package]] name = "codspeed-divan-compat-macros" -version = "4.4.1" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78aae02f2a278588e16e8ca62ea1915b8ab30f8230a09926671bba19ede801a4" +checksum = "eed8dc7bd913f259c57e2b94657a4c9fd20b3bded18462b22c647877feb43b53" dependencies = [ "divan-macros", "itertools 0.14.0", @@ -1558,9 +1558,9 @@ dependencies = [ [[package]] name = "codspeed-divan-compat-walltime" -version = "4.4.1" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ffd32c0c59ab8b674b15be65ba7c59aebac047036cfa7fa1e11bc2c178b81f" +checksum = "96500330271308ca89e68e1885951716cf2d3895ef9508192376dd125321a8f2" dependencies = [ "cfg-if", "clap", @@ -1747,11 +1747,12 @@ checksum = "d9c50fcfdf972929aff202c16b80086aa3cfc6a3a820af714096c58c7c1d0582" [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -2142,7 +2143,7 @@ dependencies = [ "log", "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser 0.59.0", "tempfile", @@ -2197,7 +2198,7 @@ dependencies = [ "object_store 0.13.2", "parking_lot", "parquet 58.1.0", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser 0.61.0", "tempfile", @@ -2423,7 +2424,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store 0.12.5", - "rand 0.9.2", + "rand 0.9.4", "tokio", "url", ] @@ -2456,7 +2457,7 @@ dependencies = [ "liblzma", "log", "object_store 0.13.2", - "rand 0.9.2", + "rand 0.9.4", "tokio", "tokio-util", "url", @@ -2681,7 +2682,7 @@ dependencies = [ "log", "object_store 0.12.5", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -2704,7 +2705,7 @@ dependencies = [ "log", "object_store 0.13.2", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -2804,7 +2805,7 @@ dependencies = [ "log", "md-5", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", "sha2 0.10.9", "unicode-segmentation", @@ -2836,7 +2837,7 @@ dependencies = [ "md-5", "memchr", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", "sha2 0.10.9", "unicode-segmentation", @@ -3402,7 +3403,7 @@ dependencies = [ "datafusion-functions-nested 53.1.0", "log", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "serde_json", "sha1", "sha2 0.10.9", @@ -3513,9 +3514,9 @@ dependencies = [ [[package]] name = "deflate64" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" [[package]] name = "deranged" @@ -3737,9 +3738,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -3782,9 +3783,9 @@ checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" [[package]] name = "ethnum" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" @@ -3863,9 +3864,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "filetime" @@ -3892,7 +3893,7 @@ checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "arbitrary", "byteorder", - "rand 0.8.5", + "rand 0.8.6", "rustc-hex", "static_assertions", ] @@ -3983,7 +3984,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" dependencies = [ "arrow-array 57.3.0", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -4194,7 +4195,7 @@ dependencies = [ "js-sys", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", "wasm-bindgen", @@ -4236,9 +4237,9 @@ dependencies = [ [[package]] name = "grid" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e2d4c0a8296178d8802098410ca05d86b17a10bb5ab559b3fb404c1f948220" +checksum = "b40ca9252762c466af32d0b1002e91e4e1bc5398f77455e55474deb466355ff5" [[package]] name = "h2" @@ -4268,7 +4269,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "zerocopy", ] @@ -4435,9 +4436,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -4449,7 +4450,6 @@ dependencies = [ "httparse", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -4457,16 +4457,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -4532,12 +4531,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -4545,9 +4545,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -4558,9 +4558,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -4572,15 +4572,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -4592,15 +4592,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -4666,12 +4666,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -4756,9 +4756,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -4943,9 +4943,9 @@ dependencies = [ [[package]] name = "jsonb" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" +checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" dependencies = [ "byteorder", "ethnum", @@ -4954,11 +4954,11 @@ dependencies = [ "jiff", "nom 8.0.0", "num-traits", - "ordered-float 5.1.0", - "rand 0.9.2", - "ryu", + "ordered-float 5.3.0", + "rand 0.9.4", "serde", "serde_json", + "zmij", ] [[package]] @@ -4982,6 +4982,21 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "lance" version = "4.0.0" @@ -5033,7 +5048,7 @@ dependencies = [ "pin-project", "prost 0.14.3", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "roaring", "semver", "serde", @@ -5067,7 +5082,7 @@ dependencies = [ "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -5126,7 +5141,7 @@ dependencies = [ "object_store 0.12.5", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "roaring", "serde_json", "snafu", @@ -5184,7 +5199,7 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rand_xoshiro 0.7.0", "random_word", @@ -5220,7 +5235,7 @@ dependencies = [ "prost 0.14.3", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "snafu", "strum 0.26.3", "tokio", @@ -5312,7 +5327,7 @@ dependencies = [ "prost 0.14.3", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rangemap", "rayon", @@ -5359,7 +5374,7 @@ dependencies = [ "path_abs", "pin-project", "prost 0.14.3", - "rand 0.9.2", + "rand 0.9.4", "serde", "snafu", "tempfile", @@ -5383,7 +5398,7 @@ dependencies = [ "lance-arrow", "lance-core", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -5440,7 +5455,7 @@ dependencies = [ "prost 0.14.3", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "rangemap", "roaring", "semver", @@ -5565,15 +5580,15 @@ dependencies = [ [[package]] name = "libbz2-rs-sys" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" +checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libfuzzer-sys" @@ -5616,9 +5631,9 @@ dependencies = [ [[package]] name = "liblzma-sys" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" dependencies = [ "cc", "libc", @@ -5633,23 +5648,23 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libmimalloc-sys" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe" +checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6" dependencies = [ "cc", ] [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "bitflags", "libc", "plain", - "redox_syscall 0.7.3", + "redox_syscall 0.7.4", ] [[package]] @@ -5666,9 +5681,9 @@ dependencies = [ [[package]] name = "line-clipping" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4de44e98ddbf09375cbf4d17714d18f39195f4f4894e8524501726fd9a8a4a" +checksum = "3f50e8f47623268b5407192d26876c4d7f89d686ca130fdc53bced4814cd29f8" dependencies = [ "bitflags", ] @@ -5696,9 +5711,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -5745,9 +5760,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", ] @@ -5884,9 +5899,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.49" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a" +checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640" dependencies = [ "libmimalloc-sys", ] @@ -6180,9 +6195,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-integer" @@ -6442,7 +6457,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", ] @@ -6463,9 +6478,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "5.1.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -6813,19 +6828,13 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "ping" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", "socket2", "thiserror 2.0.18", ] @@ -6843,9 +6852,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -6909,18 +6918,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -7083,9 +7092,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" dependencies = [ "ar_archive_writer", "cc", @@ -7289,7 +7298,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -7312,7 +7321,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7344,9 +7353,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -7355,9 +7364,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7371,7 +7380,7 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -7414,9 +7423,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_distr" @@ -7425,7 +7434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -7435,7 +7444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -7489,7 +7498,7 @@ dependencies = [ "ahash 0.8.12", "brotli", "paste", - "rand 0.9.2", + "rand 0.9.4", "unicase", ] @@ -7523,7 +7532,7 @@ dependencies = [ "indoc", "itertools 0.14.0", "kasuari", - "lru 0.16.3", + "lru 0.16.4", "strum 0.27.2", "thiserror 2.0.18", "unicode-segmentation", @@ -7586,9 +7595,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -7635,9 +7644,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ "bitflags", ] @@ -7910,7 +7919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", - "rand 0.8.5", + "rand 0.8.6", "syn 2.0.117", ] @@ -7934,7 +7943,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", @@ -8000,9 +8009,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "aws-lc-rs", "once_cell", @@ -8064,9 +8073,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -8204,9 +8213,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -8295,9 +8304,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.4" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] @@ -8455,9 +8464,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -8607,7 +8616,7 @@ dependencies = [ "libtest-mimic", "md-5", "owo-colors", - "rand 0.8.5", + "rand 0.8.6", "regex", "similar", "subst", @@ -8667,15 +8676,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9321,9 +9330,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -9443,39 +9452,39 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.4+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", - "toml_datetime 1.0.0+spec-1.1.0", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow 0.7.15", + "winnow 1.0.2", ] [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow 1.0.0", + "winnow 1.0.2", ] [[package]] name = "toml_writer" -version = "1.0.7+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tonic" @@ -9627,7 +9636,7 @@ dependencies = [ "bytes", "chrono", "prost 0.12.6", - "rand 0.8.5", + "rand 0.8.6", "thread-id", "tracing", "tracing-subscriber", @@ -9676,7 +9685,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -9687,9 +9696,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "typify" @@ -9765,9 +9774,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-truncate" @@ -10980,11 +10989,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -10993,7 +11002,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] @@ -11133,9 +11142,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ "rustls-pki-types", ] @@ -11319,6 +11328,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -11352,13 +11370,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -11380,6 +11415,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -11392,6 +11433,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -11404,12 +11451,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -11422,6 +11481,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -11434,6 +11499,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -11446,6 +11517,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -11458,20 +11535,26 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" @@ -11482,6 +11565,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -11563,9 +11652,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -11627,9 +11716,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -11638,9 +11727,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -11650,18 +11739,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -11670,18 +11759,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -11697,9 +11786,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -11708,9 +11797,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -11719,9 +11808,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/benchmarks-website/package-lock.json b/benchmarks-website/package-lock.json index 0c826ef1d73..d140b73d225 100644 --- a/benchmarks-website/package-lock.json +++ b/benchmarks-website/package-lock.json @@ -826,9 +826,9 @@ "license": "MIT" }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.0.tgz", - "integrity": "sha512-WOhNW9K8bR3kf4zLxbfg6Pxu2ybOUbB2AjMDHSQx86LIF4rH4Ft7vmMwNt0loO0eonglSNy4cpD3MKXXKQu0/A==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", + "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==", "cpu": [ "arm" ], @@ -840,9 +840,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.0.tgz", - "integrity": "sha512-u6JHLll5QKRvjciE78bQXDmqRqNs5M/3GVqZeMwvmjaNODJih/WIrJlFVEihvV0MiYFmd+ZyPr9wxOVbPAG2Iw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz", + "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==", "cpu": [ "arm64" ], @@ -854,9 +854,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.0.tgz", - "integrity": "sha512-qEF7CsKKzSRc20Ciu2Zw1wRrBz4g56F7r/vRwY430UPp/nt1x21Q/fpJ9N5l47WWvJlkNCPJz3QRVw008fi7yA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz", + "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==", "cpu": [ "arm64" ], @@ -868,9 +868,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.0.tgz", - "integrity": "sha512-WADYozJ4QCnXCH4wPB+3FuGmDPoFseVCUrANmA5LWwGmC6FL14BWC7pcq+FstOZv3baGX65tZ378uT6WG8ynTw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz", + "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==", "cpu": [ "x64" ], @@ -882,9 +882,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.0.tgz", - "integrity": "sha512-6b8wGHJlDrGeSE3aH5mGNHBjA0TTkxdoNHik5EkvPHCt351XnigA4pS7Wsj/Eo9Y8RBU6f35cjN9SYmCFBtzxw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz", + "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==", "cpu": [ "arm64" ], @@ -896,9 +896,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.0.tgz", - "integrity": "sha512-h25Ga0t4jaylMB8M/JKAyrvvfxGRjnPQIR8lnCayyzEjEOx2EJIlIiMbhpWxDRKGKF8jbNH01NnN663dH638mA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz", + "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==", "cpu": [ "x64" ], @@ -910,13 +910,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.0.tgz", - "integrity": "sha512-RzeBwv0B3qtVBWtcuABtSuCzToo2IEAIQrcyB/b2zMvBWVbjo8bZDjACUpnaafaxhTw2W+imQbP2BD1usasK4g==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz", + "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==", "cpu": [ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -924,13 +927,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.0.tgz", - "integrity": "sha512-Sf7zusNI2CIU1HLzuu9Tc5YGAHEZs5Lu7N1ssJG4Tkw6e0MEsN7NdjUDDfGNHy2IU+ENyWT+L2obgWiguWibWQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz", + "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==", "cpu": [ "arm" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -938,13 +944,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.0.tgz", - "integrity": "sha512-DX2x7CMcrJzsE91q7/O02IJQ5/aLkVtYFryqCjduJhUfGKG6yJV8hxaw8pZa93lLEpPTP/ohdN4wFz7yp/ry9A==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz", + "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -952,13 +961,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.0.tgz", - "integrity": "sha512-09EL+yFVbJZlhcQfShpswwRZ0Rg+z/CsSELFCnPt3iK+iqwGsI4zht3secj5vLEs957QvFFXnzAT0FFPIxSrkQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz", + "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -966,13 +978,16 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.0.tgz", - "integrity": "sha512-i9IcCMPr3EXm8EQg5jnja0Zyc1iFxJjZWlb4wr7U2Wx/GrddOuEafxRdMPRYVaXjgbhvqalp6np07hN1w9kAKw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz", + "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==", "cpu": [ "loong64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -980,13 +995,16 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.0.tgz", - "integrity": "sha512-DGzdJK9kyJ+B78MCkWeGnpXJ91tK/iKA6HwHxF4TAlPIY7GXEvMe8hBFRgdrR9Ly4qebR/7gfUs9y2IoaVEyog==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz", + "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==", "cpu": [ "loong64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -994,13 +1012,16 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.0.tgz", - "integrity": "sha512-RwpnLsqC8qbS8z1H1AxBA1H6qknR4YpPR9w2XX0vo2Sz10miu57PkNcnHVaZkbqyw/kUWfKMI73jhmfi9BRMUQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz", + "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1008,13 +1029,16 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.0.tgz", - "integrity": "sha512-Z8pPf54Ly3aqtdWC3G4rFigZgNvd+qJlOE52fmko3KST9SoGfAdSRCwyoyG05q1HrrAblLbk1/PSIV+80/pxLg==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz", + "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1022,13 +1046,16 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.0.tgz", - "integrity": "sha512-3a3qQustp3COCGvnP4SvrMHnPQ9d1vzCakQVRTliaz8cIp/wULGjiGpbcqrkv0WrHTEp8bQD/B3HBjzujVWLOA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz", + "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1036,13 +1063,16 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.0.tgz", - "integrity": "sha512-pjZDsVH/1VsghMJ2/kAaxt6dL0psT6ZexQVrijczOf+PeP2BUqTHYejk3l6TlPRydggINOeNRhvpLa0AYpCWSQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz", + "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1050,13 +1080,16 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.0.tgz", - "integrity": "sha512-3ObQs0BhvPgiUVZrN7gqCSvmFuMWvWvsjG5ayJ3Lraqv+2KhOsp+pUbigqbeWqueGIsnn+09HBw27rJ+gYK4VQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz", + "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1064,13 +1097,16 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.0.tgz", - "integrity": "sha512-EtylprDtQPdS5rXvAayrNDYoJhIz1/vzN2fEubo3yLE7tfAw+948dO0g4M0vkTVFhKojnF+n6C8bDNe+gDRdTg==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz", + "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1078,13 +1114,16 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.0.tgz", - "integrity": "sha512-k09oiRCi/bHU9UVFqD17r3eJR9bn03TyKraCrlz5ULFJGdJGi7VOmm9jl44vOJvRJ6P7WuBi/s2A97LxxHGIdw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz", + "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1092,9 +1131,9 @@ ] }, "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.0.tgz", - "integrity": "sha512-1o/0/pIhozoSaDJoDcec+IVLbnRtQmHwPV730+AOD29lHEEo4F5BEUB24H0OBdhbBBDwIOSuf7vgg0Ywxdfiiw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz", + "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==", "cpu": [ "x64" ], @@ -1106,9 +1145,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.0.tgz", - "integrity": "sha512-pESDkos/PDzYwtyzB5p/UoNU/8fJo68vcXM9ZW2V0kjYayj1KaaUfi1NmTUTUpMn4UhU4gTuK8gIaFO4UGuMbA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz", + "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==", "cpu": [ "arm64" ], @@ -1120,9 +1159,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.0.tgz", - "integrity": "sha512-hj1wFStD7B1YBeYmvY+lWXZ7ey73YGPcViMShYikqKT1GtstIKQAtfUI6yrzPjAy/O7pO0VLXGmUVWXQMaYgTQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz", + "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==", "cpu": [ "arm64" ], @@ -1134,9 +1173,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.0.tgz", - "integrity": "sha512-SyaIPFoxmUPlNDq5EHkTbiKzmSEmq/gOYFI/3HHJ8iS/v1mbugVa7dXUzcJGQfoytp9DJFLhHH4U3/eTy2Bq4w==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz", + "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==", "cpu": [ "ia32" ], @@ -1148,9 +1187,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.0.tgz", - "integrity": "sha512-RdcryEfzZr+lAr5kRm2ucN9aVlCCa2QNq4hXelZxb8GG0NJSazq44Z3PCCc8wISRuCVnGs0lQJVX5Vp6fKA+IA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz", + "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==", "cpu": [ "x64" ], @@ -1162,9 +1201,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.0.tgz", - "integrity": "sha512-PrsWNQ8BuE00O3Xsx3ALh2Df8fAj9+cvvX9AIA6o4KpATR98c9mud4XtDWVvsEuyia5U4tVSTKygawyJkjm60w==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz", + "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==", "cpu": [ "x64" ], @@ -1309,9 +1348,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.10.tgz", - "integrity": "sha512-sUoJ3IMxx4AyRqO4MLeHlnGDkyXRoUG0/AI9fjK+vS72ekpV0yWVY7O0BVjmBcRtkNcsAO2QDZ4tdKKGoI6YaQ==", + "version": "2.10.21", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.21.tgz", + "integrity": "sha512-Q+rUQ7Uz8AHM7DEaNdwvfFCTq7a43lNTzuS94eiWqwyxfV/wJv+oUivef51T91mmRY4d4A1u9rcSvkeufCVXlA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -1322,9 +1361,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", - "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", "dev": true, "funding": [ { @@ -1342,11 +1381,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.9.0", - "caniuse-lite": "^1.0.30001759", - "electron-to-chromium": "^1.5.263", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.2.0" + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" @@ -1356,9 +1395,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001780", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001780.tgz", - "integrity": "sha512-llngX0E7nQci5BPJDqoZSbuZ5Bcs9F5db7EtgfwBerX9XGtkkiO4NwfDDIRzHTTwcYC8vC7bmeUEPGrKlR/TkQ==", + "version": "1.0.30001790", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001790.tgz", + "integrity": "sha512-bOoxfJPyYo+ds6W0YfptaCWbFnJYjh2Y1Eow5lRv+vI2u8ganPZqNm1JwNh0t2ELQCqIWg4B3dWEusgAmsoyOw==", "dev": true, "funding": [ { @@ -1550,9 +1589,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.321", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.321.tgz", - "integrity": "sha512-L2C7Q279W2D/J4PLZLk7sebOILDSWos7bMsMNN06rK482umHUrh/3lM8G7IlHFOYip2oAg5nha1rCMxr/rs6ZQ==", + "version": "1.5.343", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.343.tgz", + "integrity": "sha512-YHnQ3MXI08icvL9ZKnEBy05F2EQ8ob01UaMOuMbM8l+4UcAq6MPPbBTJBbsBUg3H8JeZNt+O4fjsoWth3p6IFg==", "dev": true, "license": "ISC" }, @@ -1730,9 +1769,9 @@ } }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "dev": true, "license": "MIT" }, @@ -1794,9 +1833,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.36", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", - "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "version": "2.0.38", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", + "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", "dev": true, "license": "MIT" }, @@ -1821,9 +1860,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "dev": true, "funding": [ { @@ -1905,9 +1944,9 @@ } }, "node_modules/rollup": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.0.tgz", - "integrity": "sha512-yqjxruMGBQJ2gG4HtjZtAfXArHomazDHoFwFFmZZl0r7Pdo7qCIXKqKHZc8yeoMgzJJ+pO6pEEHa+V7uzWlrAQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", + "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1921,31 +1960,31 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.60.0", - "@rollup/rollup-android-arm64": "4.60.0", - "@rollup/rollup-darwin-arm64": "4.60.0", - "@rollup/rollup-darwin-x64": "4.60.0", - "@rollup/rollup-freebsd-arm64": "4.60.0", - "@rollup/rollup-freebsd-x64": "4.60.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.60.0", - "@rollup/rollup-linux-arm-musleabihf": "4.60.0", - "@rollup/rollup-linux-arm64-gnu": "4.60.0", - "@rollup/rollup-linux-arm64-musl": "4.60.0", - "@rollup/rollup-linux-loong64-gnu": "4.60.0", - "@rollup/rollup-linux-loong64-musl": "4.60.0", - "@rollup/rollup-linux-ppc64-gnu": "4.60.0", - "@rollup/rollup-linux-ppc64-musl": "4.60.0", - "@rollup/rollup-linux-riscv64-gnu": "4.60.0", - "@rollup/rollup-linux-riscv64-musl": "4.60.0", - "@rollup/rollup-linux-s390x-gnu": "4.60.0", - "@rollup/rollup-linux-x64-gnu": "4.60.0", - "@rollup/rollup-linux-x64-musl": "4.60.0", - "@rollup/rollup-openbsd-x64": "4.60.0", - "@rollup/rollup-openharmony-arm64": "4.60.0", - "@rollup/rollup-win32-arm64-msvc": "4.60.0", - "@rollup/rollup-win32-ia32-msvc": "4.60.0", - "@rollup/rollup-win32-x64-gnu": "4.60.0", - "@rollup/rollup-win32-x64-msvc": "4.60.0", + "@rollup/rollup-android-arm-eabi": "4.60.2", + "@rollup/rollup-android-arm64": "4.60.2", + "@rollup/rollup-darwin-arm64": "4.60.2", + "@rollup/rollup-darwin-x64": "4.60.2", + "@rollup/rollup-freebsd-arm64": "4.60.2", + "@rollup/rollup-freebsd-x64": "4.60.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.2", + "@rollup/rollup-linux-arm-musleabihf": "4.60.2", + "@rollup/rollup-linux-arm64-gnu": "4.60.2", + "@rollup/rollup-linux-arm64-musl": "4.60.2", + "@rollup/rollup-linux-loong64-gnu": "4.60.2", + "@rollup/rollup-linux-loong64-musl": "4.60.2", + "@rollup/rollup-linux-ppc64-gnu": "4.60.2", + "@rollup/rollup-linux-ppc64-musl": "4.60.2", + "@rollup/rollup-linux-riscv64-gnu": "4.60.2", + "@rollup/rollup-linux-riscv64-musl": "4.60.2", + "@rollup/rollup-linux-s390x-gnu": "4.60.2", + "@rollup/rollup-linux-x64-gnu": "4.60.2", + "@rollup/rollup-linux-x64-musl": "4.60.2", + "@rollup/rollup-openbsd-x64": "4.60.2", + "@rollup/rollup-openharmony-arm64": "4.60.2", + "@rollup/rollup-win32-arm64-msvc": "4.60.2", + "@rollup/rollup-win32-ia32-msvc": "4.60.2", + "@rollup/rollup-win32-x64-gnu": "4.60.2", + "@rollup/rollup-win32-x64-msvc": "4.60.2", "fsevents": "~2.3.2" } }, @@ -2052,14 +2091,14 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" diff --git a/java/testfiles/Cargo.lock b/java/testfiles/Cargo.lock index b6618bc3fdc..e3fe6731a30 100644 --- a/java/testfiles/Cargo.lock +++ b/java/testfiles/Cargo.lock @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "async-signal" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c070bbf59cd3570b6b2dd54cd772527c7c3620fce8be898406dd3ed6adc64c" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" dependencies = [ "async-io", "async-lock", @@ -403,9 +403,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitvec" @@ -458,9 +458,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.57" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -491,7 +491,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -598,9 +598,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "custom-labels" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a750ea4bdb7dbf9584b5d5c668bfa3835f88275781a947b5ea0212945bbdd41f" +checksum = "4121f4f539659ad975da5bc6702b7d7d6de59ff1bbaec328dfe7dd5058052ef2" dependencies = [ "bindgen", "cc", @@ -735,9 +735,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" @@ -769,9 +769,9 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "fsst-rs" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbfc42bd5e2b76f850ba4ff6dd6cb170389bc3c1e64dc45238a742d79bbfb2ca" +checksum = "3bf53d7c403a2b76873d4d66ba7d79c54bde2784cdaba6083f223d6e33270708" dependencies = [ "rustc-hash", ] @@ -915,7 +915,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -949,6 +949,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "hashbrown" @@ -964,6 +968,12 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" dependencies = [ "allocator-api2", "equivalent", @@ -1023,21 +1033,21 @@ checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -1119,10 +1129,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1137,6 +1149,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "lasso" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e14eda50a3494b3bf7b9ce51c52434a761e383d7238ce1dd5dcec2fbc13e9fb" +dependencies = [ + "dashmap", + "hashbrown 0.14.5", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -1227,9 +1249,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.183" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libloading" @@ -1515,9 +1537,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "polling" @@ -1547,9 +1569,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] @@ -1640,7 +1662,7 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1651,9 +1673,9 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_xoshiro" @@ -1714,9 +1736,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -1760,9 +1782,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "seq-macro" @@ -1940,9 +1962,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "pin-project-lite", @@ -1955,14 +1977,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tracing-core" version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] [[package]] name = "unicode-ident" @@ -1978,9 +2015,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -2456,6 +2493,7 @@ version = "0.1.0" dependencies = [ "arcref", "dashmap", + "lasso", "parking_lot", "vortex-error", "vortex-utils", @@ -2480,7 +2518,7 @@ name = "vortex-utils" version = "0.1.0" dependencies = [ "dashmap", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "vortex-error", ] @@ -2518,11 +2556,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -2531,14 +2569,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -2549,23 +2587,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2573,9 +2607,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -2586,9 +2620,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -2627,16 +2661,6 @@ dependencies = [ "semver", ] -[[package]] -name = "web-sys" -version = "0.3.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -2714,6 +2738,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -2804,18 +2834,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 762acb6af94..aa41e786f74 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -339,9 +339,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", "cpu": [ "ppc64" ], @@ -356,9 +356,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", "cpu": [ "arm" ], @@ -373,9 +373,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", "cpu": [ "arm64" ], @@ -390,9 +390,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", "cpu": [ "x64" ], @@ -407,9 +407,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", "cpu": [ "arm64" ], @@ -424,9 +424,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", "cpu": [ "x64" ], @@ -441,9 +441,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", "cpu": [ "arm64" ], @@ -458,9 +458,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", "cpu": [ "x64" ], @@ -475,9 +475,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", "cpu": [ "arm" ], @@ -492,9 +492,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", "cpu": [ "arm64" ], @@ -509,9 +509,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", "cpu": [ "ia32" ], @@ -526,9 +526,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", "cpu": [ "loong64" ], @@ -543,9 +543,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", "cpu": [ "mips64el" ], @@ -560,9 +560,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", "cpu": [ "ppc64" ], @@ -577,9 +577,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", "cpu": [ "riscv64" ], @@ -594,9 +594,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", "cpu": [ "s390x" ], @@ -611,9 +611,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", "cpu": [ "x64" ], @@ -628,9 +628,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", "cpu": [ "arm64" ], @@ -645,9 +645,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", "cpu": [ "x64" ], @@ -662,9 +662,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", "cpu": [ "arm64" ], @@ -679,9 +679,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", "cpu": [ "x64" ], @@ -696,9 +696,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", "cpu": [ "arm64" ], @@ -713,9 +713,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", "cpu": [ "x64" ], @@ -730,9 +730,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", "cpu": [ "arm64" ], @@ -747,9 +747,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", "cpu": [ "ia32" ], @@ -764,9 +764,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", "cpu": [ "x64" ], @@ -938,29 +938,43 @@ } }, "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@humanfs/core": "^0.19.1", + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -1108,9 +1122,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.0.tgz", - "integrity": "sha512-WOhNW9K8bR3kf4zLxbfg6Pxu2ybOUbB2AjMDHSQx86LIF4rH4Ft7vmMwNt0loO0eonglSNy4cpD3MKXXKQu0/A==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", + "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==", "cpu": [ "arm" ], @@ -1122,9 +1136,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.0.tgz", - "integrity": "sha512-u6JHLll5QKRvjciE78bQXDmqRqNs5M/3GVqZeMwvmjaNODJih/WIrJlFVEihvV0MiYFmd+ZyPr9wxOVbPAG2Iw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz", + "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==", "cpu": [ "arm64" ], @@ -1136,9 +1150,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.0.tgz", - "integrity": "sha512-qEF7CsKKzSRc20Ciu2Zw1wRrBz4g56F7r/vRwY430UPp/nt1x21Q/fpJ9N5l47WWvJlkNCPJz3QRVw008fi7yA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz", + "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==", "cpu": [ "arm64" ], @@ -1150,9 +1164,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.0.tgz", - "integrity": "sha512-WADYozJ4QCnXCH4wPB+3FuGmDPoFseVCUrANmA5LWwGmC6FL14BWC7pcq+FstOZv3baGX65tZ378uT6WG8ynTw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz", + "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==", "cpu": [ "x64" ], @@ -1164,9 +1178,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.0.tgz", - "integrity": "sha512-6b8wGHJlDrGeSE3aH5mGNHBjA0TTkxdoNHik5EkvPHCt351XnigA4pS7Wsj/Eo9Y8RBU6f35cjN9SYmCFBtzxw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz", + "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==", "cpu": [ "arm64" ], @@ -1178,9 +1192,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.0.tgz", - "integrity": "sha512-h25Ga0t4jaylMB8M/JKAyrvvfxGRjnPQIR8lnCayyzEjEOx2EJIlIiMbhpWxDRKGKF8jbNH01NnN663dH638mA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz", + "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==", "cpu": [ "x64" ], @@ -1192,13 +1206,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.0.tgz", - "integrity": "sha512-RzeBwv0B3qtVBWtcuABtSuCzToo2IEAIQrcyB/b2zMvBWVbjo8bZDjACUpnaafaxhTw2W+imQbP2BD1usasK4g==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz", + "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==", "cpu": [ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1206,13 +1223,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.0.tgz", - "integrity": "sha512-Sf7zusNI2CIU1HLzuu9Tc5YGAHEZs5Lu7N1ssJG4Tkw6e0MEsN7NdjUDDfGNHy2IU+ENyWT+L2obgWiguWibWQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz", + "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==", "cpu": [ "arm" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1220,13 +1240,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.0.tgz", - "integrity": "sha512-DX2x7CMcrJzsE91q7/O02IJQ5/aLkVtYFryqCjduJhUfGKG6yJV8hxaw8pZa93lLEpPTP/ohdN4wFz7yp/ry9A==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz", + "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1234,13 +1257,16 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.0.tgz", - "integrity": "sha512-09EL+yFVbJZlhcQfShpswwRZ0Rg+z/CsSELFCnPt3iK+iqwGsI4zht3secj5vLEs957QvFFXnzAT0FFPIxSrkQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz", + "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1248,13 +1274,16 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.0.tgz", - "integrity": "sha512-i9IcCMPr3EXm8EQg5jnja0Zyc1iFxJjZWlb4wr7U2Wx/GrddOuEafxRdMPRYVaXjgbhvqalp6np07hN1w9kAKw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz", + "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==", "cpu": [ "loong64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1262,13 +1291,16 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.0.tgz", - "integrity": "sha512-DGzdJK9kyJ+B78MCkWeGnpXJ91tK/iKA6HwHxF4TAlPIY7GXEvMe8hBFRgdrR9Ly4qebR/7gfUs9y2IoaVEyog==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz", + "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==", "cpu": [ "loong64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1276,13 +1308,16 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.0.tgz", - "integrity": "sha512-RwpnLsqC8qbS8z1H1AxBA1H6qknR4YpPR9w2XX0vo2Sz10miu57PkNcnHVaZkbqyw/kUWfKMI73jhmfi9BRMUQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz", + "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1290,13 +1325,16 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.0.tgz", - "integrity": "sha512-Z8pPf54Ly3aqtdWC3G4rFigZgNvd+qJlOE52fmko3KST9SoGfAdSRCwyoyG05q1HrrAblLbk1/PSIV+80/pxLg==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz", + "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1304,13 +1342,16 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.0.tgz", - "integrity": "sha512-3a3qQustp3COCGvnP4SvrMHnPQ9d1vzCakQVRTliaz8cIp/wULGjiGpbcqrkv0WrHTEp8bQD/B3HBjzujVWLOA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz", + "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1318,13 +1359,16 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.0.tgz", - "integrity": "sha512-pjZDsVH/1VsghMJ2/kAaxt6dL0psT6ZexQVrijczOf+PeP2BUqTHYejk3l6TlPRydggINOeNRhvpLa0AYpCWSQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz", + "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1332,13 +1376,16 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.0.tgz", - "integrity": "sha512-3ObQs0BhvPgiUVZrN7gqCSvmFuMWvWvsjG5ayJ3Lraqv+2KhOsp+pUbigqbeWqueGIsnn+09HBw27rJ+gYK4VQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz", + "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1346,13 +1393,16 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.0.tgz", - "integrity": "sha512-EtylprDtQPdS5rXvAayrNDYoJhIz1/vzN2fEubo3yLE7tfAw+948dO0g4M0vkTVFhKojnF+n6C8bDNe+gDRdTg==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz", + "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1360,13 +1410,16 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.0.tgz", - "integrity": "sha512-k09oiRCi/bHU9UVFqD17r3eJR9bn03TyKraCrlz5ULFJGdJGi7VOmm9jl44vOJvRJ6P7WuBi/s2A97LxxHGIdw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz", + "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1374,9 +1427,9 @@ ] }, "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.0.tgz", - "integrity": "sha512-1o/0/pIhozoSaDJoDcec+IVLbnRtQmHwPV730+AOD29lHEEo4F5BEUB24H0OBdhbBBDwIOSuf7vgg0Ywxdfiiw==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz", + "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==", "cpu": [ "x64" ], @@ -1388,9 +1441,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.0.tgz", - "integrity": "sha512-pESDkos/PDzYwtyzB5p/UoNU/8fJo68vcXM9ZW2V0kjYayj1KaaUfi1NmTUTUpMn4UhU4gTuK8gIaFO4UGuMbA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz", + "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==", "cpu": [ "arm64" ], @@ -1402,9 +1455,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.0.tgz", - "integrity": "sha512-hj1wFStD7B1YBeYmvY+lWXZ7ey73YGPcViMShYikqKT1GtstIKQAtfUI6yrzPjAy/O7pO0VLXGmUVWXQMaYgTQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz", + "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==", "cpu": [ "arm64" ], @@ -1416,9 +1469,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.0.tgz", - "integrity": "sha512-SyaIPFoxmUPlNDq5EHkTbiKzmSEmq/gOYFI/3HHJ8iS/v1mbugVa7dXUzcJGQfoytp9DJFLhHH4U3/eTy2Bq4w==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz", + "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==", "cpu": [ "ia32" ], @@ -1430,9 +1483,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.0.tgz", - "integrity": "sha512-RdcryEfzZr+lAr5kRm2ucN9aVlCCa2QNq4hXelZxb8GG0NJSazq44Z3PCCc8wISRuCVnGs0lQJVX5Vp6fKA+IA==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz", + "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==", "cpu": [ "x64" ], @@ -1444,9 +1497,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.0.tgz", - "integrity": "sha512-PrsWNQ8BuE00O3Xsx3ALh2Df8fAj9+cvvX9AIA6o4KpATR98c9mud4XtDWVvsEuyia5U4tVSTKygawyJkjm60w==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz", + "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==", "cpu": [ "x64" ], @@ -1625,18 +1678,18 @@ } }, "node_modules/@swc/helpers": { - "version": "0.5.20", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.20.tgz", - "integrity": "sha512-2egEBHUMasdypIzrprsu8g+OEVd7Vp2MM3a2eVlM/cyFYto0nGz5BX5BTgh/ShZZI9ed+ozEq+Ngt+rgmUs8tw==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.21.tgz", + "integrity": "sha512-jI/VAmtdjB/RnI8GTnokyX7Ug8c+g+ffD6QRLa6XQewtnGyukKkKSk3wLTM3b5cjt1jNh9x0jfVlagdN2gDKQg==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.8.0" } }, "node_modules/@tailwindcss/node": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.2.tgz", - "integrity": "sha512-pXS+wJ2gZpVXqFaUEjojq7jzMpTGf8rU6ipJz5ovJV6PUGmlJ+jvIwGrzdHdQ80Sg+wmQxUFuoW1UAAwHNEdFA==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.2.4.tgz", + "integrity": "sha512-Ai7+yQPxz3ddrDQzFfBKdHEVBg0w3Zl83jnjuwxnZOsnH9pGn93QHQtpU0p/8rYWxvbFZHneni6p1BSLK4DkGA==", "dev": true, "license": "MIT", "dependencies": { @@ -1646,37 +1699,37 @@ "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", - "tailwindcss": "4.2.2" + "tailwindcss": "4.2.4" } }, "node_modules/@tailwindcss/oxide": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.2.tgz", - "integrity": "sha512-qEUA07+E5kehxYp9BVMpq9E8vnJuBHfJEC0vPC5e7iL/hw7HR61aDKoVoKzrG+QKp56vhNZe4qwkRmMC0zDLvg==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.2.4.tgz", + "integrity": "sha512-9El/iI069DKDSXwTvB9J4BwdO5JhRrOweGaK25taBAvBXyXqJAX+Jqdvs8r8gKpsI/1m0LeJLyQYTf/WLrBT1Q==", "dev": true, "license": "MIT", "engines": { "node": ">= 20" }, "optionalDependencies": { - "@tailwindcss/oxide-android-arm64": "4.2.2", - "@tailwindcss/oxide-darwin-arm64": "4.2.2", - "@tailwindcss/oxide-darwin-x64": "4.2.2", - "@tailwindcss/oxide-freebsd-x64": "4.2.2", - "@tailwindcss/oxide-linux-arm-gnueabihf": "4.2.2", - "@tailwindcss/oxide-linux-arm64-gnu": "4.2.2", - "@tailwindcss/oxide-linux-arm64-musl": "4.2.2", - "@tailwindcss/oxide-linux-x64-gnu": "4.2.2", - "@tailwindcss/oxide-linux-x64-musl": "4.2.2", - "@tailwindcss/oxide-wasm32-wasi": "4.2.2", - "@tailwindcss/oxide-win32-arm64-msvc": "4.2.2", - "@tailwindcss/oxide-win32-x64-msvc": "4.2.2" + "@tailwindcss/oxide-android-arm64": "4.2.4", + "@tailwindcss/oxide-darwin-arm64": "4.2.4", + "@tailwindcss/oxide-darwin-x64": "4.2.4", + "@tailwindcss/oxide-freebsd-x64": "4.2.4", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.2.4", + "@tailwindcss/oxide-linux-arm64-gnu": "4.2.4", + "@tailwindcss/oxide-linux-arm64-musl": "4.2.4", + "@tailwindcss/oxide-linux-x64-gnu": "4.2.4", + "@tailwindcss/oxide-linux-x64-musl": "4.2.4", + "@tailwindcss/oxide-wasm32-wasi": "4.2.4", + "@tailwindcss/oxide-win32-arm64-msvc": "4.2.4", + "@tailwindcss/oxide-win32-x64-msvc": "4.2.4" } }, "node_modules/@tailwindcss/oxide-android-arm64": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.2.2.tgz", - "integrity": "sha512-dXGR1n+P3B6748jZO/SvHZq7qBOqqzQ+yFrXpoOWWALWndF9MoSKAT3Q0fYgAzYzGhxNYOoysRvYlpixRBBoDg==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.2.4.tgz", + "integrity": "sha512-e7MOr1SAn9U8KlZzPi1ZXGZHeC5anY36qjNwmZv9pOJ8E4Q6jmD1vyEHkQFmNOIN7twGPEMXRHmitN4zCMN03g==", "cpu": [ "arm64" ], @@ -1691,9 +1744,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-arm64": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.2.2.tgz", - "integrity": "sha512-iq9Qjr6knfMpZHj55/37ouZeykwbDqF21gPFtfnhCCKGDcPI/21FKC9XdMO/XyBM7qKORx6UIhGgg6jLl7BZlg==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.2.4.tgz", + "integrity": "sha512-tSC/Kbqpz/5/o/C2sG7QvOxAKqyd10bq+ypZNf+9Fi2TvbVbv1zNpcEptcsU7DPROaSbVgUXmrzKhurFvo5eDg==", "cpu": [ "arm64" ], @@ -1708,9 +1761,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-x64": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.2.2.tgz", - "integrity": "sha512-BlR+2c3nzc8f2G639LpL89YY4bdcIdUmiOOkv2GQv4/4M0vJlpXEa0JXNHhCHU7VWOKWT/CjqHdTP8aUuDJkuw==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.2.4.tgz", + "integrity": "sha512-yPyUXn3yO/ufR6+Kzv0t4fCg2qNr90jxXc5QqBpjlPNd0NqyDXcmQb/6weunH/MEDXW5dhyEi+agTDiqa3WsGg==", "cpu": [ "x64" ], @@ -1725,9 +1778,9 @@ } }, "node_modules/@tailwindcss/oxide-freebsd-x64": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.2.2.tgz", - "integrity": "sha512-YUqUgrGMSu2CDO82hzlQ5qSb5xmx3RUrke/QgnoEx7KvmRJHQuZHZmZTLSuuHwFf0DJPybFMXMYf+WJdxHy/nQ==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.2.4.tgz", + "integrity": "sha512-BoMIB4vMQtZsXdGLVc2z+P9DbETkiopogfWZKbWwM8b/1Vinbs4YcUwo+kM/KeLkX3Ygrf4/PsRndKaYhS8Eiw==", "cpu": [ "x64" ], @@ -1742,9 +1795,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.2.2.tgz", - "integrity": "sha512-FPdhvsW6g06T9BWT0qTwiVZYE2WIFo2dY5aCSpjG/S/u1tby+wXoslXS0kl3/KXnULlLr1E3NPRRw0g7t2kgaQ==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.2.4.tgz", + "integrity": "sha512-7pIHBLTHYRAlS7V22JNuTh33yLH4VElwKtB3bwchK/UaKUPpQ0lPQiOWcbm4V3WP2I6fNIJ23vABIvoy2izdwA==", "cpu": [ "arm" ], @@ -1759,13 +1812,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.2.2.tgz", - "integrity": "sha512-4og1V+ftEPXGttOO7eCmW7VICmzzJWgMx+QXAJRAhjrSjumCwWqMfkDrNu1LXEQzNAwz28NCUpucgQPrR4S2yw==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.2.4.tgz", + "integrity": "sha512-+E4wxJ0ZGOzSH325reXTWB48l42i93kQqMvDyz5gqfRzRZ7faNhnmvlV4EPGJU3QJM/3Ab5jhJ5pCRUsKn6OQw==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1776,13 +1832,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-musl": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.2.2.tgz", - "integrity": "sha512-oCfG/mS+/+XRlwNjnsNLVwnMWYH7tn/kYPsNPh+JSOMlnt93mYNCKHYzylRhI51X+TbR+ufNhhKKzm6QkqX8ag==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.2.4.tgz", + "integrity": "sha512-bBADEGAbo4ASnppIziaQJelekCxdMaxisrk+fB7Thit72IBnALp9K6ffA2G4ruj90G9XRS2VQ6q2bCKbfFV82g==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1793,13 +1852,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-gnu": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.2.2.tgz", - "integrity": "sha512-rTAGAkDgqbXHNp/xW0iugLVmX62wOp2PoE39BTCGKjv3Iocf6AFbRP/wZT/kuCxC9QBh9Pu8XPkv/zCZB2mcMg==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.2.4.tgz", + "integrity": "sha512-7Mx25E4WTfnht0TVRTyC00j3i0M+EeFe7wguMDTlX4mRxafznw0CA8WJkFjWYH5BlgELd1kSjuU2JiPnNZbJDA==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -1810,13 +1872,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-musl": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.2.2.tgz", - "integrity": "sha512-XW3t3qwbIwiSyRCggeO2zxe3KWaEbM0/kW9e8+0XpBgyKU4ATYzcVSMKteZJ1iukJ3HgHBjbg9P5YPRCVUxlnQ==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.2.4.tgz", + "integrity": "sha512-2wwJRF7nyhOR0hhHoChc04xngV3iS+akccHTGtz965FwF0up4b2lOdo6kI1EbDaEXKgvcrFBYcYQQ/rrnWFVfA==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -1827,9 +1892,9 @@ } }, "node_modules/@tailwindcss/oxide-wasm32-wasi": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.2.2.tgz", - "integrity": "sha512-eKSztKsmEsn1O5lJ4ZAfyn41NfG7vzCg496YiGtMDV86jz1q/irhms5O0VrY6ZwTUkFy/EKG3RfWgxSI3VbZ8Q==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.2.4.tgz", + "integrity": "sha512-FQsqApeor8Fo6gUEklzmaa9994orJZZDBAlQpK2Mq+DslRKFJeD6AjHpBQ0kZFQohVr8o85PPh8eOy86VlSCmw==", "bundleDependencies": [ "@napi-rs/wasm-runtime", "@emnapi/core", @@ -1921,9 +1986,9 @@ "optional": true }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.2.tgz", - "integrity": "sha512-qPmaQM4iKu5mxpsrWZMOZRgZv1tOZpUm+zdhhQP0VhJfyGGO3aUKdbh3gDZc/dPLQwW4eSqWGrrcWNBZWUWaXQ==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.2.4.tgz", + "integrity": "sha512-L9BXqxC4ToVgwMFqj3pmZRqyHEztulpUJzCxUtLjobMCzTPsGt1Fa9enKbOpY2iIyVtaHNeNvAK8ERP/64sqGQ==", "cpu": [ "arm64" ], @@ -1938,9 +2003,9 @@ } }, "node_modules/@tailwindcss/oxide-win32-x64-msvc": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.2.2.tgz", - "integrity": "sha512-1T/37VvI7WyH66b+vqHj/cLwnCxt7Qt3WFu5Q8hk65aOvlwAhs7rAp1VkulBJw/N4tMirXjVnylTR72uI0HGcA==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.2.4.tgz", + "integrity": "sha512-ESlKG0EpVJQwRjXDDa9rLvhEAh0mhP1sF7sap9dNZT0yyl9SAG6T7gdP09EH0vIv0UNTlo6jPWyujD6559fZvw==", "cpu": [ "x64" ], @@ -1955,15 +2020,15 @@ } }, "node_modules/@tailwindcss/vite": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.2.2.tgz", - "integrity": "sha512-mEiF5HO1QqCLXoNEfXVA1Tzo+cYsrqV7w9Juj2wdUFyW07JRenqMG225MvPwr3ZD9N1bFQj46X7r33iHxLUW0w==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.2.4.tgz", + "integrity": "sha512-pCvohwOCspk3ZFn6eJzrrX3g4n2JY73H6MmYC87XfGPyTty4YsCjYTMArRZm/zOI8dIt3+EcrLHAFPe5A4bgtw==", "dev": true, "license": "MIT", "dependencies": { - "@tailwindcss/node": "4.2.2", - "@tailwindcss/oxide": "4.2.2", - "tailwindcss": "4.2.2" + "@tailwindcss/node": "4.2.4", + "@tailwindcss/oxide": "4.2.4", + "tailwindcss": "4.2.4" }, "peerDependencies": { "vite": "^5.2.0 || ^6 || ^7 || ^8" @@ -2210,9 +2275,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "24.12.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", - "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "version": "24.12.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.2.tgz", + "integrity": "sha512-A1sre26ke7HDIuY/M23nd9gfB+nrmhtYyMINbjI1zHJxYteKR6qSMX56FsmjMcDb3SMcjJg5BiRRgOCC/yBD0g==", "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -2246,17 +2311,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.2.tgz", - "integrity": "sha512-aC2qc5thQahutKjP+cl8cgN9DWe3ZUqVko30CMSZHnFEHyhOYoZSzkGtAI2mcwZ38xeImDucI4dnqsHiOYuuCw==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.0.tgz", + "integrity": "sha512-HyAZtpdkgZwpq8Sz3FSUvCR4c+ScbuWa9AksK2Jweub7w4M3yTz4O11AqVJzLYjy/B9ZWPyc81I+mOdJU/bDQw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.58.2", - "@typescript-eslint/type-utils": "8.58.2", - "@typescript-eslint/utils": "8.58.2", - "@typescript-eslint/visitor-keys": "8.58.2", + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/type-utils": "8.59.0", + "@typescript-eslint/utils": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -2269,7 +2334,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.58.2", + "@typescript-eslint/parser": "^8.59.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } @@ -2285,16 +2350,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.2.tgz", - "integrity": "sha512-/Zb/xaIDfxeJnvishjGdcR4jmr7S+bda8PKNhRGdljDM+elXhlvN0FyPSsMnLmJUrVG9aPO6dof80wjMawsASg==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.0.tgz", + "integrity": "sha512-TI1XGwKbDpo9tRW8UDIXCOeLk55qe9ZFGs8MTKU6/M08HWTw52DD/IYhfQtOEhEdPhLMT26Ka/x7p70nd3dzDg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.58.2", - "@typescript-eslint/types": "8.58.2", - "@typescript-eslint/typescript-estree": "8.58.2", - "@typescript-eslint/visitor-keys": "8.58.2", + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", "debug": "^4.4.3" }, "engines": { @@ -2310,14 +2375,14 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.2.tgz", - "integrity": "sha512-Cq6UfpZZk15+r87BkIh5rDpi38W4b+Sjnb8wQCPPDDweS/LRCFjCyViEbzHk5Ck3f2QDfgmlxqSa7S7clDtlfg==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.0.tgz", + "integrity": "sha512-Lw5ITrR5s5TbC19YSvlr63ZfLaJoU6vtKTHyB0GQOpX0W7d5/Ir6vUahWi/8Sps/nOukZQ0IB3SmlxZnjaKVnw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.58.2", - "@typescript-eslint/types": "^8.58.2", + "@typescript-eslint/tsconfig-utils": "^8.59.0", + "@typescript-eslint/types": "^8.59.0", "debug": "^4.4.3" }, "engines": { @@ -2332,14 +2397,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.2.tgz", - "integrity": "sha512-SgmyvDPexWETQek+qzZnrG6844IaO02UVyOLhI4wpo82dpZJY9+6YZCKAMFzXb7qhx37mFK1QcPQ18tud+vo6Q==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.0.tgz", + "integrity": "sha512-UzR16Ut8IpA3Mc4DbgAShlPPkVm8xXMWafXxB0BocaVRHs8ZGakAxGRskF7FId3sdk9lgGD73GSFaWmWFDE4dg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.2", - "@typescript-eslint/visitor-keys": "8.58.2" + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2350,9 +2415,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.2.tgz", - "integrity": "sha512-3SR+RukipDvkkKp/d0jP0dyzuls3DbGmwDpVEc5wqk5f38KFThakqAAO0XMirWAE+kT00oTauTbzMFGPoAzB0A==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.0.tgz", + "integrity": "sha512-91Sbl3s4Kb3SybliIY6muFBmHVv+pYXfybC4Oolp3dvk8BvIE3wOPc+403CWIT7mJNkfQRGtdqghzs2+Z91Tqg==", "dev": true, "license": "MIT", "engines": { @@ -2367,15 +2432,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.2.tgz", - "integrity": "sha512-Z7EloNR/B389FvabdGeTo2XMs4W9TjtPiO9DAsmT0yom0bwlPyRjkJ1uCdW1DvrrrYP50AJZ9Xc3sByZA9+dcg==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.0.tgz", + "integrity": "sha512-3TRiZaQSltGqGeNrJzzr1+8YcEobKH9rHnqIp/1psfKFmhRQDNMGP5hBufanYTGznwShzVLs3Mz+gDN7HkWfXg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.2", - "@typescript-eslint/typescript-estree": "8.58.2", - "@typescript-eslint/utils": "8.58.2", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0", + "@typescript-eslint/utils": "8.59.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -2392,9 +2457,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.2.tgz", - "integrity": "sha512-9TukXyATBQf/Jq9AMQXfvurk+G5R2MwfqQGDR2GzGz28HvY/lXNKGhkY+6IOubwcquikWk5cjlgPvD2uAA7htQ==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.0.tgz", + "integrity": "sha512-nLzdsT1gdOgFxxxwrlNVUBzSNBEEHJ86bblmk4QAS6stfig7rcJzWKqCyxFy3YRRHXDWEkb2NralA1nOYkkm/A==", "dev": true, "license": "MIT", "engines": { @@ -2406,16 +2471,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.2.tgz", - "integrity": "sha512-ELGuoofuhhoCvNbQjFFiobFcGgcDCEm0ThWdmO4Z0UzLqPXS3KFvnEZ+SHewwOYHjM09tkzOWXNTv9u6Gqtyuw==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.0.tgz", + "integrity": "sha512-O9Re9P1BmBLFJyikRbQpLku/QA3/AueZNO9WePLBwQrvkixTmDe8u76B6CYUAITRl/rHawggEqUGn5QIkVRLMw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.58.2", - "@typescript-eslint/tsconfig-utils": "8.58.2", - "@typescript-eslint/types": "8.58.2", - "@typescript-eslint/visitor-keys": "8.58.2", + "@typescript-eslint/project-service": "8.59.0", + "@typescript-eslint/tsconfig-utils": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/visitor-keys": "8.59.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -2486,16 +2551,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.2.tgz", - "integrity": "sha512-QZfjHNEzPY8+l0+fIXMvuQ2sJlplB4zgDZvA+NmvZsZv3EQwOcc1DuIU1VJUTWZ/RKouBMhDyNaBMx4sWvrzRA==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.0.tgz", + "integrity": "sha512-I1R/K7V07XsMJ12Oaxg/O9GfrysGTmCRhvZJBv0RE0NcULMzjqVpR5kRRQjHsz3J/bElU7HwCO7zkqL+MSUz+g==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.58.2", - "@typescript-eslint/types": "8.58.2", - "@typescript-eslint/typescript-estree": "8.58.2" + "@typescript-eslint/scope-manager": "8.59.0", + "@typescript-eslint/types": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2510,13 +2575,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.2.tgz", - "integrity": "sha512-f1WO2Lx8a9t8DARmcWAUPJbu0G20bJlj8L4z72K00TMeJAoyLr/tHhI/pzYBLrR4dXWkcxO1cWYZEOX8DKHTqA==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.0.tgz", + "integrity": "sha512-/uejZt4dSere1bx12WLlPfv8GktzcaDtuJ7s42/HEZ5zGj9oxRaD4bj7qwSunXkf+pbAhFt2zjpHYUiT5lHf0Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.58.2", + "@typescript-eslint/types": "8.59.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2578,7 +2643,7 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/@vitest/pretty-format": { + "node_modules/@vitest/pretty-format": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", @@ -2591,39 +2656,29 @@ "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/@vitest/utils": { + "node_modules/@vitest/spy": { "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", - "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", + "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", - "loupe": "^3.1.4", - "tinyrainbow": "^2.0.0" + "tinyspy": "^4.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/expect/node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@vitest/spy": { + "node_modules/@vitest/utils": { "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", + "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", "dev": true, "license": "MIT", "dependencies": { - "tinyspy": "^4.0.3" + "@vitest/pretty-format": "3.2.4", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" }, "funding": { "url": "https://opencollective.com/vitest" @@ -2779,9 +2834,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.10.tgz", - "integrity": "sha512-sUoJ3IMxx4AyRqO4MLeHlnGDkyXRoUG0/AI9fjK+vS72ekpV0yWVY7O0BVjmBcRtkNcsAO2QDZ4tdKKGoI6YaQ==", + "version": "2.10.21", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.21.tgz", + "integrity": "sha512-Q+rUQ7Uz8AHM7DEaNdwvfFCTq7a43lNTzuS94eiWqwyxfV/wJv+oUivef51T91mmRY4d4A1u9rcSvkeufCVXlA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -2792,9 +2847,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", + "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", "dev": true, "license": "MIT", "dependencies": { @@ -2803,9 +2858,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", - "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", "dev": true, "funding": [ { @@ -2823,11 +2878,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.9.0", - "caniuse-lite": "^1.0.30001759", - "electron-to-chromium": "^1.5.263", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.2.0" + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" @@ -2863,9 +2918,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001781", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001781.tgz", - "integrity": "sha512-RdwNCyMsNBftLjW6w01z8bKEvT6e/5tpPVEgtn22TiLGlstHOVecsX2KHFkD5e/vRnIE4EGzpuIODb3mtswtkw==", + "version": "1.0.30001790", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001790.tgz", + "integrity": "sha512-bOoxfJPyYo+ds6W0YfptaCWbFnJYjh2Y1Eow5lRv+vI2u8ganPZqNm1JwNh0t2ELQCqIWg4B3dWEusgAmsoyOw==", "dev": true, "funding": [ { @@ -3169,9 +3224,9 @@ "peer": true }, "node_modules/electron-to-chromium": { - "version": "1.5.322", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.322.tgz", - "integrity": "sha512-vFU34OcrvMcH66T+dYC3G4nURmgfDVewMIu6Q2urXpumAPSMmzvcn04KVVV8Opikq8Vs5nUbO/8laNhNRqSzYw==", + "version": "1.5.343", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.343.tgz", + "integrity": "sha512-YHnQ3MXI08icvL9ZKnEBy05F2EQ8ob01UaMOuMbM8l+4UcAq6MPPbBTJBbsBUg3H8JeZNt+O4fjsoWth3p6IFg==", "dev": true, "license": "ISC" }, @@ -3199,10 +3254,20 @@ "node": ">=10.13.0" } }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3213,32 +3278,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" } }, "node_modules/escalade": { @@ -3695,13 +3760,13 @@ } }, "node_modules/glob/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "brace-expansion": "^5.0.5" }, "engines": { "node": "18 || 20 || >=22" @@ -3740,9 +3805,9 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", "dev": true, "license": "MIT", "dependencies": { @@ -4148,6 +4213,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4169,6 +4237,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4190,6 +4261,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4211,6 +4285,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4410,9 +4487,9 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.36", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.36.tgz", - "integrity": "sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==", + "version": "2.0.38", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", + "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", "dev": true, "license": "MIT" }, @@ -4543,9 +4620,9 @@ } }, "node_modules/path-scurry/node_modules/lru-cache": { - "version": "11.2.7", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", - "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", "dev": true, "license": "BlueOak-1.0.0", "engines": { @@ -4583,9 +4660,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "dev": true, "funding": [ { @@ -4793,12 +4870,13 @@ } }, "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", "dev": true, "license": "MIT", "dependencies": { + "es-errors": "^1.3.0", "is-core-module": "^2.16.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" @@ -4824,9 +4902,9 @@ } }, "node_modules/rollup": { - "version": "4.60.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.0.tgz", - "integrity": "sha512-yqjxruMGBQJ2gG4HtjZtAfXArHomazDHoFwFFmZZl0r7Pdo7qCIXKqKHZc8yeoMgzJJ+pO6pEEHa+V7uzWlrAQ==", + "version": "4.60.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", + "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4840,31 +4918,31 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.60.0", - "@rollup/rollup-android-arm64": "4.60.0", - "@rollup/rollup-darwin-arm64": "4.60.0", - "@rollup/rollup-darwin-x64": "4.60.0", - "@rollup/rollup-freebsd-arm64": "4.60.0", - "@rollup/rollup-freebsd-x64": "4.60.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.60.0", - "@rollup/rollup-linux-arm-musleabihf": "4.60.0", - "@rollup/rollup-linux-arm64-gnu": "4.60.0", - "@rollup/rollup-linux-arm64-musl": "4.60.0", - "@rollup/rollup-linux-loong64-gnu": "4.60.0", - "@rollup/rollup-linux-loong64-musl": "4.60.0", - "@rollup/rollup-linux-ppc64-gnu": "4.60.0", - "@rollup/rollup-linux-ppc64-musl": "4.60.0", - "@rollup/rollup-linux-riscv64-gnu": "4.60.0", - "@rollup/rollup-linux-riscv64-musl": "4.60.0", - "@rollup/rollup-linux-s390x-gnu": "4.60.0", - "@rollup/rollup-linux-x64-gnu": "4.60.0", - "@rollup/rollup-linux-x64-musl": "4.60.0", - "@rollup/rollup-openbsd-x64": "4.60.0", - "@rollup/rollup-openharmony-arm64": "4.60.0", - "@rollup/rollup-win32-arm64-msvc": "4.60.0", - "@rollup/rollup-win32-ia32-msvc": "4.60.0", - "@rollup/rollup-win32-x64-gnu": "4.60.0", - "@rollup/rollup-win32-x64-msvc": "4.60.0", + "@rollup/rollup-android-arm-eabi": "4.60.2", + "@rollup/rollup-android-arm64": "4.60.2", + "@rollup/rollup-darwin-arm64": "4.60.2", + "@rollup/rollup-darwin-x64": "4.60.2", + "@rollup/rollup-freebsd-arm64": "4.60.2", + "@rollup/rollup-freebsd-x64": "4.60.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.2", + "@rollup/rollup-linux-arm-musleabihf": "4.60.2", + "@rollup/rollup-linux-arm64-gnu": "4.60.2", + "@rollup/rollup-linux-arm64-musl": "4.60.2", + "@rollup/rollup-linux-loong64-gnu": "4.60.2", + "@rollup/rollup-linux-loong64-musl": "4.60.2", + "@rollup/rollup-linux-ppc64-gnu": "4.60.2", + "@rollup/rollup-linux-ppc64-musl": "4.60.2", + "@rollup/rollup-linux-riscv64-gnu": "4.60.2", + "@rollup/rollup-linux-riscv64-musl": "4.60.2", + "@rollup/rollup-linux-s390x-gnu": "4.60.2", + "@rollup/rollup-linux-x64-gnu": "4.60.2", + "@rollup/rollup-linux-x64-musl": "4.60.2", + "@rollup/rollup-openbsd-x64": "4.60.2", + "@rollup/rollup-openharmony-arm64": "4.60.2", + "@rollup/rollup-win32-arm64-msvc": "4.60.2", + "@rollup/rollup-win32-ia32-msvc": "4.60.2", + "@rollup/rollup-win32-x64-gnu": "4.60.2", + "@rollup/rollup-win32-x64-msvc": "4.60.2", "fsevents": "~2.3.2" } }, @@ -5065,16 +5143,16 @@ } }, "node_modules/tailwindcss": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz", - "integrity": "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.4.tgz", + "integrity": "sha512-HhKppgO81FQof5m6TEnuBWCZGgfRAWbaeOaGT00KOy/Pf/j6oUihdvBpA7ltCeAvZpFhW3j0PTclkxsd4IXYDA==", "dev": true, "license": "MIT" }, "node_modules/tapable": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.2.tgz", - "integrity": "sha512-1MOpMXuhGzGL5TTCZFItxCc0AARf1EZFQkGqMm7ERKj8+Hgr5oLvJOVFcC+lRmR8hCe2S3jC4T5D7Vg/d7/fhA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "license": "MIT", "engines": { @@ -5093,14 +5171,14 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -5109,6 +5187,16 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tinyspy": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", @@ -5191,16 +5279,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.58.2", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.58.2.tgz", - "integrity": "sha512-V8iSng9mRbdZjl54VJ9NKr6ZB+dW0J3TzRXRGcSbLIej9jV86ZRtlYeTKDR/QLxXykocJ5icNzbsl2+5TzIvcQ==", + "version": "8.59.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.59.0.tgz", + "integrity": "sha512-BU3ONW9X+v90EcCH9ZS6LMackcVtxRLlI3XrYyqZIwVSHIk7Qf7bFw1z0M9Q0IUxhTMZCf8piY9hTYaNEIASrw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.58.2", - "@typescript-eslint/parser": "8.58.2", - "@typescript-eslint/typescript-estree": "8.58.2", - "@typescript-eslint/utils": "8.58.2" + "@typescript-eslint/eslint-plugin": "8.59.0", + "@typescript-eslint/parser": "8.59.0", + "@typescript-eslint/typescript-estree": "8.59.0", + "@typescript-eslint/utils": "8.59.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -5371,6 +5459,490 @@ } } }, + "node_modules/vite/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/vite/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, "node_modules/webpack-virtual-modules": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", From f3e7b3cb05e6fd382abd1a11c8dad3dcde4dd487 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 22 Apr 2026 19:01:41 +0100 Subject: [PATCH 165/250] perf: faster execution ctx and no opt (#7597) Makes the `ExecutionCtx` faster to create and destroy. Remove unconditional optimize from execute Signed-off-by: Joe Isaacs --- vortex-array/src/executor.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 4e2b2dfce53..e35b485972a 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -21,7 +21,6 @@ use std::env::VarError; use std::fmt; use std::fmt::Display; use std::sync::LazyLock; -use std::sync::atomic::AtomicUsize; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -107,7 +106,7 @@ impl ArrayRef { /// For safety, we will error when the number of execution iterations reaches a configurable /// maximum (default 128, override with `VORTEX_MAX_ITERATIONS`). pub fn execute_until(self, ctx: &mut ExecutionCtx) -> VortexResult { - let mut current = self.optimize()?; + let mut current = self; let mut stack: Vec = Vec::new(); for _ in 0..max_iterations() { @@ -221,19 +220,25 @@ impl StackFrame { /// Execution context for batch CPU compute. #[derive(Debug, Clone)] pub struct ExecutionCtx { - id: usize, session: VortexSession, + #[cfg(debug_assertions)] + id: usize, + #[cfg(debug_assertions)] ops: Vec, } impl ExecutionCtx { /// Create a new execution context with the given session. pub fn new(session: VortexSession) -> Self { - static EXEC_CTX_ID: AtomicUsize = AtomicUsize::new(0); - let id = EXEC_CTX_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); Self { session, - id, + #[cfg(debug_assertions)] + id: { + static EXEC_CTX_ID: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + EXEC_CTX_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + }, + #[cfg(debug_assertions)] ops: Vec::new(), } } @@ -255,20 +260,26 @@ impl ExecutionCtx { /// /// Use the [`format_args!`] macro to create the `msg` argument. pub fn log(&mut self, msg: fmt::Arguments<'_>) { + #[cfg(debug_assertions)] if tracing::enabled!(tracing::Level::DEBUG) { let formatted = format!(" - {msg}"); tracing::trace!("exec[{}]: {formatted}", self.id); self.ops.push(formatted); } + let _ = msg; } } impl Display for ExecutionCtx { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "exec[{}]", self.id) + #[cfg(debug_assertions)] + return write!(f, "exec[{}]", self.id); + #[cfg(not(debug_assertions))] + write!(f, "exec") } } +#[cfg(debug_assertions)] impl Drop for ExecutionCtx { fn drop(&mut self) { if !self.ops.is_empty() && tracing::enabled!(tracing::Level::DEBUG) { From 6302ee8b9e689e9efeaba1d22ddf95c4f1a3eb0f Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 22 Apr 2026 21:00:25 +0100 Subject: [PATCH 166/250] duckdb to_string: don't copy (#7582) 1. Don't allocate new string maps in to_string, copy to existing. 2. Add EXPLAIN test for to_string 3. Fix a bug in duckdb sql logic tests runner which didn't compare JSON correctly. Signed-off-by: Mikhail Kot --- .../cpp/include/duckdb_vx/table_function.h | 12 +---- vortex-duckdb/cpp/table_function.cpp | 47 ++++--------------- vortex-duckdb/src/datasource.rs | 12 ++--- .../src/duckdb/table_function/mod.rs | 46 +++++++----------- .../src/e2e_test/object_cache_test.rs | 2 + .../bin/sqllogictests-runner.rs | 14 ++++++ vortex-sqllogictest/slt/explain.slt | 26 ++++++++++ 7 files changed, 75 insertions(+), 84 deletions(-) create mode 100644 vortex-sqllogictest/slt/explain.slt diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h index 7fe9eed251c..80ef93acab7 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h @@ -38,18 +38,10 @@ void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_result ffi_resu size_t name_len, duckdb_logical_type ffi_type); -// String map for to_string result typedef struct duckdb_vx_string_map_ *duckdb_vx_string_map; - -// Create a new string map -duckdb_vx_string_map duckdb_vx_string_map_create(); - // Add a key-value pair to the string map void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, const char *value); -// Free the string map -void duckdb_vx_string_map_free(duckdb_vx_string_map map); - // Input data passed into the init_global and init_local callbacks. typedef struct { const void *bind_data; @@ -153,8 +145,8 @@ typedef struct { bool (*pushdown_complex_filter)(void *bind_data, duckdb_vx_expr expr, duckdb_vx_error *error_out); void *pushdown_expression; - duckdb_vx_string_map (*to_string)(void *bind_data); - // void *dynamic_to_string; + + void (*to_string)(void *bind_data, duckdb_vx_string_map map); double (*table_scan_progress)(duckdb_client_context ctx, void *bind_data, void *global_state); diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index 5a114613623..94406f6f949 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -397,24 +397,18 @@ OperatorPartitionData c_get_partition_data(ClientContext & /*context*/, return OperatorPartitionData(index); } +extern "C" void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, const char *value) { + D_ASSERT(map); + D_ASSERT(key); + D_ASSERT(value); + reinterpret_cast *>(map)->insert(key, value); +} + InsertionOrderPreservingMap c_to_string(TableFunctionToStringInput &input) { InsertionOrderPreservingMap result; - auto &bind = input.bind_data->Cast(); - - // Call the Rust side to get custom string representation if available - if (bind.info->vtab.to_string) { - auto map = bind.info->vtab.to_string(bind.ffi_data->DataPtr()); - if (map) { - // Copy the map contents to the result - auto *cpp_map = reinterpret_cast *>(map); - for (const auto &[key, value] : *cpp_map) { - result[key] = value; - } - // Free the map allocated by Rust - duckdb_vx_string_map_free(map); - } - } - + duckdb_vx_string_map ffi_map = reinterpret_cast(&result); + void *const ffi_bind = input.bind_data->Cast().ffi_data->DataPtr(); + static_cast(*input.table_function.function_info).vtab.to_string(ffi_bind, ffi_map); return result; } @@ -471,25 +465,4 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d } return DuckDBSuccess; } - -extern "C" duckdb_vx_string_map duckdb_vx_string_map_create() { - auto map = new InsertionOrderPreservingMap(); - return reinterpret_cast(map); -} - -extern "C" void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, const char *value) { - if (!map || !key || !value) { - return; - } - auto *cpp_map = reinterpret_cast *>(map); - (*cpp_map)[string(key)] = string(value); -} - -extern "C" void duckdb_vx_string_map_free(duckdb_vx_string_map map) { - if (!map) { - return; - } - auto *cpp_map = reinterpret_cast *>(map); - delete cpp_map; -} } // namespace vortex diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index ed416b088a2..ecac7f2388e 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -64,6 +64,7 @@ use crate::duckdb::Cardinality; use crate::duckdb::ClientContextRef; use crate::duckdb::ColumnStatistics; use crate::duckdb::DataChunkRef; +use crate::duckdb::DuckdbStringMapRef; use crate::duckdb::ExpressionRef; use crate::duckdb::LogicalType; use crate::duckdb::TableFilterSetRef; @@ -540,17 +541,12 @@ impl TableFunction for T { .ok_or_else(|| vortex_err!("batch id missing, no batches exported")) } - fn to_string(bind_data: &Self::BindData) -> Option> { - let mut result = Vec::new(); - - result.push(("Function".to_string(), "Vortex Scan".to_string())); - + fn to_string(bind_data: &Self::BindData, map: &mut DuckdbStringMapRef) { + map.push("Function", "Vortex Scan"); if !bind_data.filter_exprs.is_empty() { let mut filters = bind_data.filter_exprs.iter().map(|f| format!("{}", f)); - result.push(("Filters".to_string(), filters.join(" /\\\n"))); + map.push("Filters", &filters.join(" /\\\n")); } - - Some(result) } } diff --git a/vortex-duckdb/src/duckdb/table_function/mod.rs b/vortex-duckdb/src/duckdb/table_function/mod.rs index 7b9f9592728..5345d915fca 100644 --- a/vortex-duckdb/src/duckdb/table_function/mod.rs +++ b/vortex-duckdb/src/duckdb/table_function/mod.rs @@ -45,6 +45,18 @@ pub struct ColumnStatistics { pub has_null: bool, } +// String map lifetime is managed by C++ code +crate::lifetime_wrapper!(DuckdbStringMap, cpp::duckdb_vx_string_map, |_| {}); +impl DuckdbStringMapRef { + pub fn push(&mut self, key: &str, value: &str) { + let key = CString::new(key).unwrap_or_else(|_| CString::default()); + let value = CString::new(value).unwrap_or_else(|_| CString::default()); + unsafe { + cpp::duckdb_vx_string_map_insert(self.as_ptr(), key.as_ptr(), value.as_ptr()); + } + } +} + /// A trait that defines the supported operations for a table function in DuckDB. /// /// This trait does not yet cover the full C++ API, see table_function.hpp. @@ -154,9 +166,7 @@ pub trait TableFunction: Sized + Debug { ) -> VortexResult; /// Returns a vector of key-value pairs for EXPLAIN output - fn to_string(_bind_data: &Self::BindData) -> Option> { - None - } + fn to_string(bind_data: &Self::BindData, map: &mut DuckdbStringMapRef); // TODO(ngates): there are many more callbacks that can be configured. } @@ -223,35 +233,13 @@ impl DatabaseRef { } } -/// The to_string callback for a table function. unsafe extern "C-unwind" fn to_string_callback( bind_data: *mut c_void, -) -> cpp::duckdb_vx_string_map { + map: cpp::duckdb_vx_string_map, +) { let bind_data = unsafe { &*(bind_data as *const T::BindData) }; - - match T::to_string(bind_data) { - Some(map) => { - // Create a new C++ map - let cpp_map = unsafe { cpp::duckdb_vx_string_map_create() }; - - // Fill the map with key-value pairs - for (key, value) in map { - let key_cstr = CString::new(key).unwrap_or_else(|_| CString::default()); - let value_cstr = CString::new(value).unwrap_or_else(|_| CString::default()); - - unsafe { - cpp::duckdb_vx_string_map_insert( - cpp_map, - key_cstr.as_ptr(), - value_cstr.as_ptr(), - ); - } - } - - cpp_map - } - None => ptr::null_mut(), - } + let map = unsafe { DuckdbStringMap::borrow_mut(map) }; + T::to_string(bind_data, map); } /// The native function callback for a table function. diff --git a/vortex-duckdb/src/e2e_test/object_cache_test.rs b/vortex-duckdb/src/e2e_test/object_cache_test.rs index 4aab6d131d7..ae9dc38d0db 100644 --- a/vortex-duckdb/src/e2e_test/object_cache_test.rs +++ b/vortex-duckdb/src/e2e_test/object_cache_test.rs @@ -121,6 +121,8 @@ impl TableFunction for TestTableFunction { ) -> Option { None } + + fn to_string(_bind_data: &Self::BindData, _map: &mut crate::duckdb::DuckdbStringMapRef) {} } use crate::duckdb::Database; diff --git a/vortex-sqllogictest/bin/sqllogictests-runner.rs b/vortex-sqllogictest/bin/sqllogictests-runner.rs index d20b851677f..1dde50ada35 100644 --- a/vortex-sqllogictest/bin/sqllogictests-runner.rs +++ b/vortex-sqllogictest/bin/sqllogictests-runner.rs @@ -17,6 +17,7 @@ use indicatif::MultiProgress; use indicatif::ProgressBar; use indicatif::ProgressDrawTarget; use indicatif::ProgressStyle; +use sqllogictest::Normalizer; use sqllogictest::Record; use sqllogictest::Runner; use sqllogictest::parse_file; @@ -28,6 +29,18 @@ use vortex_sqllogictest::duckdb::DuckDB; use vortex_sqllogictest::duckdb::DuckDBTestError; use vortex_sqllogictest::utils::list_files; +fn duckdb_validator(normalizer: Normalizer, actual: &[Vec], expected: &[String]) -> bool { + let actual = actual.iter().flat_map(|strings| { + strings + .join(" ") + .trim_end() + .split('\n') + .map(|line| line.trim_end().to_string()) + .collect::>() + }); + Iterator::eq(actual, expected.iter().map(normalizer)) +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); @@ -127,6 +140,7 @@ async fn main() -> anyhow::Result<()> { duckdb_runner.add_label("duckdb"); duckdb_runner.with_column_validator(strict_column_validator); duckdb_runner.with_normalizer(value_normalizer); + duckdb_runner.with_validator(duckdb_validator); for record in records.iter() { if let Record::Halt { .. } = record { diff --git a/vortex-sqllogictest/slt/explain.slt b/vortex-sqllogictest/slt/explain.slt new file mode 100644 index 00000000000..81619291bf6 --- /dev/null +++ b/vortex-sqllogictest/slt/explain.slt @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +include ./setup.slt + +onlyif duckdb +query I +COPY (SELECT * FROM (VALUES ('Hello'), ('Hi'), ('Hey')) AS t(str)) TO '$__TEST_DIR__/explain.vortex'; +---- +3 + +onlyif duckdb +query TT +EXPLAIN (FORMAT json) SELECT * FROM '$__TEST_DIR__/explain.vortex'; +---- +physical_plan [ + { + "name": "READ_VORTEX", + "children": [], + "extra_info": { + "Function": "Vortex Scan", + "Projections": "str", + "Estimated Cardinality": "3" + } + } +] From c91d83305e4cea98095431111e449eb632bf79ef Mon Sep 17 00:00:00 2001 From: Baris Palaska Date: Wed, 22 Apr 2026 23:18:25 +0100 Subject: [PATCH 167/250] Extend DType::least_supertype to cover FixedSizeList, List, and tensor extensions (#7596) Widening Vector and Vector now returns Vector instead of None. --------- Signed-off-by: Baris Palaska --- vortex-array/src/dtype/coercion.rs | 326 +++++++++++++++++- vortex-tensor/public-api.lock | 4 + vortex-tensor/src/types/fixed_shape/vtable.rs | 88 +++++ vortex-tensor/src/types/vector/vtable.rs | 71 ++++ 4 files changed, 470 insertions(+), 19 deletions(-) diff --git a/vortex-array/src/dtype/coercion.rs b/vortex-array/src/dtype/coercion.rs index 528d9b673b2..d57c0b9c1f4 100644 --- a/vortex-array/src/dtype/coercion.rs +++ b/vortex-array/src/dtype/coercion.rs @@ -3,6 +3,8 @@ //! Utilities for performing type coercion. +use std::sync::Arc; + use crate::dtype::DType; use crate::dtype::PType; use crate::dtype::decimal::DecimalDType; @@ -77,14 +79,29 @@ impl DType { /// The core primitive — what type can hold both `self` and `other`? /// Returns `None` if no common supertype exists. pub fn least_supertype(&self, other: &DType) -> Option { - // 1. Identity (ignoring nullability): return self with union nullability - if self.eq_ignore_nullability(other) { - return Some(self.with_nullability(self.nullability() | other.nullability())); + let union_null = self.nullability() | other.nullability(); + + if let ( + DType::FixedSizeList(lhs_elem, lhs_size, _), + DType::FixedSizeList(rhs_elem, rhs_size, _), + ) = (self, other) + && lhs_size == rhs_size + { + let elem = lhs_elem.least_supertype(rhs_elem)?; + return Some(DType::FixedSizeList(Arc::new(elem), *lhs_size, union_null)); } - let union_null = self.nullability() | other.nullability(); + if let (DType::List(lhs_elem, _), DType::List(rhs_elem, _)) = (self, other) { + let elem = lhs_elem.least_supertype(rhs_elem)?; + return Some(DType::List(Arc::new(elem), union_null)); + } - // 2. Null + X: return X as nullable + // Identity (ignoring nullability): return self with union nullability + if self.eq_ignore_nullability(other) { + return Some(self.with_nullability(union_null)); + } + + // Null + X: return X as nullable if matches!(self, DType::Null) { return Some(other.as_nullable()); } @@ -92,7 +109,7 @@ impl DType { return Some(self.as_nullable()); } - // 3. Bool + numeric: return the numeric type (with union nullability) + // Bool + numeric: return the numeric type (with union nullability) if self.is_boolean() && other.is_numeric() { return Some(other.with_nullability(union_null)); } @@ -100,19 +117,19 @@ impl DType { return Some(self.with_nullability(union_null)); } - // 4. Primitive + Primitive (different ptypes): delegate to PType::least_supertype + // Primitive + Primitive (different ptypes): delegate to PType::least_supertype if let (DType::Primitive(lhs_p, _), DType::Primitive(rhs_p, _)) = (self, other) { return lhs_p .least_supertype(*rhs_p) .map(|p| DType::Primitive(p, union_null)); } - // 5. Decimal + Decimal: compute wider decimal + // Decimal + Decimal: compute wider decimal if let (DType::Decimal(lhs_d, _), DType::Decimal(rhs_d, _)) = (self, other) { return decimal_least_supertype(*lhs_d, *rhs_d).map(|d| DType::Decimal(d, union_null)); } - // 6. Decimal + integer Primitive: convert integer to Decimal, then widen + // Decimal + integer Primitive: convert integer to Decimal, then widen if let (DType::Decimal(dec, _), DType::Primitive(p, _)) = (self, other) && p.is_int() { @@ -126,7 +143,7 @@ impl DType { return decimal_least_supertype(int_dec, *dec).map(|d| DType::Decimal(d, union_null)); } - // 7. Extension + anything: delegate to vtable + // Extension + anything: delegate to vtable if let DType::Extension(ext) = self { return ext .least_supertype(other) @@ -138,7 +155,6 @@ impl DType { .map(|dt| dt.with_nullability(union_null)); } - // 8. Everything else: no common supertype None } @@ -152,22 +168,37 @@ impl DType { /// Is there any implicit coercion path from `other` to `self`? pub fn can_coerce_from(&self, other: &DType) -> bool { - // 1. Same type (ignoring nullability): check nullability compatibility + if let ( + DType::FixedSizeList(target_elem, target_size, _), + DType::FixedSizeList(source_elem, source_size, _), + ) = (self, other) + { + return target_size == source_size + && (self.is_nullable() || !other.is_nullable()) + && target_elem.can_coerce_from(source_elem); + } + + if let (DType::List(target_elem, _), DType::List(source_elem, _)) = (self, other) { + return (self.is_nullable() || !other.is_nullable()) + && target_elem.can_coerce_from(source_elem); + } + + // Same type (ignoring nullability): check nullability compatibility if self.eq_ignore_nullability(other) { return self.is_nullable() || !other.is_nullable(); } - // 2. Null → nullable target + // Null → nullable target if matches!(other, DType::Null) { return self.is_nullable(); } - // 3. Bool → numeric + // Bool → numeric if other.is_boolean() && self.is_numeric() { return self.is_nullable() || !other.is_nullable(); } - // 4. Primitive widening: true if least_supertype(source, target) == target + // Primitive widening: true if least_supertype(source, target) == target if let (DType::Primitive(..), DType::Primitive(..)) = (self, other) { return other .least_supertype(self) @@ -175,7 +206,7 @@ impl DType { && (self.is_nullable() || !other.is_nullable()); } - // 5. Decimal widening + // Decimal widening if let (DType::Decimal(target, _), DType::Decimal(source, _)) = (self, other) { let target_integral = target.precision() as i16 - target.scale() as i16; let source_integral = source.precision() as i16 - source.scale() as i16; @@ -184,7 +215,7 @@ impl DType { && (self.is_nullable() || !other.is_nullable()); } - // 6. Integer → Decimal + // Integer → Decimal if let (DType::Decimal(dec, _), DType::Primitive(p, _)) = (self, other) && p.is_int() { @@ -194,12 +225,11 @@ impl DType { && (self.is_nullable() || !other.is_nullable()); } - // 7. Extension: delegate to vtable + // Extension: delegate to vtable if let DType::Extension(ext) = self { return ext.can_coerce_from(other); } - // 8. Everything else: false false } @@ -274,6 +304,8 @@ fn decimal_least_supertype(a: DecimalDType, b: DecimalDType) -> Option vortex_array::dtype::extension::ExtId +pub fn vortex_tensor::fixed_shape::FixedShapeTensor::least_supertype(ext_dtype: &vortex_array::dtype::extension::typed::ExtDType, other: &vortex_array::dtype::DType) -> core::option::Option + pub fn vortex_tensor::fixed_shape::FixedShapeTensor::serialize_metadata(&self, metadata: &Self::Metadata) -> vortex_error::VortexResult> pub fn vortex_tensor::fixed_shape::FixedShapeTensor::unpack_native<'a>(_ext_dtype: &'a vortex_array::dtype::extension::typed::ExtDType, storage_value: &'a vortex_array::scalar::scalar_value::ScalarValue) -> vortex_error::VortexResult @@ -560,6 +562,8 @@ pub fn vortex_tensor::vector::Vector::deserialize_metadata(&self, _metadata: &[u pub fn vortex_tensor::vector::Vector::id(&self) -> vortex_array::dtype::extension::ExtId +pub fn vortex_tensor::vector::Vector::least_supertype(ext_dtype: &vortex_array::dtype::extension::typed::ExtDType, other: &vortex_array::dtype::DType) -> core::option::Option + pub fn vortex_tensor::vector::Vector::serialize_metadata(&self, _metadata: &Self::Metadata) -> vortex_error::VortexResult> pub fn vortex_tensor::vector::Vector::unpack_native<'a>(_ext_dtype: &'a vortex_array::dtype::extension::typed::ExtDType, storage_value: &'a vortex_array::scalar::scalar_value::ScalarValue) -> vortex_error::VortexResult diff --git a/vortex-tensor/src/types/fixed_shape/vtable.rs b/vortex-tensor/src/types/fixed_shape/vtable.rs index 7d69b9a7c44..d0a4b7842f1 100644 --- a/vortex-tensor/src/types/fixed_shape/vtable.rs +++ b/vortex-tensor/src/types/fixed_shape/vtable.rs @@ -33,6 +33,22 @@ impl ExtVTable for FixedShapeTensor { proto::deserialize(metadata) } + fn least_supertype(ext_dtype: &ExtDType, other: &DType) -> Option { + let DType::Extension(other_ext) = other else { + return None; + }; + // Only element dtype may widen — shape, dim_names, and permutation must match exactly. + let other_metadata = other_ext.metadata_opt::()?; + if ext_dtype.metadata() != other_metadata { + return None; + } + let widened = ext_dtype + .storage_dtype() + .least_supertype(other_ext.storage_dtype())?; + let ext = ExtDType::::try_new(ext_dtype.metadata().clone(), widened).ok()?; + Some(DType::Extension(ext.erased())) + } + fn validate_dtype(ext_dtype: &ExtDType) -> VortexResult<()> { let storage_dtype = ext_dtype.storage_dtype(); let DType::FixedSizeList(element_dtype, list_size, _nullability) = storage_dtype else { @@ -76,7 +92,13 @@ impl ExtVTable for FixedShapeTensor { #[cfg(test)] mod tests { + use std::sync::Arc; + use rstest::rstest; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtVTable; use vortex_error::VortexResult; @@ -118,4 +140,70 @@ mod tests { ) -> VortexResult<()> { assert_roundtrip(&metadata?) } + + /// Constructs a `FixedShapeTensor` ext dtype wrapped in `DType::Extension`. + fn tensor_dtype( + metadata: FixedShapeTensorMetadata, + element: PType, + list_size: u32, + ) -> VortexResult { + let storage = DType::FixedSizeList( + Arc::new(DType::Primitive(element, Nullability::NonNullable)), + list_size, + Nullability::NonNullable, + ); + Ok(DType::Extension( + ExtDType::::try_new(metadata, storage)?.erased(), + )) + } + + #[test] + fn tensor_widens_element_when_metadata_matches() -> VortexResult<()> { + let metadata = FixedShapeTensorMetadata::new(vec![2, 3]); + let lhs = tensor_dtype(metadata.clone(), PType::F32, 6)?; + let rhs = tensor_dtype(metadata.clone(), PType::F64, 6)?; + let expected = tensor_dtype(metadata, PType::F64, 6)?; + assert_eq!(lhs.least_supertype(&rhs), Some(expected)); + Ok(()) + } + + #[test] + fn tensor_different_shape_returns_none() -> VortexResult<()> { + let lhs = tensor_dtype(FixedShapeTensorMetadata::new(vec![2, 3]), PType::F32, 6)?; + let rhs = tensor_dtype(FixedShapeTensorMetadata::new(vec![3, 2]), PType::F32, 6)?; + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } + + #[test] + fn tensor_different_permutation_returns_none() -> VortexResult<()> { + let lhs_metadata = + FixedShapeTensorMetadata::new(vec![2, 3]).with_permutation(vec![0, 1])?; + let rhs_metadata = + FixedShapeTensorMetadata::new(vec![2, 3]).with_permutation(vec![1, 0])?; + let lhs = tensor_dtype(lhs_metadata, PType::F32, 6)?; + let rhs = tensor_dtype(rhs_metadata, PType::F32, 6)?; + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } + + #[test] + fn tensor_different_dim_names_returns_none() -> VortexResult<()> { + let lhs_metadata = FixedShapeTensorMetadata::new(vec![2, 3]) + .with_dim_names(vec!["x".into(), "y".into()])?; + let rhs_metadata = FixedShapeTensorMetadata::new(vec![2, 3]) + .with_dim_names(vec!["rows".into(), "cols".into()])?; + let lhs = tensor_dtype(lhs_metadata, PType::F32, 6)?; + let rhs = tensor_dtype(rhs_metadata, PType::F32, 6)?; + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } + + #[test] + fn tensor_vs_non_extension_returns_none() -> VortexResult<()> { + let lhs = tensor_dtype(FixedShapeTensorMetadata::new(vec![2, 3]), PType::F32, 6)?; + let rhs = DType::Primitive(PType::F32, Nullability::NonNullable); + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } } diff --git a/vortex-tensor/src/types/vector/vtable.rs b/vortex-tensor/src/types/vector/vtable.rs index d870e7a8e0d..a8b3c10994a 100644 --- a/vortex-tensor/src/types/vector/vtable.rs +++ b/vortex-tensor/src/types/vector/vtable.rs @@ -31,6 +31,20 @@ impl ExtVTable for Vector { Ok(EmptyMetadata) } + fn least_supertype(ext_dtype: &ExtDType, other: &DType) -> Option { + let DType::Extension(other_ext) = other else { + return None; + }; + if !other_ext.is::() { + return None; + } + let widened = ext_dtype + .storage_dtype() + .least_supertype(other_ext.storage_dtype())?; + let ext = ExtDType::::try_new(EmptyMetadata, widened).ok()?; + Some(DType::Extension(ext.erased())) + } + fn validate_dtype(ext_dtype: &ExtDType) -> VortexResult<()> { let storage_dtype = ext_dtype.storage_dtype(); let DType::FixedSizeList(element_dtype, _list_size, _nullability) = storage_dtype else { @@ -138,4 +152,61 @@ mod tests { assert_eq!(deserialized, EmptyMetadata); Ok(()) } + + /// Constructs a `Vector` ext dtype wrapped in `DType::Extension`. + fn vector_dtype(ptype: PType, dims: u32) -> VortexResult { + vector_dtype_with_outer(ptype, dims, Nullability::NonNullable) + } + + /// Constructs a `Vector` ext dtype with the given outer `Nullability`, wrapped in + /// `DType::Extension`. + fn vector_dtype_with_outer(ptype: PType, dims: u32, outer: Nullability) -> VortexResult { + let storage = vector_storage_dtype(ptype, dims, outer); + Ok(DType::Extension( + ExtDType::::try_new(EmptyMetadata, storage)?.erased(), + )) + } + + #[test] + fn vector_widens_float_precision() -> VortexResult<()> { + let lhs = vector_dtype(PType::F32, 768)?; + let rhs = vector_dtype(PType::F64, 768)?; + let expected = vector_dtype(PType::F64, 768)?; + assert_eq!(lhs.least_supertype(&rhs), Some(expected)); + Ok(()) + } + + #[test] + fn vector_dim_mismatch_returns_none() -> VortexResult<()> { + let lhs = vector_dtype(PType::F32, 768)?; + let rhs = vector_dtype(PType::F32, 1024)?; + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } + + #[test] + fn vector_vs_non_extension_returns_none() -> VortexResult<()> { + let lhs = vector_dtype(PType::F32, 768)?; + let rhs = DType::Primitive(PType::F32, Nullability::NonNullable); + assert_eq!(lhs.least_supertype(&rhs), None); + Ok(()) + } + + #[test] + fn vector_unions_outer_nullability_with_float_widening() -> VortexResult<()> { + let lhs = vector_dtype_with_outer(PType::F32, 4, Nullability::NonNullable)?; + let rhs = vector_dtype_with_outer(PType::F64, 4, Nullability::Nullable)?; + let expected = vector_dtype_with_outer(PType::F64, 4, Nullability::Nullable)?; + assert_eq!(lhs.least_supertype(&rhs), Some(expected)); + Ok(()) + } + + #[test] + fn vector_same_ptype_unions_outer_nullability() -> VortexResult<()> { + let lhs = vector_dtype_with_outer(PType::F32, 4, Nullability::NonNullable)?; + let rhs = vector_dtype_with_outer(PType::F32, 4, Nullability::Nullable)?; + let expected = vector_dtype_with_outer(PType::F32, 4, Nullability::Nullable)?; + assert_eq!(lhs.least_supertype(&rhs), Some(expected)); + Ok(()) + } } From 2167e611b55d7f9d88f536a06dbffde81a538c39 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Thu, 23 Apr 2026 13:39:47 +0100 Subject: [PATCH 168/250] remove unused methods from duckdb vtab (#7602) 1. Remove unused and not set methods from duckdb vtab. 2. Correctly set MAX_THREADS for table function (was u64::MAX but duckdb uses another constant). 3. Move boolean definitions like filter pushdown to C++ part. 4. Remove named parameters since we don't support them. 5. Fix a memory leak when throwing errors. Signed-off-by: Mikhail Kot --- vortex-duckdb/cpp/error.cpp | 34 ++-- vortex-duckdb/cpp/file_system.cpp | 3 - vortex-duckdb/cpp/include/duckdb_vx/error.hpp | 1 - .../cpp/include/duckdb_vx/table_function.h | 49 +----- vortex-duckdb/cpp/table_function.cpp | 158 +++++++----------- vortex-duckdb/src/datasource.rs | 26 +-- .../src/duckdb/table_function/bind.rs | 19 --- .../src/duckdb/table_function/mod.rs | 45 +---- 8 files changed, 77 insertions(+), 258 deletions(-) diff --git a/vortex-duckdb/cpp/error.cpp b/vortex-duckdb/cpp/error.cpp index 4e14b34a76d..f00a9a3ecd1 100644 --- a/vortex-duckdb/cpp/error.cpp +++ b/vortex-duckdb/cpp/error.cpp @@ -1,18 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include -#include - +#include +#include "duckdb_vx/error.h" #include "duckdb_vx/duckdb_diagnostics.h" + DUCKDB_INCLUDES_BEGIN -#include "duckdb/common/exception.hpp" -#include "duckdb/common/types/vector_buffer.hpp" -#include "duckdb/common/types/vector.hpp" +#include "duckdb.h" +#include "duckdb/common/assert.hpp" DUCKDB_INCLUDES_END -#include "duckdb_vx.h" - extern "C" duckdb_vx_error duckdb_vx_error_create(const char *message, size_t message_length) { return reinterpret_cast(new std::string(message, message_length)); } @@ -33,26 +30,15 @@ std::string IntoErrString(duckdb_vx_error error) { if (!error) { return {}; } - return *reinterpret_cast(error); + std::string *const error_str = reinterpret_cast(error); + std::string out = std::move(*error_str); + duckdb_vx_error_free(error); + return out; } duckdb_state SetError(duckdb_vx_error *error_out, std::string_view message) { - assert(error_out != nullptr && "SetError called with null error_out"); + D_ASSERT(error_out != nullptr && "SetError called with null error_out"); *error_out = duckdb_vx_error_create(message.data(), message.size()); return DuckDBError; } - -duckdb_state HandleException(std::exception_ptr ex, duckdb_vx_error *error_out) { - if (!ex) { - return SetError(error_out, "Unknown error"); - } - - try { - std::rethrow_exception(ex); - } catch (const std::exception &caught) { - return SetError(error_out, caught.what()); - } catch (...) { - return SetError(error_out, "Unknown error"); - } -} } // namespace vortex diff --git a/vortex-duckdb/cpp/file_system.cpp b/vortex-duckdb/cpp/file_system.cpp index 43c2f3e9907..4e5c6cab414 100644 --- a/vortex-duckdb/cpp/file_system.cpp +++ b/vortex-duckdb/cpp/file_system.cpp @@ -12,12 +12,9 @@ DUCKDB_INCLUDES_BEGIN #include DUCKDB_INCLUDES_END -#include -#include #include using namespace duckdb; -using vortex::HandleException; using vortex::SetError; extern "C" duckdb_vx_file_handle diff --git a/vortex-duckdb/cpp/include/duckdb_vx/error.hpp b/vortex-duckdb/cpp/include/duckdb_vx/error.hpp index fc4b185ac69..5768421acf1 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/error.hpp +++ b/vortex-duckdb/cpp/include/duckdb_vx/error.hpp @@ -17,5 +17,4 @@ DUCKDB_INCLUDES_END namespace vortex { std::string IntoErrString(duckdb_vx_error error); duckdb_state SetError(duckdb_vx_error *error_out, std::string_view message); -duckdb_state HandleException(std::exception_ptr ex, duckdb_vx_error *error_out); } // namespace vortex diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h index 80ef93acab7..ddf55b532a1 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/table_function.h @@ -12,9 +12,8 @@ #include "error.h" #include "table_filter.h" #include "duckdb_vx/data.h" -#include "duckdb_vx/client_context.h" -#ifdef __cplusplus /* If compiled as C++, use C ABI */ +#ifdef __cplusplus extern "C" { #endif @@ -22,16 +21,10 @@ extern "C" { typedef struct duckdb_vx_tfunc_bind_input_ *duckdb_vx_tfunc_bind_input; typedef struct duckdb_vx_tfunc_bind_result_ *duckdb_vx_tfunc_bind_result; -// Fetch the parameter count from the bind info. -size_t duckdb_vx_tfunc_bind_input_get_parameter_count(duckdb_vx_tfunc_bind_input ffi_input); - // Fetch a parameter from the bind info. // The caller is responsible for freeing the value using duckdb_value_free. duckdb_value duckdb_vx_tfunc_bind_input_get_parameter(duckdb_vx_tfunc_bind_input ffi_input, size_t index); -duckdb_value duckdb_vx_tfunc_bind_input_get_named_parameter(duckdb_vx_tfunc_bind_input ffi_input, - const char *name_str); - // Add a result column to the bind info. void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_result ffi_result, const char *name_str, @@ -90,23 +83,13 @@ typedef struct { bool has_null; } duckdb_column_statistics; -typedef idx_t column_t; - -// A transparent DuckDB table function vtable, which can be used to configure a table function. -// See duckdb/include/function/tfunc.hpp for details on each field. +// vtable mimicking subset of TableFunction. +// See duckdb/include/function/tfunc.hpp typedef struct { - // The name of the table function. const char *name; - - // The parameters of the table function. const duckdb_logical_type *parameters; size_t parameter_count; - // The named parameters of the table function. - const duckdb_logical_type *named_parameter_types; - const char *const *named_parameter_names; - size_t named_parameter_count; - duckdb_vx_data (*bind)(duckdb_client_context ctx, duckdb_vx_tfunc_bind_input input, duckdb_vx_tfunc_bind_result result, @@ -114,9 +97,6 @@ typedef struct { duckdb_vx_data (*bind_data_clone)(const void *bind_data, duckdb_vx_error *error_out); - // void *bind_replace; - // void *bind_operator; - duckdb_vx_data (*init_global)(const duckdb_vx_tfunc_init_input *input, duckdb_vx_error *error_out); duckdb_vx_data (*init_local)(const duckdb_vx_tfunc_init_input *input, @@ -130,22 +110,15 @@ typedef struct { duckdb_data_chunk data_chunk_out, duckdb_vx_error *error_out); - // void *in_out_function; - // void *in_out_function_final; - - // false if statistics are not available bool (*statistics)(duckdb_client_context context, const void *bind_data, size_t column_index, duckdb_column_statistics *stats_out); - // void *dependency; void (*cardinality)(void *bind_data, duckdb_vx_node_statistics *node_stats_out); bool (*pushdown_complex_filter)(void *bind_data, duckdb_vx_expr expr, duckdb_vx_error *error_out); - void *pushdown_expression; - void (*to_string)(void *bind_data, duckdb_vx_string_map map); double (*table_scan_progress)(duckdb_client_context ctx, void *bind_data, void *global_state); @@ -154,25 +127,11 @@ typedef struct { void *init_global_data, void *init_local_data, duckdb_vx_error *error_out); - // void *get_bind_info; - // void *type_pushdown; - // void *get_multi_file_reader; - // void *supports_pushdown_type; - // void *get_partition_info; - // void *get_partition_stats; - // void *get_row_id_columns; - - bool projection_pushdown; - bool filter_pushdown; - bool filter_prune; - bool sampling_pushdown; - bool late_materialization; - idx_t max_threads; } duckdb_vx_tfunc_vtab_t; // A single function for configuring the DuckDB table function vtable. duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const duckdb_vx_tfunc_vtab_t *vtab); -#ifdef __cplusplus /* End C ABI */ +#ifdef __cplusplus } #endif diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index 94406f6f949..21832a20950 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "duckdb_vx/table_function.h" +#include "duckdb_vx/data.hpp" #include "duckdb_vx/duckdb_diagnostics.h" +#include "duckdb_vx/error.hpp" +#include "duckdb_vx/table_function.h" DUCKDB_INCLUDES_BEGIN #include "duckdb.h" @@ -14,40 +16,36 @@ DUCKDB_INCLUDES_BEGIN #include "duckdb/parser/parsed_data/create_table_function_info.hpp" DUCKDB_INCLUDES_END -#include "duckdb_vx.h" -#include "duckdb_vx/data.hpp" -#include "duckdb_vx/error.hpp" - using namespace duckdb; +using vortex::CData; +using vortex::IntoErrString; -namespace vortex { struct CTableFunctionInfo final : TableFunctionInfo { - explicit CTableFunctionInfo(const duckdb_vx_tfunc_vtab_t &vtab) - : vtab(vtab), max_threads(vtab.max_threads) { + explicit CTableFunctionInfo(const duckdb_vx_tfunc_vtab_t &vtab) : vtab(vtab) { } duckdb_vx_tfunc_vtab_t vtab; - idx_t max_threads; }; struct CTableBindData final : TableFunctionData { CTableBindData(unique_ptr info_p, - unique_ptr ffi_data_p, + unique_ptr ffi_data_p, const vector &types) : info(std::move(info_p)), ffi_data(std::move(ffi_data_p)), types(types) { } unique_ptr Copy() const override { - assert(info->vtab.bind_data_clone != nullptr); + D_ASSERT(info->vtab.bind_data_clone); duckdb_vx_error error_out = nullptr; const auto copied_ffi_data = info->vtab.bind_data_clone(ffi_data->DataPtr(), &error_out); if (error_out) { throw BinderException(IntoErrString(error_out)); } - return make_uniq(make_uniq(info->vtab), - unique_ptr(reinterpret_cast(copied_ffi_data)), - types); + + auto info_p = make_uniq(info->vtab); + auto ffi_data_p = unique_ptr(reinterpret_cast(copied_ffi_data)); + return make_uniq(std::move(info_p), std::move(ffi_data_p), types); } unique_ptr info; @@ -56,23 +54,21 @@ struct CTableBindData final : TableFunctionData { }; struct CTableGlobalData final : GlobalTableFunctionState { - explicit CTableGlobalData(unique_ptr ffi_data_p, idx_t max_threads_p) - : ffi_data(std::move(ffi_data_p)), max_threads(max_threads_p) { + explicit CTableGlobalData(unique_ptr ffi_data_p) : ffi_data(std::move(ffi_data_p)) { } - unique_ptr ffi_data; - idx_t max_threads; - idx_t MaxThreads() const override { - return max_threads; + return GlobalTableFunctionState::MAX_THREADS; } + + unique_ptr ffi_data; }; struct CTableLocalData final : LocalTableFunctionState { - explicit CTableLocalData(unique_ptr ffi_data_p) : ffi_data(std::move(ffi_data_p)) { + explicit CTableLocalData(unique_ptr ffi_data_p) : ffi_data(std::move(ffi_data_p)) { } - unique_ptr ffi_data; + unique_ptr ffi_data; }; /** @@ -236,9 +232,8 @@ unique_ptr c_init_global(ClientContext &context, Table throw BinderException(IntoErrString(error_out)); } - return make_uniq( - unique_ptr(reinterpret_cast(ffi_global_data)), - bind.info->max_threads); + auto cdata = unique_ptr(reinterpret_cast(ffi_global_data)); + return make_uniq(std::move(cdata)); } unique_ptr c_init_local(ExecutionContext &context, @@ -263,8 +258,7 @@ unique_ptr c_init_local(ExecutionContext &context, throw BinderException(IntoErrString(error_out)); } - return make_uniq( - unique_ptr(reinterpret_cast(ffi_local_data))); + return make_uniq(unique_ptr(reinterpret_cast(ffi_local_data))); } void c_function(ClientContext &context, TableFunctionInput &input, DataChunk &output) { @@ -275,34 +269,26 @@ void c_function(ClientContext &context, TableFunctionInput &input, DataChunk &ou auto global_data = input.global_state->Cast().ffi_data->DataPtr(); auto local_data = input.local_state->Cast().ffi_data->DataPtr(); + duckdb_data_chunk chunk = reinterpret_cast(&output); duckdb_vx_error error_out = nullptr; - bind.info->vtab.function(ctx, - bind_data, - global_data, - local_data, - reinterpret_cast(&output), - &error_out); + bind.info->vtab.function(ctx, bind_data, global_data, local_data, chunk, &error_out); if (error_out) { throw InvalidInputException(IntoErrString(error_out)); } } -void c_pushdown_complex_filter(ClientContext & /*context*/, - LogicalGet & /*get*/, +void c_pushdown_complex_filter(ClientContext &, + LogicalGet &, FunctionData *bind_data, vector> &filters) { - if (filters.empty()) { - return; - } - auto &bind = bind_data->Cast(); + void *const ffi_bind = bind.ffi_data->DataPtr(); for (auto iter = filters.begin(); iter != filters.end();) { duckdb_vx_error error_out = nullptr; - auto pushed = - bind.info->vtab.pushdown_complex_filter(bind_data->Cast().ffi_data->DataPtr(), - reinterpret_cast(iter->get()), - &error_out); + duckdb_vx_expr ffi_expr = reinterpret_cast(iter->get()); + + const bool pushed = bind.info->vtab.pushdown_complex_filter(ffi_bind, ffi_expr, &error_out); if (error_out) { throw BinderException(IntoErrString(error_out)); } @@ -312,55 +298,32 @@ void c_pushdown_complex_filter(ClientContext & /*context*/, } } -unique_ptr c_cardinality(ClientContext & /*context*/, const FunctionData *bind_data) { +unique_ptr c_cardinality(ClientContext &, const FunctionData *bind_data) { auto &bind = bind_data->Cast(); - duckdb_vx_node_statistics node_stats_out = { - .estimated_cardinality = 0, - .max_cardinality = 0, - .has_estimated_cardinality = false, - .has_max_cardinality = false, - }; - bind.info->vtab.cardinality(bind_data->Cast().ffi_data->DataPtr(), &node_stats_out); - - auto stats = make_uniq(); - stats->has_estimated_cardinality = node_stats_out.has_estimated_cardinality; - stats->estimated_cardinality = node_stats_out.estimated_cardinality; - stats->has_max_cardinality = node_stats_out.has_max_cardinality; - stats->max_cardinality = node_stats_out.max_cardinality; + duckdb_vx_node_statistics stats = {}; + bind.info->vtab.cardinality(bind.ffi_data->DataPtr(), &stats); - return stats; -} + auto out = make_uniq(); + out->has_estimated_cardinality = stats.has_estimated_cardinality; + out->estimated_cardinality = stats.estimated_cardinality; + out->has_max_cardinality = stats.has_max_cardinality; + out->max_cardinality = stats.max_cardinality; -extern "C" size_t duckdb_vx_tfunc_bind_input_get_parameter_count(duckdb_vx_tfunc_bind_input ffi_input) { - if (!ffi_input) { - return 0; - } - const auto input = reinterpret_cast(ffi_input); - return input->inputs.size(); + return out; } extern "C" duckdb_value duckdb_vx_tfunc_bind_input_get_parameter(duckdb_vx_tfunc_bind_input ffi_input, size_t index) { - if (!ffi_input || index >= duckdb_vx_tfunc_bind_input_get_parameter_count(ffi_input)) { - return nullptr; - } - const auto info = reinterpret_cast(ffi_input); - return reinterpret_cast(new Value(info->inputs[index])); -} - -extern "C" duckdb_value duckdb_vx_tfunc_bind_input_get_named_parameter(duckdb_vx_tfunc_bind_input ffi_input, - const char *name) { - if (!ffi_input || !name) { + if (!ffi_input) { return nullptr; } - const auto info = reinterpret_cast(ffi_input); - if (!info->named_parameters.contains(name)) { + const TableFunctionBindInput &input = *reinterpret_cast(ffi_input); + if (index >= input.inputs.size()) { return nullptr; } - auto value = duckdb::make_uniq(info->named_parameters.at(name)); - return reinterpret_cast(value.release()); + return reinterpret_cast(new Value(input.inputs[index])); } extern "C" void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_result ffi_result, @@ -377,24 +340,21 @@ extern "C" void duckdb_vx_tfunc_bind_result_add_column(duckdb_vx_tfunc_bind_resu result->return_types.emplace_back(*logical_type); } -OperatorPartitionData c_get_partition_data(ClientContext & /*context*/, - TableFunctionGetPartitionInput &input) { +OperatorPartitionData c_get_partition_data(ClientContext &, TableFunctionGetPartitionInput &input) { if (input.partition_info.RequiresPartitionColumns()) { throw InternalException("TableScan::GetPartitionData: partition columns not supported"); } auto &bind = input.bind_data->Cast(); - auto &global = input.global_state->Cast(); - auto &local = input.local_state->Cast(); + void *const ffi_bind = bind.ffi_data->DataPtr(); + void *const ffi_global = input.global_state->Cast().ffi_data->DataPtr(); + void *const ffi_local = input.local_state->Cast().ffi_data->DataPtr(); duckdb_vx_error error_out = nullptr; - auto index = bind.info->vtab.get_partition_data(bind.ffi_data->DataPtr(), - global.ffi_data->DataPtr(), - local.ffi_data->DataPtr(), - &error_out); + const idx_t batch_index = bind.info->vtab.get_partition_data(ffi_bind, ffi_global, ffi_local, &error_out); if (error_out) { throw InvalidInputException(IntoErrString(error_out)); } - return OperatorPartitionData(index); + return OperatorPartitionData(batch_index); } extern "C" void duckdb_vx_string_map_insert(duckdb_vx_string_map map, const char *key, const char *value) { @@ -421,13 +381,15 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d auto db = wrapper->database->instance; auto tf = TableFunction(vtab->name, {}, c_function, c_bind, c_init_global, c_init_local); - tf.pushdown_complex_filter = c_pushdown_complex_filter; + tf.projection_pushdown = true; + tf.filter_pushdown = true; + // We can prune out filter columns that are unused in the remainder of the query plan. + // e.g. in "SELECT i FROM tbl WHERE j = 42" j does not leave Vortex table function. + tf.filter_prune = true; + tf.sampling_pushdown = false; + tf.late_materialization = false; - tf.projection_pushdown = vtab->projection_pushdown; - tf.filter_pushdown = vtab->filter_pushdown; - tf.filter_prune = vtab->filter_prune; - tf.sampling_pushdown = vtab->sampling_pushdown; - tf.late_materialization = vtab->late_materialization; + tf.pushdown_complex_filter = c_pushdown_complex_filter; tf.cardinality = c_cardinality; tf.get_partition_data = c_get_partition_data; tf.to_string = c_to_string; @@ -438,19 +400,12 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d return {{COLUMN_IDENTIFIER_EMPTY, TableColumn("", LogicalTypeId::BOOLEAN)}}; }; - // Set up the parameters tf.arguments.reserve(vtab->parameter_count); for (size_t i = 0; i < vtab->parameter_count; i++) { auto logical_type = reinterpret_cast(vtab->parameters[i]); tf.arguments.emplace_back(*logical_type); } - // And the named parameters - for (size_t i = 0; i < vtab->named_parameter_count; i++) { - auto logical_type = reinterpret_cast(vtab->named_parameter_types[i]); - tf.named_parameters.emplace(vtab->named_parameter_names[i], *logical_type); - } - // Assign the VTable to the function info so we can access it later to invoke the callbacks. tf.function_info = make_shared_ptr(*vtab); try { @@ -465,4 +420,3 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d } return DuckDBSuccess; } -} // namespace vortex diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index ecac7f2388e..c59fe6cbc2b 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -7,7 +7,6 @@ //! to get a blanket [`TableFunction`] implementation covering init, scan, progress, filter //! pushdown, cardinality, and partitioning. -use std::ffi::CString; use std::fmt::Debug; use std::sync::Arc; use std::sync::atomic::AtomicU64; @@ -84,7 +83,7 @@ use crate::exporter::ConversionCache; /// first column. As we don't want to fill the output chunk and we can leave /// it uninitialized in this case, we define COLUMN_IDENTIFIER_EMPTY as a /// virtual column. -/// See vortex-duckdb/cpp/table_function.cpp +/// See virtual_columns in vortex-duckdb/cpp/table_function.cpp static EMPTY_COLUMN_IDX: u64 = 18446744073709551614; /// A trait for table functions that resolve to a [`DataSourceRef`]. @@ -93,15 +92,8 @@ static EMPTY_COLUMN_IDX: u64 = 18446744073709551614; /// data source. All other [`TableFunction`] methods (init, scan, progress, filter pushdown, /// cardinality, partitioning) are provided by a blanket implementation. pub(crate) trait DataSourceTableFunction: Sized + Debug { - /// Returns the positional parameters of the table function. - fn parameters() -> Vec { - vec![] - } - - /// Returns the named parameters of the table function, if any. - fn named_parameters() -> Vec<(CString, LogicalType)> { - vec![] - } + /// Positional parameters + fn parameters() -> Vec; /// Bind the table function and return a [`DataSourceRef`]. fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult; @@ -260,18 +252,10 @@ impl TableFunction for T { type GlobalState = DataSourceGlobal; type LocalState = DataSourceLocal; - const PROJECTION_PUSHDOWN: bool = true; - const FILTER_PUSHDOWN: bool = true; - const FILTER_PRUNE: bool = true; - fn parameters() -> Vec { T::parameters() } - fn named_parameters() -> Vec<(CString, LogicalType)> { - T::named_parameters() - } - fn bind( ctx: &ClientContextRef, input: &BindInputRef, @@ -507,7 +491,7 @@ impl TableFunction for T { // Otherwise we'd have to open all files eagerly which is a performance // regression. Duckdb's Parquet reader only gets metadata for multiple // files with a UNION BY NAME and we don't support it (yet) - // https://github.com/duckdb/duckdb/blob/471de9f0e0e157ae672e56710e8c43b132a5ddc4/src/include/duckdb/common/multi_file/multi_file_function.hpp#L691 + // See duckdb/common/multi_file/multi_file_function.hpp#L691 if children.len() != 1 { return None; } @@ -581,7 +565,7 @@ fn extract_projection_expr( column_fields: &[DuckdbField], ) -> Expression { // Projection ids may be empty, in which case you need to use projection_ids - // https://github.com/duckdb/duckdb/blob/6e211da91657a94803c465fd0ce585f4c6754b54/src/planner/operator/logical_get.cpp#L168 + // See duckdb/src/planner/operator/logical_get.cpp#L168 let (projection_ids, has_projection_ids) = match projection_ids { Some(ids) => (ids, true), None => (column_ids, false), diff --git a/vortex-duckdb/src/duckdb/table_function/bind.rs b/vortex-duckdb/src/duckdb/table_function/bind.rs index adbbd028453..f56ca65f974 100644 --- a/vortex-duckdb/src/duckdb/table_function/bind.rs +++ b/vortex-duckdb/src/duckdb/table_function/bind.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::ffi::CStr; - use vortex::error::vortex_err; use crate::cpp; @@ -60,23 +58,6 @@ impl BindInputRef { Some(unsafe { Value::own(value_ptr) }) } } - - /// Returns the named parameter with the given name, if it exists. - pub fn get_named_parameter(&self, name: &CStr) -> Option { - let value_ptr = unsafe { - cpp::duckdb_vx_tfunc_bind_input_get_named_parameter(self.as_ptr(), name.as_ptr()) - }; - if value_ptr.is_null() { - None - } else { - Some(unsafe { Value::own(value_ptr) }) - } - } - - /// Returns the number of parameters bound to this function. - pub fn parameter_count(&self) -> usize { - unsafe { cpp::duckdb_vx_tfunc_bind_input_get_parameter_count(self.as_ptr()) } - } } lifetime_wrapper!(BindResult, cpp::duckdb_vx_tfunc_bind_result, |_| {}); diff --git a/vortex-duckdb/src/duckdb/table_function/mod.rs b/vortex-duckdb/src/duckdb/table_function/mod.rs index 5345d915fca..ac671f409e8 100644 --- a/vortex-duckdb/src/duckdb/table_function/mod.rs +++ b/vortex-duckdb/src/duckdb/table_function/mod.rs @@ -5,7 +5,6 @@ use std::ffi::CStr; use std::ffi::CString; use std::ffi::c_void; use std::fmt::Debug; -use std::ptr; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -65,36 +64,12 @@ pub trait TableFunction: Sized + Debug { type GlobalState: Send + Sync; type LocalState; - /// Whether the table function supports projection pushdown. - /// If not supported a projection will be added that filters out unused columns. - const PROJECTION_PUSHDOWN: bool = false; - - /// Whether the table function supports filter pushdown. - /// If not supported a filter will be added that applies the table filter directly. - const FILTER_PUSHDOWN: bool = false; - - /// Whether the table function can immediately prune out filter columns that are unused - /// in the remainder of the query plan. - /// e.g. "SELECT i FROM tbl WHERE j = 42;" - /// - j does not need to leave the table function at all. - const FILTER_PRUNE: bool = false; - - /// Maximum number of threads the table function can use. - /// If not specified, DuckDB will use its default (GlobalTableFunctionState::MAX_THREADS). - const MAX_THREADS: u64 = u64::MAX; - /// Returns the parameters of the table function. fn parameters() -> Vec { // By default, we don't have any parameters. vec![] } - /// Returns the named parameters of the table function, if any. - fn named_parameters() -> Vec<(CString, LogicalType)> { - // By default, we don't have any named parameters. - vec![] - } - /// This function is used for determining the schema of a table producing function and /// returning bind data. fn bind( @@ -103,6 +78,8 @@ pub trait TableFunction: Sized + Debug { result: &mut BindResultRef, ) -> VortexResult; + /// Report column statistics for a file or collections of files e.g. + /// registered as a VIEW. fn statistics( client_context: &ClientContextRef, bind_data: &Self::BindData, @@ -167,8 +144,6 @@ pub trait TableFunction: Sized + Debug { /// Returns a vector of key-value pairs for EXPLAIN output fn to_string(bind_data: &Self::BindData, map: &mut DuckdbStringMapRef); - - // TODO(ngates): there are many more callbacks that can be configured. } #[derive(Debug)] @@ -190,19 +165,10 @@ impl DatabaseRef { .map(|logical_type| logical_type.as_ptr()) .collect::>(); - let param_names = T::named_parameters(); - let (param_names_ptrs, param_types_ptr) = param_names - .into_iter() - .map(|(name, logical_type)| (name.as_ptr(), logical_type.as_ptr())) - .unzip::<_, _, Vec<_>, Vec<_>>(); - let vtab = cpp::duckdb_vx_tfunc_vtab_t { name: name.as_ptr(), parameters: parameter_ptrs.as_ptr(), parameter_count: parameters.len() as _, - named_parameter_names: param_names_ptrs.as_ptr(), - named_parameter_types: param_types_ptr.as_ptr(), - named_parameter_count: param_names_ptrs.len() as _, bind: Some(bind_callback::), bind_data_clone: Some(bind_data_clone_callback::), init_global: Some(init_global_callback::), @@ -211,16 +177,9 @@ impl DatabaseRef { statistics: Some(statistics::), cardinality: Some(cardinality_callback::), pushdown_complex_filter: Some(pushdown_complex_filter_callback::), - pushdown_expression: ptr::null_mut::(), to_string: Some(to_string_callback::), table_scan_progress: Some(table_scan_progress_callback::), get_partition_data: Some(get_partition_data_callback::), - projection_pushdown: T::PROJECTION_PUSHDOWN, - filter_pushdown: T::FILTER_PUSHDOWN, - filter_prune: T::FILTER_PRUNE, - sampling_pushdown: false, - late_materialization: false, - max_threads: T::MAX_THREADS, }; duckdb_try!( From 5b0c5f959b0a8e00d5dbdb59c44cbca18579c61f Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 23 Apr 2026 13:49:27 +0100 Subject: [PATCH 169/250] Modify renovate config for better experience (#7605) ## Summary This PR includes 3 somewhat related changes, which should help maintain a better experience and keep the repo somewhat more secure: 1. Pin github actions to hashes instead of versions automatically, this is desirable because github-actions have mutable releases, and no guarantee the code and the release matches, a fact which has been repeatedly abused in recent years. 2. Split the lock file updates by ecosystem, so they don't get blocked on unrelated issues. 3. Instead of automerging patch/minor releases, only do that once a dependency is "stable" in semver terms (post 1.0) Signed-off-by: Adam Gutglick --- renovate.json | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/renovate.json b/renovate.json index fbc453fae56..80fb4c20238 100644 --- a/renovate.json +++ b/renovate.json @@ -2,8 +2,7 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:recommended", - ":automergePatch", - ":automergeMinor", + ":automergeStableNonMajor", ":automergePr", ":automergeRequireAllStatusChecks", ":combinePatchMinorReleases", @@ -11,6 +10,7 @@ ":separateMultipleMajorReleases", ":configMigration", "group:rust-futuresMonorepo", + "helpers:pinGitHubActionDigests", "schedule:earlyMondays", "docker:disable" ], @@ -60,6 +60,36 @@ "matchSourceUrls": [ "https://github.com/hyperium/tonic" ] + }, + { + "groupName": "Rust lock file maintenance", + "groupSlug": "rust-lock-file-maintenance", + "matchManagers": [ + "cargo" + ], + "matchUpdateTypes": [ + "lockFileMaintenance" + ] + }, + { + "groupName": "Python lock file maintenance", + "groupSlug": "python-lock-file-maintenance", + "matchManagers": [ + "pep621" + ], + "matchUpdateTypes": [ + "lockFileMaintenance" + ] + }, + { + "groupName": "JS lock file maintenance", + "groupSlug": "js-lock-file-maintenance", + "matchManagers": [ + "npm" + ], + "matchUpdateTypes": [ + "lockFileMaintenance" + ] } ], "customManagers": [ @@ -77,4 +107,4 @@ "extractVersionTemplate": "^v?(?.*)$" } ] -} \ No newline at end of file +} From e06e371cfb372b9ae448c27640e7203e6421bf93 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 23 Apr 2026 08:53:36 -0400 Subject: [PATCH 170/250] Faster BoolArray::min_max via true_count instead of set_slices (#7599) There's no need to compute valid slices, we can just use true_count fix #7598 Signed-off-by: Robert Kruszewski --- .../src/aggregate_fn/fns/min_max/bool.rs | 59 ++++++------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs index 3d814c693cf..69471eec556 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs @@ -4,7 +4,7 @@ use std::ops::BitAnd; use vortex_error::VortexResult; -use vortex_mask::Mask; +use vortex_mask::AllOr; use super::MinMaxPartial; use super::MinMaxResult; @@ -27,53 +27,30 @@ pub(super) fn accumulate_bool( .as_ref() .validity()? .execute_mask(array.as_ref().len(), ctx)?; - let true_non_null = match &mask { - Mask::AllTrue(_) => array.to_bit_buffer(), - Mask::AllFalse(_) => return Ok(()), - Mask::Values(v) => array.to_bit_buffer().bitand(v.bit_buffer()), + let (true_count, valid_count) = match mask.bit_buffer() { + AllOr::None => return Ok(()), + AllOr::All => (array.to_bit_buffer().true_count(), array.as_ref().len()), + AllOr::Some(validity) => ( + array.to_bit_buffer().bitand(validity).true_count(), + validity.true_count(), + ), }; - let mut true_slices = true_non_null.set_slices(); - - let Some(slice) = true_slices.next() else { - // all false - partial.merge(Some(MinMaxResult { - min: Scalar::bool(false, NonNullable), - max: Scalar::bool(false, NonNullable), - })); - return Ok(()); - }; - - if slice.0 == 0 && slice.1 == array.len() { - // all true - partial.merge(Some(MinMaxResult { - min: Scalar::bool(true, NonNullable), - max: Scalar::bool(true, NonNullable), - })); + if valid_count == 0 { return Ok(()); } - // Check for valid false values when we have a partial validity mask - match &mask { - Mask::AllTrue(_) | Mask::AllFalse(_) => {} - Mask::Values(v) => { - let false_non_null = (!array.to_bit_buffer()).bitand(v.bit_buffer()); - let mut false_slices = false_non_null.set_slices(); - - if false_slices.next().is_none() { - // No false values, so all valid values are true - partial.merge(Some(MinMaxResult { - min: Scalar::bool(true, NonNullable), - max: Scalar::bool(true, NonNullable), - })); - return Ok(()); - } - } - } + let (min, max) = if true_count == 0 { + (false, false) + } else if true_count == valid_count { + (true, true) + } else { + (false, true) + }; partial.merge(Some(MinMaxResult { - min: Scalar::bool(false, NonNullable), - max: Scalar::bool(true, NonNullable), + min: Scalar::bool(min, NonNullable), + max: Scalar::bool(max, NonNullable), })); Ok(()) } From 75808d5ef0c07ff278c85450001bfc90097a9d56 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Thu, 23 Apr 2026 14:23:51 +0100 Subject: [PATCH 171/250] fix deadlock on unknown policy request (#7606) Signed-off-by: Mikhail Kot --- vortex-session/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vortex-session/src/lib.rs b/vortex-session/src/lib.rs index 205acc9bef9..146d8390ac9 100644 --- a/vortex-session/src/lib.rs +++ b/vortex-session/src/lib.rs @@ -72,7 +72,9 @@ impl VortexSession { /// Returns whether unknown plugins should deserialize as foreign placeholders. pub fn allows_unknown(&self) -> bool { - ::get::(self).allow_unknown + ::get_opt::(self) + .map(|p| p.allow_unknown) + .unwrap_or(false) } } From 143fb615edf72698eb3d8dd0e53e1350c912d598 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Thu, 23 Apr 2026 15:21:29 +0100 Subject: [PATCH 172/250] add another oneshot tsan suppression (#7608) Another false positive causing errors in https://github.com/vortex-data/vortex/actions/runs/24559595293/job/71804664847?pr=7509#step:10:215 Signed-off-by: Mikhail Kot --- .github/workflows/rust-instrumented.yml | 2 +- vortex-ffi/test/scan.cpp | 6 +++++- vortex-ffi/tsan_suppressions.txt | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index 9af1471f44f..3bcdad8af53 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -155,7 +155,7 @@ jobs: ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" MSAN_OPTIONS: "symbolize=1" MSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" - TSAN_OPTIONS: "symbolize=1" + TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt" TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" VORTEX_SKIP_SLOW_TESTS: "1" # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp index 93a476ce614..976cdc7f355 100644 --- a/vortex-ffi/test/scan.cpp +++ b/vortex-ffi/test/scan.cpp @@ -1,6 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include #include #include #include @@ -520,10 +519,15 @@ TEST_CASE("Multithreaded scan", "[datasource]") { arrays[i] = array; }); } + for (auto &thread : threads) { thread.join(); } + vx_partition *const partition = vx_scan_next_partition(scan, &error); + require_no_error(error); + REQUIRE(partition == nullptr); + for (const vx_array *array : arrays) { REQUIRE(array != nullptr); defer { diff --git a/vortex-ffi/tsan_suppressions.txt b/vortex-ffi/tsan_suppressions.txt index 6f9ffaae597..ee8ac6e926d 100644 --- a/vortex-ffi/tsan_suppressions.txt +++ b/vortex-ffi/tsan_suppressions.txt @@ -5,3 +5,4 @@ # This is likely a false positive in TSan for oneshot::channel and kanal # where ordering is correct but uses an explicit fence and not relaxed load race:oneshot-*/src/channel.rs +race:oneshot-*/src/lib.rs From dfb9992baf52011feb87ac3156cf52612c7679f0 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:24:29 -0400 Subject: [PATCH 173/250] Add spans in compressor for perfetto (#7607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7216 Adds some more interesting spans to the compressor. This lets us use perfetto to visualize where time is being spent more easily. Also removes some noisy logs that don't tell us that much. Also removes some tests that are probably useless. Screenshot 2026-04-23 at 9 44
52 AM Definitely gives us some interesting insights on things! ## Testing N/A Signed-off-by: Connor Tsui --- vortex-compressor/src/compressor.rs | 326 ++------------------------- vortex-compressor/src/estimate.rs | 6 +- vortex-compressor/src/lib.rs | 18 +- vortex-compressor/src/stats/cache.rs | 6 +- vortex-compressor/src/trace.rs | 206 +++++++++-------- 5 files changed, 149 insertions(+), 413 deletions(-) diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index aa18b61637b..b99d31c6d73 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -314,16 +314,15 @@ impl CascadingCompressor { // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the // scheme name and cascade history before propagating. let error_ctx = trace::enabled_error_context(&compress_ctx); - let compressed = match winner.compress(self, &data, compress_ctx, exec_ctx) { - Ok(compressed) => compressed, - Err(err) => { + let _winner_span = trace::winner_compress_span(winner.id(), before_nbytes).entered(); + let compressed = winner + .compress(self, &data, compress_ctx, exec_ctx) + .inspect_err(|err| { // NB: this is the only way we can tell which scheme panicked / bailed on their // data, especially for third-party schemes where the error site may not carry any // compressor context. - trace::scheme_compress_failed(winner.id(), before_nbytes, error_ctx.as_ref(), &err); - return Err(err); - } - }; + trace::scheme_compress_failed(winner.id(), before_nbytes, error_ctx.as_ref(), err); + })?; let after_nbytes = compressed.nbytes(); let actual_ratio = (after_nbytes != 0).then(|| before_nbytes as f64 / after_nbytes as f64); @@ -331,9 +330,7 @@ impl CascadingCompressor { // TODO(connor): HACK TO SUPPORT L2 DENORMALIZATION!!! let accepted = after_nbytes < before_nbytes || compressed.is::(); - trace::scheme_compress_result( - winner.id(), - before_nbytes, + trace::record_winner_compress_result( after_nbytes, winner_estimate.trace_ratio(), actual_ratio, @@ -373,21 +370,24 @@ impl CascadingCompressor { let mut deferred: Vec<(&'static dyn Scheme, DeferredEstimate)> = Vec::new(); // Pass 1: evaluate every immediate verdict. Stash deferred work for pass 2. - for &scheme in schemes { - match scheme.expected_compression_ratio(data, compress_ctx.clone(), exec_ctx) { - CompressionEstimate::Verdict(EstimateVerdict::Skip) => {} - CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) => { - return Ok(Some((scheme, WinnerEstimate::AlwaysUse))); - } - CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio)) => { - let score = EstimateScore::FiniteCompression(ratio); + { + let _verdict_pass = trace::verdict_pass_span().entered(); + for &scheme in schemes { + match scheme.expected_compression_ratio(data, compress_ctx.clone(), exec_ctx) { + CompressionEstimate::Verdict(EstimateVerdict::Skip) => {} + CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) => { + return Ok(Some((scheme, WinnerEstimate::AlwaysUse))); + } + CompressionEstimate::Verdict(EstimateVerdict::Ratio(ratio)) => { + let score = EstimateScore::FiniteCompression(ratio); - if is_better_score(score, &best) { - best = Some((scheme, score)); + if is_better_score(score, &best) { + best = Some((scheme, score)); + } + } + CompressionEstimate::Deferred(deferred_estimate) => { + deferred.push((scheme, deferred_estimate)); } - } - CompressionEstimate::Deferred(deferred_estimate) => { - deferred.push((scheme, deferred_estimate)); } } } @@ -395,6 +395,7 @@ impl CascadingCompressor { // Pass 2: run deferred work. Callbacks receive the current best as a threshold so they can // short-circuit with `Skip` when they cannot beat it. for (scheme, deferred_estimate) in deferred { + let _span = trace::scheme_eval_span(scheme.id()).entered(); let threshold: Option = best.map(|(_, score)| score); match deferred_estimate { DeferredEstimate::Sample => { @@ -560,18 +561,9 @@ impl CascadingCompressor { #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use std::sync::Arc; use std::sync::LazyLock; use parking_lot::Mutex; - use tracing::Event; - use tracing::Subscriber; - use tracing::field::Field; - use tracing::field::Visit; - use tracing_subscriber::Layer; - use tracing_subscriber::layer::Context; - use tracing_subscriber::prelude::*; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::VortexSessionExecute; @@ -595,7 +587,6 @@ mod tests { use crate::estimate::EstimateVerdict; use crate::estimate::WinnerEstimate; use crate::scheme::SchemeExt; - use crate::trace::TARGET_TRACE; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -613,98 +604,6 @@ mod tests { matches!(canonical, Canonical::Primitive(primitive) if primitive.ptype().is_int()) } - fn test_integer_array() -> ArrayRef { - PrimitiveArray::new(buffer![1i32, 2, 3, 4], Validity::NonNullable).into_array() - } - - #[derive(Debug, Clone, PartialEq, Eq)] - struct RecordedEvent { - target: String, - fields: BTreeMap, - } - - #[derive(Default)] - struct EventVisitor { - fields: BTreeMap, - } - - impl Visit for EventVisitor { - fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.fields - .insert(field.name().to_owned(), format!("{value:?}")); - } - - fn record_i64(&mut self, field: &Field, value: i64) { - self.fields - .insert(field.name().to_owned(), value.to_string()); - } - - fn record_u64(&mut self, field: &Field, value: u64) { - self.fields - .insert(field.name().to_owned(), value.to_string()); - } - - fn record_bool(&mut self, field: &Field, value: bool) { - self.fields - .insert(field.name().to_owned(), value.to_string()); - } - - fn record_str(&mut self, field: &Field, value: &str) { - self.fields - .insert(field.name().to_owned(), value.to_owned()); - } - } - - struct RecordingLayer { - events: Arc>>, - } - - impl RecordingLayer { - fn new(events: Arc>>) -> Self { - Self { events } - } - } - - impl Layer for RecordingLayer - where - S: Subscriber, - { - fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - let mut visitor = EventVisitor::default(); - event.record(&mut visitor); - self.events.lock().push(RecordedEvent { - target: event.metadata().target().to_owned(), - fields: visitor.fields, - }); - } - } - - fn record_events(f: impl FnOnce() -> T) -> (T, Vec) { - let events = Arc::new(Mutex::new(Vec::new())); - let subscriber = - tracing_subscriber::registry().with(RecordingLayer::new(Arc::clone(&events))); - let result = tracing::subscriber::with_default(subscriber, f); - let recorded = events.lock().clone(); - (result, recorded) - } - - fn find_event<'a>( - events: &'a [RecordedEvent], - target: &str, - message: &str, - ) -> &'a RecordedEvent { - events - .iter() - .find(|event| { - event.target == target - && event - .fields - .get("message") - .is_some_and(|value| value == message) - }) - .expect("expected event not found") - } - #[derive(Debug)] struct DirectRatioScheme; @@ -935,102 +834,6 @@ mod tests { } } - #[derive(Debug)] - struct NestedFailureParentScheme; - - impl Scheme for NestedFailureParentScheme { - fn scheme_name(&self) -> &'static str { - "test.nested_failure_parent" - } - - fn matches(&self, canonical: &Canonical) -> bool { - matches_integer_primitive(canonical) - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) - } - - fn compress( - &self, - compressor: &CascadingCompressor, - data: &ArrayAndStats, - compress_ctx: CompressorContext, - exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - compressor.compress_child(data.array(), &compress_ctx, self.id(), 1, exec_ctx) - } - } - - #[derive(Debug)] - struct NestedFailureLeafScheme; - - impl Scheme for NestedFailureLeafScheme { - fn scheme_name(&self) -> &'static str { - "test.nested_failure_leaf" - } - - fn matches(&self, canonical: &Canonical) -> bool { - matches_integer_primitive(canonical) - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - CompressionEstimate::Verdict(EstimateVerdict::AlwaysUse) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - vortex_error::vortex_bail!("nested failure") - } - } - - #[derive(Debug)] - struct SamplingFailureScheme; - - impl Scheme for SamplingFailureScheme { - fn scheme_name(&self) -> &'static str { - "test.sampling_failure" - } - - fn matches(&self, canonical: &Canonical) -> bool { - matches_integer_primitive(canonical) - } - - fn expected_compression_ratio( - &self, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> CompressionEstimate { - CompressionEstimate::Deferred(DeferredEstimate::Sample) - } - - fn compress( - &self, - _compressor: &CascadingCompressor, - _data: &ArrayAndStats, - _compress_ctx: CompressorContext, - _exec_ctx: &mut ExecutionCtx, - ) -> VortexResult { - vortex_error::vortex_bail!("sample failure") - } - } - #[test] fn test_self_exclusion() { let c = compressor(); @@ -1494,85 +1297,4 @@ mod tests { assert!(matches!(score, EstimateScore::FiniteCompression(ratio) if ratio.is_finite())); Ok(()) } - - #[test] - fn compress_failure_event_includes_cascade_path_and_depth() { - let compressor = - CascadingCompressor::new(vec![&NestedFailureParentScheme, &NestedFailureLeafScheme]); - let array = test_integer_array(); - - let (result, events) = record_events(|| { - let mut exec_ctx = SESSION.create_execution_ctx(); - compressor.compress(&array, &mut exec_ctx) - }); - - assert!(result.is_err()); - let event = find_event(&events, TARGET_TRACE, "scheme.compress_failed"); - assert_eq!( - event.fields.get("scheme").map(String::as_str), - Some("test.nested_failure_leaf") - ); - assert_eq!( - event.fields.get("cascade_path").map(String::as_str), - Some("test.nested_failure_parent[1]") - ); - assert_eq!( - event.fields.get("cascade_depth").map(String::as_str), - Some("1") - ); - } - - #[test] - fn sample_failure_event_includes_cascade_path_and_depth() { - let compressor = CascadingCompressor::new(vec![&SamplingFailureScheme]); - let array = test_integer_array(); - - let (result, events) = record_events(|| { - let mut exec_ctx = SESSION.create_execution_ctx(); - compressor.compress(&array, &mut exec_ctx) - }); - - assert!(result.is_err()); - let event = find_event(&events, TARGET_TRACE, "sample.compress_failed"); - assert_eq!( - event.fields.get("scheme").map(String::as_str), - Some("test.sampling_failure") - ); - assert_eq!( - event.fields.get("cascade_path").map(String::as_str), - Some("root") - ); - assert_eq!( - event.fields.get("cascade_depth").map(String::as_str), - Some("0") - ); - } - - #[test] - fn zero_byte_sample_result_omits_ratio_fields_and_selects_no_scheme() { - let compressor = CascadingCompressor::new(vec![&ZeroBytesSamplingScheme]); - let array = test_integer_array(); - - let (result, events) = record_events(|| { - let mut exec_ctx = SESSION.create_execution_ctx(); - compressor.compress(&array, &mut exec_ctx) - }); - - assert!(result.is_ok()); - - let sample_event = find_event(&events, TARGET_TRACE, "sample.result"); - assert_eq!( - sample_event.fields.get("sampled_after").map(String::as_str), - Some("0") - ); - assert!(!sample_event.fields.contains_key("sampled_ratio")); - - assert!(!events.iter().any(|event| { - event.target == TARGET_TRACE - && event - .fields - .get("message") - .is_some_and(|value| value == "scheme.compress_result") - })); - } } diff --git a/vortex-compressor/src/estimate.rs b/vortex-compressor/src/estimate.rs index 9fbf434352a..065937ffac9 100644 --- a/vortex-compressor/src/estimate.rs +++ b/vortex-compressor/src/estimate.rs @@ -228,9 +228,9 @@ pub(super) fn estimate_compression_ratio_with_sampling( let score = EstimateScore::from_sample_sizes(before, after); - // Single DEBUG event per sampled scheme. Downstream tooling can join this with the eventual - // `scheme.compress_result` on the same scheme to compute sample-vs-full divergence. - trace::sample_result(scheme.id(), before, after, score.finite_ratio()); + if matches!(score, EstimateScore::ZeroBytes) { + trace::zero_byte_sample_result(scheme.id(), before); + } Ok(score) } diff --git a/vortex-compressor/src/lib.rs b/vortex-compressor/src/lib.rs index 9d6c7d7321f..1ecc9e4d5b4 100644 --- a/vortex-compressor/src/lib.rs +++ b/vortex-compressor/src/lib.rs @@ -18,17 +18,17 @@ //! //! # Observability //! -//! The compressor emits a small set of `tracing` events on a single target so you can see what -//! it's doing without attaching a profiler. +//! The compressor emits a small set of `tracing` spans and events on a single target so you can +//! see what it's doing without attaching a profiler. //! -//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see one line per leaf compression -//! decision. The `vortex_compressor::encode` target carries the main decision events -//! (`scheme.compress_result`, `sample.result`, and both `*.compress_failed`) plus the coarse -//! top-level `compress` span and `cascade_exhausted` event. +//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see compression decision spans +//! and exceptional events. The `vortex_compressor::encode` target carries the top-level `compress` +//! span, per-scheme evaluation and winning-compression spans, the `cascade_exhausted` event, +//! `sample.result` events for zero-byte sample outputs, and both `*.compress_failed` events. //! -//! The primary event is `scheme.compress_result`, which carries `scheme`, `before_nbytes`, -//! `after_nbytes`, `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 -//! bytes), `actual_ratio` (absent when the compressed output is 0 bytes), and `accepted`. +//! The winning-compression span carries `scheme_chosen`, `input_nbytes`, `compressed_nbytes`, +//! `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 bytes), +//! `achieved_ratio` (absent when the compressed output is 0 bytes), and `accepted`. //! //! Failure events additionally carry `cascade_path` and `cascade_depth`, so nested compression //! errors can be tied back to the ancestor branch that triggered them. diff --git a/vortex-compressor/src/stats/cache.rs b/vortex-compressor/src/stats/cache.rs index 91f0bf711fb..6f7020191a1 100644 --- a/vortex-compressor/src/stats/cache.rs +++ b/vortex-compressor/src/stats/cache.rs @@ -21,6 +21,7 @@ use super::FloatStats; use super::GenerateStatsOptions; use super::IntegerStats; use super::StringStats; +use crate::trace; /// A single cache entry: a concrete [`TypeId`] paired with a type-erased value. type StatsEntry = (TypeId, Arc); @@ -58,7 +59,10 @@ impl StatsCache { .ok() .vortex_expect("we just checked the TypeID") } else { - let new_arc: Arc = Arc::new(f()); + let new_arc: Arc = { + let _span = trace::generate_stats_span(std::any::type_name::()).entered(); + Arc::new(f()) + }; guard.push((type_id, Arc::clone(&new_arc) as Arc)); new_arc } diff --git a/vortex-compressor/src/trace.rs b/vortex-compressor/src/trace.rs index d499f18359f..84027272b31 100644 --- a/vortex-compressor/src/trace.rs +++ b/vortex-compressor/src/trace.rs @@ -12,6 +12,9 @@ use crate::scheme::SchemeId; pub(super) const TARGET_TRACE: &str = "vortex_compressor::encode"; /// Builds the top-level compression span. +/// +/// `input_nbytes` is known up front; `compressed_nbytes` / `compression_ratio` are filled in by +/// [`record_compress_outcome`] once the cascade returns. #[inline] pub(super) fn compress_span( len: usize, @@ -21,20 +24,117 @@ pub(super) fn compress_span( tracing::debug_span!( target: TARGET_TRACE, "compress", - len, + array_len = len, dtype = %dtype, - before_nbytes, - after_nbytes = tracing::field::Empty, - ratio = tracing::field::Empty, + input_nbytes = before_nbytes, + compressed_nbytes = tracing::field::Empty, + compression_ratio = tracing::field::Empty, ) } +/// Builds a span covering on-demand materialization of a cached stats type. +/// +/// Child of whatever span is active when a stats accessor first fires. Typically that's +/// [`verdict_pass_span`]; entering this span disambiguates stats cost from the rest of Pass 1. +/// `kind` is usually `std::any::type_name::()` so the args identify which group was generated +/// (e.g. `IntegerStats`, `FloatStats`). +#[inline] +pub(super) fn generate_stats_span(kind: &'static str) -> tracing::Span { + tracing::debug_span!( + target: TARGET_TRACE, + "generate_stats", + stats_kind = kind, + ) +} + +/// Builds a span covering Pass 1 of scheme selection (the cheap-verdict pass). +/// +/// Stats batches merged across eligible schemes are materialized lazily by the first +/// `expected_compression_ratio` call that touches them. Grouping those calls under one span makes +/// the stats cost (and unexpectedly slow verdicts) visible independently of per-candidate sampling. +#[inline] +pub(super) fn verdict_pass_span() -> tracing::Span { + tracing::debug_span!( + target: TARGET_TRACE, + "verdict_pass", + ) +} + +/// Builds a span covering one deferred per-scheme evaluation (sample or callback). +/// +/// `scheme_candidate` is the scheme being evaluated, not necessarily chosen. +#[inline] +pub(super) fn scheme_eval_span(scheme: SchemeId) -> tracing::Span { + tracing::debug_span!( + target: TARGET_TRACE, + "scheme_eval", + scheme_candidate = %scheme, + ) +} + +/// Emits the sampling result event for zero-byte sample outputs. +#[inline] +pub(super) fn zero_byte_sample_result(scheme: SchemeId, sampled_before: u64) { + tracing::debug!( + target: TARGET_TRACE, + scheme = %scheme, + sampled_before, + sampled_after = 0_u64, + "sample.result", + ); +} + +/// Builds a span covering the winning scheme's full-array compression. +/// +/// `scheme_chosen` and `input_nbytes` are known up front. `compressed_nbytes`, +/// `estimated_ratio`, `achieved_ratio`, and `accepted` are filled in by +/// [`record_winner_compress_result`] once the encode completes. +#[inline] +pub(super) fn winner_compress_span(scheme: SchemeId, before_nbytes: u64) -> tracing::Span { + tracing::debug_span!( + target: TARGET_TRACE, + "winner_compress", + scheme_chosen = %scheme, + input_nbytes = before_nbytes, + compressed_nbytes = tracing::field::Empty, + estimated_ratio = tracing::field::Empty, + achieved_ratio = tracing::field::Empty, + accepted = tracing::field::Empty, + ) +} + +/// Records the outcome of a winning-scheme compression on the current `winner_compress` span. +#[inline] +pub(super) fn record_winner_compress_result( + compressed_nbytes: u64, + estimated_ratio: Option, + achieved_ratio: Option, + accepted: bool, +) { + let span = tracing::Span::current(); + span.record("compressed_nbytes", compressed_nbytes); + if let Some(r) = estimated_ratio { + span.record("estimated_ratio", r); + } + if let Some(r) = achieved_ratio { + span.record("achieved_ratio", r); + } + span.record("accepted", accepted); +} + /// Records the final output size and, when finite, the top-level compression ratio. #[inline] -pub(super) fn record_compress_outcome(span: &tracing::Span, before_nbytes: u64, after_nbytes: u64) { - span.record("after_nbytes", after_nbytes); - if after_nbytes != 0 { - span.record("ratio", before_nbytes as f64 / after_nbytes as f64); +pub(super) fn record_compress_outcome( + span: &tracing::Span, + input_nbytes: u64, + compressed_nbytes: u64, +) { + span.record("compressed_nbytes", compressed_nbytes); + if compressed_nbytes != 0 { + span.record( + "compression_ratio", + input_nbytes as f64 / compressed_nbytes as f64, + ); } } @@ -76,68 +176,6 @@ pub(super) fn scheme_compress_failed( } } -/// Emits the leaf compression result event. -#[inline] -#[allow( - clippy::cognitive_complexity, - reason = "tracing sometimes triggers this" -)] -pub(super) fn scheme_compress_result( - scheme: SchemeId, - before_nbytes: u64, - after_nbytes: u64, - estimated_ratio: Option, - actual_ratio: Option, - accepted: bool, -) { - match (estimated_ratio, actual_ratio) { - (Some(estimated_ratio), Some(actual_ratio)) => { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - before_nbytes, - after_nbytes, - estimated_ratio, - actual_ratio, - accepted, - "scheme.compress_result", - ); - } - (Some(estimated_ratio), None) => { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - before_nbytes, - after_nbytes, - estimated_ratio, - accepted, - "scheme.compress_result", - ); - } - (None, Some(actual_ratio)) => { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - before_nbytes, - after_nbytes, - actual_ratio, - accepted, - "scheme.compress_result", - ); - } - (None, None) => { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - before_nbytes, - after_nbytes, - accepted, - "scheme.compress_result", - ); - } - } -} - /// Emits a sampling-failure event. #[inline] pub(super) fn sample_compress_failed( @@ -156,31 +194,3 @@ pub(super) fn sample_compress_failed( ); } } - -/// Emits the sampling result event. -#[inline] -pub(super) fn sample_result( - scheme: SchemeId, - sampled_before: u64, - sampled_after: u64, - sampled_ratio: Option, -) { - if let Some(sampled_ratio) = sampled_ratio { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - sampled_before, - sampled_after, - sampled_ratio, - "sample.result", - ); - } else { - tracing::debug!( - target: TARGET_TRACE, - scheme = %scheme, - sampled_before, - sampled_after, - "sample.result", - ); - } -} From 09ae129b7dc6a6f26df5b591c4f88b24512a3c49 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Thu, 23 Apr 2026 15:24:59 +0100 Subject: [PATCH 174/250] feat(cuda): fuse narrower-than-output Dict codes and RunEnd ends (#7603) Dict codes and RunEnd ends that are narrower than the output type (e.g. u8 BitPacked codes in a u32 Dict) previously required a separate kernel launch. They are now fused by decoding at the source's native width and widening to T in shared memory. --------- Signed-off-by: Alexander Droste --- vortex-cuda/kernels/src/dynamic_dispatch.cu | 116 ++++++++-- vortex-cuda/kernels/src/dynamic_dispatch.h | 21 ++ vortex-cuda/src/dynamic_dispatch/mod.rs | 203 +++++++++++++++++- .../src/dynamic_dispatch/plan_builder.rs | 180 ++++++---------- 4 files changed, 387 insertions(+), 133 deletions(-) diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.cu b/vortex-cuda/kernels/src/dynamic_dispatch.cu index 9a413af17c1..a0a1039900e 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.cu +++ b/vortex-cuda/kernels/src/dynamic_dispatch.cu @@ -39,9 +39,16 @@ // // ## Mixed-width support // -// LOAD sources from pending subtrees may have a narrower type than the -// output (e.g. u8 dict codes in a u32 plan). load_element() widens -// to T via static_cast — no separate widen kernel or smem intermediate. +// Dict codes, RunEnd ends, and other child arrays may have a narrower +// element type than the output T. Two mechanisms handle this: +// +// LOAD load_element() dispatches on the per-stage PTypeTag to +// read at the source's native width and static_cast to T. +// BITUNPACK bitunpack_typed() unpacks at the source's native width, +// then widens to T in-place via a backward scan +// (widen_inplace). The smem region is pre-allocated at +// max(source_width, T) bytes per element by the Rust plan +// builder, so the widen never overflows. #include #include @@ -203,6 +210,22 @@ scatter_patches_chunk(const GPUPatches &patches, T *__restrict out, uint32_t chu // Source ops // ═══════════════════════════════════════════════════════════════════════════ +/// Widen U-sized elements in shared memory to T-sized, in-place. +/// Backward scan ensures no unread element is overwritten since +/// sizeof(T) >= sizeof(U) guarantees the write at index i touches +/// only bytes beyond those of src[i]. +template +__device__ inline void widen_inplace(T *dst, uint32_t len) { + if constexpr (sizeof(T) <= sizeof(U)) { + return; + } + const U *src = reinterpret_cast(dst); + for (int32_t i = static_cast(len) - 1 - threadIdx.x; i >= 0; i -= blockDim.x) { + dst[i] = static_cast(src[i]); + } + __syncthreads(); +} + /// FastLanes cooperative unpack — all threads in the block scatter-write /// decoded elements into `dst`. Caller must issue __syncthreads() before /// any thread reads from `dst`. @@ -236,6 +259,68 @@ __device__ inline void bitunpack(const T *__restrict packed, } } +/// Dispatch bitunpack at the source's native element width, then widen +/// to T in-place so all downstream scalar ops and smem consumers see +/// T-sized elements. Falls back to the direct `bitunpack` path when +/// the source ptype already matches T. Issues __syncthreads() before +/// returning on all paths. +/// +/// Accepts explicit chunk_start / chunk_len so it works for both input +/// stages (full decode with chunk_start=0, chunk_len=stage.len) and +/// the output stage (tiled with varying chunk_start / chunk_len). +template +__device__ inline void bitunpack_typed(T *__restrict dst, + const void *__restrict packed, + uint64_t chunk_start, + uint32_t chunk_len, + const struct SourceOp &src, + PTypeTag source_ptype) { + // Fast path: source width matches T — no widening needed. + if (ptype_byte_width(source_ptype) == sizeof(T)) { + bitunpack(reinterpret_cast(packed), dst, chunk_start, chunk_len, src); + __syncthreads(); + return; + } + + // Compute total elements written by bitunpack (including alignment + // padding) so widen_inplace covers the full scratch region. + const uint32_t elem_off = src.params.bitunpack.element_offset; + const uint32_t dst_off = (chunk_start + elem_off) % FL_CHUNK; + const uint32_t n_chunks = (chunk_len + dst_off + FL_CHUNK - 1) / FL_CHUNK; + const uint32_t total_elems = n_chunks * FL_CHUNK; + + // Narrow source: unpack at native width, then widen to T. + switch (source_ptype) { + case PTYPE_U8: + case PTYPE_I8: { + auto *narrow = reinterpret_cast(dst); + bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); + __syncthreads(); + widen_inplace(dst, total_elems); + break; + } + case PTYPE_U16: + case PTYPE_I16: { + auto *narrow = reinterpret_cast(dst); + bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); + __syncthreads(); + widen_inplace(dst, total_elems); + break; + } + case PTYPE_U32: + case PTYPE_I32: + case PTYPE_F32: { + auto *narrow = reinterpret_cast(dst); + bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); + __syncthreads(); + widen_inplace(dst, total_elems); + break; + } + default: + __builtin_unreachable(); + } +} + /// Read N values from a source op into `out`. /// /// Dispatches on `src.op_code` to handle each encoding: @@ -354,16 +439,14 @@ __device__ void execute_output_stage(T *__restrict output, if (src.op_code == SourceOp::BITUNPACK) { chunk_len = bitunpack_tile_len(stage, block_len, elem_idx); T *scratch = reinterpret_cast(smem + stage.smem_byte_offset); - bitunpack(reinterpret_cast(stage.input_ptr), - scratch, - block_start + elem_idx, - chunk_len, - src); + bitunpack_typed(scratch, + reinterpret_cast(stage.input_ptr), + block_start + elem_idx, + chunk_len, + src, + ptype); const uint32_t align = (block_start + elem_idx + src.params.bitunpack.element_offset) % FL_CHUNK; smem_src = scratch + align; - // Write barrier: all threads finished bitunpack (and any - // patches), safe to read from scratch. - __syncthreads(); } else { chunk_len = block_len; } @@ -438,11 +521,12 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) { const auto &src = stage.source; if (src.op_code == SourceOp::BITUNPACK) { - T *raw_smem = smem_out; - bitunpack(reinterpret_cast(stage.input_ptr), smem_out, 0, stage.len, src); - // Write barrier: cooperative bitunpack finished, safe to read - // decoded elements below. - __syncthreads(); + bitunpack_typed(smem_out, + reinterpret_cast(stage.input_ptr), + 0, + stage.len, + src, + stage.source_ptype); smem_out += src.params.bitunpack.element_offset % SMEM_TILE_SIZE; diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.h b/vortex-cuda/kernels/src/dynamic_dispatch.h index 9bba3f50a04..2e51240ed4f 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.h +++ b/vortex-cuda/kernels/src/dynamic_dispatch.h @@ -78,6 +78,27 @@ PTYPE_HOST_DEVICE constexpr PTypeTag ptype_to_unsigned(PTypeTag tag) { return tag; } } + +PTYPE_HOST_DEVICE constexpr uint8_t ptype_byte_width(PTypeTag tag) { + switch (tag) { + case PTYPE_U8: + case PTYPE_I8: + return 1; + case PTYPE_U16: + case PTYPE_I16: + return 2; + case PTYPE_U32: + case PTYPE_I32: + case PTYPE_F32: + return 4; + case PTYPE_U64: + case PTYPE_I64: + case PTYPE_F64: + return 8; + default: + return 0; + } +} #endif /// Number of threads per CUDA block. diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 56df2f21d45..d4d2a3c408b 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -1806,7 +1806,8 @@ mod tests { #[crate::test] async fn test_dict_bitpacked_u8_codes_u32_values() -> VortexResult<()> { // Dict with BitPacked u8 codes (narrower than u32 output) and u32 values. - // Codes become a pending subtree, values fuse. + // The kernel's bitunpack_typed decodes at the source's native width and + // widens to T, so this fuses into a single kernel launch. let dict_values: Vec = vec![100, 200, 300, 400]; let len = 2048; let codes: Vec = (0..len).map(|i| (i % dict_values.len()) as u8).collect(); @@ -1825,8 +1826,8 @@ mod tests { let plan = DispatchPlan::new(&array)?; assert!( - matches!(plan, DispatchPlan::PartiallyFused { .. }), - "expected PartiallyFused for mixed-width Dict with BitPacked codes" + matches!(plan, DispatchPlan::Fused(..)), + "expected Fused for mixed-width Dict with BitPacked codes" ); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; @@ -1840,6 +1841,202 @@ mod tests { Ok(()) } + /// Dict with non-Primitive narrower codes (BitPacked) at various + /// code/value widths. These fuse into a single kernel because + /// bitunpack_typed decodes at the source's native width and widens to T. + #[rstest] + #[case::bp_u8_codes_u16_values(2u8, 3000usize, vec![100u16, 200, 300, 400])] + #[case::bp_u8_codes_u32_values(2u8, 3000usize, vec![100_000u32, 200_000, 300_000, 400_000])] + #[case::bp_u16_codes_u32_values(4u8, 2048usize, vec![1_000_000u32, 2_000_000, 3_000_000, 4_000_000, 5_000_000, 6_000_000, 7_000_000, 8_000_000])] + #[crate::test] + async fn test_dict_mixed_width_bitpacked_codes( + #[case] bit_width: u8, + #[case] len: usize, + #[case] dict_values: Vec, + ) -> VortexResult<()> { + let dict_size = dict_values.len(); + let codes: Vec = (0..len).map(|i| (i % dict_size) as u8).collect(); + + let codes_prim = PrimitiveArray::new(Buffer::from(codes.clone()), NonNullable); + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack codes"); + let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); + let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; + let array = dict.into_array(); + + let plan = DispatchPlan::new(&array)?; + assert!( + matches!(plan, DispatchPlan::Fused(..)), + "expected Fused for mixed-width Dict with BitPacked codes" + ); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let expected: Vec = codes.iter().map(|&c| dict_values[c as usize]).collect(); + let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + + Ok(()) + } + + /// Dict with FoR(BitPacked) codes narrower than the value type. + /// The entire codes decode chain runs at the narrower width; the kernel + /// decodes at native width and widens to T, fusing everything. + #[rstest] + #[case::for_bp_u8_codes_u32_values(3u8, 3000usize, vec![100u32, 200, 300, 400, 500, 600, 700, 800])] + #[case::for_bp_u16_codes_u32_values(4u8, 2048usize, vec![10_000u32, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000])] + #[crate::test] + async fn test_dict_mixed_width_for_bp_codes( + #[case] bit_width: u8, + #[case] len: usize, + #[case] dict_values: Vec, + ) -> VortexResult<()> { + let dict_size = dict_values.len(); + let codes: Vec = (0..len).map(|i| (i % dict_size) as u8).collect(); + + let codes_prim = PrimitiveArray::new(Buffer::from(codes.clone()), NonNullable); + let codes_bp = BitPacked::encode( + &codes_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack codes"); + let codes_for = FoR::try_new(codes_bp.into_array(), Scalar::from(0u8))?; + let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); + let dict = DictArray::try_new(codes_for.into_array(), values_prim.into_array())?; + let array = dict.into_array(); + + let plan = DispatchPlan::new(&array)?; + assert!( + matches!(plan, DispatchPlan::Fused(..)), + "expected Fused for mixed-width Dict with FoR(BitPacked) codes" + ); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let expected: Vec = codes.iter().map(|&c| dict_values[c as usize]).collect(); + let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + + Ok(()) + } + + /// RunEnd with non-Primitive narrower ends (BitPacked) and wider output + /// values. The kernel decodes at the source's native width and widens + /// to T in shared memory, fusing everything into one launch. + #[rstest] + #[case::bp_u8_ends_u16_values( + vec![25u8, 50, 75, 100], + vec![10u16, 20, 30, 40], + )] + #[case::bp_u8_ends_u32_values( + vec![40u8, 80, 120], + vec![1000u32, 2000, 3000], + )] + #[case::bp_u16_ends_u32_values( + vec![500u16, 1000, 1500, 2000], + vec![100_000u32, 200_000, 300_000, 400_000], + )] + #[crate::test] + async fn test_runend_mixed_width_bitpacked_ends< + E: vortex::dtype::NativePType + Into, + V: vortex::dtype::NativePType, + >( + #[case] ends: Vec, + #[case] values: Vec, + ) -> VortexResult<()> { + let ends_u64: Vec = ends.iter().map(|e| (*e).into()).collect(); + let len = *ends_u64.last().unwrap() as usize; + let bit_width = 64 - ends_u64.iter().max().unwrap().leading_zeros() as u8; + let ends_prim = PrimitiveArray::new(Buffer::from(ends.clone()), NonNullable); + let ends_bp = BitPacked::encode( + &ends_prim.into_array(), + bit_width, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack ends"); + + let values_prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let re = RunEnd::new(ends_bp.into_array(), values_prim.into_array(), &mut ctx); + let array = re.into_array(); + + let plan = DispatchPlan::new(&array)?; + assert!( + matches!(plan, DispatchPlan::Fused(..)), + "expected Fused for mixed-width RunEnd with BitPacked ends" + ); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let mut expected: Vec = Vec::with_capacity(len); + let mut prev: u64 = 0; + for (end, val) in ends_u64.iter().zip(values.iter()) { + let run_len = (*end - prev) as usize; + expected.extend(std::iter::repeat_n(*val, run_len)); + prev = *end; + } + let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + + Ok(()) + } + + /// RunEnd with FoR(BitPacked) narrower ends: the full decode chain for + /// ends runs at a narrower width; the kernel's bitunpack_typed decodes + /// at native width and widens to T, fusing everything. + #[crate::test] + async fn test_runend_mixed_width_for_bp_u16_ends_u32_values() -> VortexResult<()> { + let ends: Vec = vec![500, 1000, 1500, 2000]; + let values: Vec = vec![100, 200, 300, 400]; + let len = 2000usize; + + let ends_prim = PrimitiveArray::new(Buffer::from(ends.clone()), NonNullable); + let ends_bp = BitPacked::encode( + &ends_prim.into_array(), + 11, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("bitpack ends"); + let ends_for = FoR::try_new(ends_bp.into_array(), Scalar::from(0u16))?; + + let values_prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let re = RunEnd::new(ends_for.into_array(), values_prim.into_array(), &mut ctx); + let array = re.into_array(); + + let plan = DispatchPlan::new(&array)?; + assert!( + matches!(plan, DispatchPlan::Fused(..)), + "expected Fused for mixed-width RunEnd with FoR(BitPacked) ends" + ); + + let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; + let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); + + let mut expected: Vec = Vec::with_capacity(len); + let mut prev = 0u16; + for (&end, &val) in ends.iter().zip(values.iter()) { + expected.extend(std::iter::repeat_n(val, (end - prev) as usize)); + prev = end; + } + let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); + vortex::array::assert_arrays_eq!(expected_arr, result); + + Ok(()) + } + #[crate::test] async fn test_sliced_dict_mixed_width() -> VortexResult<()> { // Sliced Dict with u8 codes and u32 values — combines PartiallyFused + slice handling. diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index 1b63630906c..88a5afc69be 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -178,55 +178,28 @@ pub enum DispatchPlan { /// /// Stages are stored in kernel execution order. There are two phases: /// -/// 1. All stages except the last run first and decode their output -/// into shared memory (e.g. all dict values, all run-end endpoints). -/// This data stays resident for the output stage to index into. +/// 1. All stages except the last decode into shared memory (dict values, +/// run-end endpoints). The kernel writes `T`-wide elements even when +/// a stage's source ptype is narrower, widening in-place as needed. /// -/// 2. The last stage (the output stage) iterates over the input in tiles -/// of `SMEM_TILE_SIZE` (1024) elements, decoding each tile into a -/// scratch region of shared memory, applying scalar ops (which may -/// reference data from the earlier stages), and writing the result to -/// global memory. -/// -/// # Per-stage PType tracking -/// -/// Each stage carries a `source_ptype` (`PTypeTag`) that identifies the -/// primitive type produced by its source op (LOAD, BITUNPACK, etc.). -/// Scalar ops may change the type (e.g. DICT transforms codes → values, -/// ALP transforms encoded ints → floats); each `ScalarOp` declares its -/// `output_ptype`. The kernel uses these tags to dispatch typed memory -/// operations and cross-stage references at the correct element width. +/// 2. The last stage (the output stage) tiles at `SMEM_TILE_SIZE` (1024) +/// elements, decoding each tile into a scratch region, applying scalar +/// ops (which may reference earlier stages), and streaming to global +/// memory. /// /// # Shared memory allocation /// -/// Total shared memory = `smem_byte_cursor` + `SMEM_TILE_SIZE` × `output_elem_bytes`. -/// -/// `smem_byte_cursor` is tracked in bytes and covers the preceding -/// fully-decoded stages (dict values, run-end endpoints). Each stage's -/// shared memory footprint is `len × final_ptype_byte_width`, where the -/// final ptype is determined by the last scalar op's `output_ptype` (or -/// `source_ptype` if there are no scalar ops). -/// -/// All shared memory offsets are byte offsets — the C ABI uses byte -/// offsets and per-field `PTypeTag` values so that stages with different -/// element widths can coexist in the same shared memory pool. -/// -/// This is sufficient because: +/// Total = `smem_byte_cursor` + `SMEM_TILE_SIZE × output_elem_bytes`. /// -/// - Earlier stages only originate from dict (values) and run-end (ends, -/// values). `push_smem_stage` reserves the appropriate number of bytes -/// in `smem_byte_cursor`, so each stage's source op has room to decode -/// the complete input. +/// Each input stage occupies `len × max(final_width, output_elem_bytes)` +/// bytes, where `final_width` is the byte width of the last scalar op's +/// `output_ptype` (or `source_ptype` if none). The `max` is necessary +/// because `execute_input_stage` writes `T`-wide elements even when +/// the stage's logical type is narrower. /// -/// - The output stage (last) tiles at `SMEM_TILE_SIZE` (1024 elements), -/// so its source op never writes more than 1024 elements into the -/// scratch region, even though each block is responsible for -/// `ELEMENTS_PER_BLOCK` (2048) output elements — it processes them in -/// two passes through the scratch. -/// -/// Note: `BITUNPACK` writes full FastLanes blocks (1024 elements), which can -/// exceed `stage.len` by up to 1023 elements. This overflow is absorbed by -/// the scratch region (`SMEM_TILE_SIZE` ≥ `FL_CHUNK_SIZE`). +/// `BITUNPACK` writes full FastLanes blocks (1024 elements) which may +/// exceed `stage.len` by up to 1023 elements; this overflow is absorbed +/// by the scratch region (`SMEM_TILE_SIZE` ≥ `FL_CHUNK_SIZE`). pub struct FusedPlan { /// Stages in kernel execution order; all but the last decode into /// shared memory, the last decodes into global memory. @@ -445,22 +418,7 @@ impl FusedPlan { pending_subtrees: &mut Vec, ) -> VortexResult { if !is_dyn_dispatch_compatible(&array) { - // Subtree can't be fused — record it as a deferred LOAD source. - // Bail if dtype is non-primitive (can't become a LOAD stage). - let ptype = PType::try_from(array.dtype()).map_err(|_| { - vortex_err!( - "unfusable subtree has non-primitive dtype {:?}, cannot partially fuse", - array.dtype() - ) - })?; - let buf_idx = self.source_buffers.len(); - self.source_buffers.push(None); // placeholder, filled at materialize time - pending_subtrees.push(array); - return Ok(Stage::new( - SourceOp::load(), - Some(buf_idx), - ptype_to_tag(ptype), - )); + return self.push_subtree(array, pending_subtrees); } let id = array.encoding_id(); @@ -639,24 +597,53 @@ impl FusedPlan { Ok(pipeline) } - /// Handle a child array whose element width differs from the output type. + /// Walk a child that may have a different element width than the output. /// - /// If the child is a `Primitive`, its buffer is grabbed directly as a LOAD - /// source — no separate kernel launch needed, since `load_element()` - /// handles the widening in-kernel. Otherwise, the child is recorded as a - /// pending subtree for separate execution. - fn walk_mixed_width_child( + /// Primitives are always handled directly (`load_element()` widens + /// in-kernel). Non-primitive children are recursively walked; the kernel's + /// `bitunpack_typed` decodes at the source's native width and widens to + /// `T` in shared memory, and `push_smem_stage` allocates accordingly. + fn walk_child( &mut self, - child: ArrayRef, + array: ArrayRef, pending_subtrees: &mut Vec, ) -> VortexResult { - let ptype = PType::try_from(child.dtype())?; - if child.encoding_id() == Primitive.id() { - return self.walk_primitive(child); + if array.encoding_id() == Primitive.id() { + return self.walk_primitive(array); } + self.walk(array, pending_subtrees) + } + + /// Reserve a placeholder buffer slot and record the array as a pending subtree. + /// + /// Called from [`walk`] when [`is_dyn_dispatch_compatible`] rejects a child. + /// Cases that require a separate kernel dispatch: + /// + /// - **F16 / F64 primitives** — no reinterpret path in the kernel. + /// - **ALP with non-F32 dtype** — only F32 ALP is supported. + /// - **Dict with nullable codes** — garbage at null positions could OOB + /// the DICT gather in shared memory. + /// - **Dict with codes wider than values** — `load_element()` would + /// truncate the code indices. + /// - **RunEnd with nullable ends** — garbage values break the binary + /// search / forward-scan. + /// - **RunEnd with ends wider than values** — same truncation issue. + /// - **Unrecognized encoding** — anything outside the kernel's allow-list + /// (e.g. FSST, Pco, Zstd). + fn push_subtree( + &mut self, + array: ArrayRef, + pending_subtrees: &mut Vec, + ) -> VortexResult { + let ptype = PType::try_from(array.dtype()).map_err(|_| { + vortex_err!( + "unfusable subtree has non-primitive dtype {:?}, cannot partially fuse", + array.dtype() + ) + })?; let buf_idx = self.source_buffers.len(); self.source_buffers.push(None); - pending_subtrees.push(child); + pending_subtrees.push(array); Ok(Stage::new( SourceOp::load(), Some(buf_idx), @@ -674,28 +661,12 @@ impl FusedPlan { let codes = dict.codes().clone(); let values_ptype = PType::try_from(values.dtype())?; - let values_elem_bytes = values_ptype.byte_width() as u32; - let codes_ptype = PType::try_from(codes.dtype())?; - let codes_elem_bytes = codes_ptype.byte_width() as u32; - - // If values have a different width than the output type, they - // can't be fused into the same kernel instantiation. Primitives - // are handled directly (just grab the buffer); other encodings - // become pending subtrees executed by a separate kernel. + let values_len = values.len() as u32; - let values_spec = if values_elem_bytes != self.output_elem_bytes { - self.walk_mixed_width_child(values, pending_subtrees)? - } else { - self.walk(values, pending_subtrees)? - }; + let values_spec = self.walk_child(values, pending_subtrees)?; let values_smem_byte_offset = self.push_smem_stage(values_spec, values_len); - // Same for codes. - let mut pipeline = if codes_elem_bytes != self.output_elem_bytes { - self.walk_mixed_width_child(codes, pending_subtrees)? - } else { - self.walk(codes, pending_subtrees)? - }; + let mut pipeline = self.walk_child(codes, pending_subtrees)?; // DICT scalar op: pass byte offset directly (C ABI uses byte offsets). // output_ptype is the values' ptype — DICT transforms codes → values. pipeline.scalar_ops.push(( @@ -727,26 +698,10 @@ impl FusedPlan { let num_runs = ends.len() as u32; let num_values = values.len() as u32; - let ends_ptype = PType::try_from(ends.dtype())?; - let ends_elem_bytes = ends_ptype.byte_width() as u32; - let values_ptype = PType::try_from(values.dtype())?; - let values_elem_bytes = values_ptype.byte_width() as u32; - - // If ends or values have a different width than the output type, - // they can't be fused into the same kernel instantiation. - // Primitives are handled directly; others become pending subtrees. - let ends_spec = if ends_elem_bytes != self.output_elem_bytes { - self.walk_mixed_width_child(ends, pending_subtrees)? - } else { - self.walk(ends, pending_subtrees)? - }; + let ends_spec = self.walk_child(ends, pending_subtrees)?; let ends_smem_byte_offset = self.push_smem_stage(ends_spec, num_runs); - let values_spec = if values_elem_bytes != self.output_elem_bytes { - self.walk_mixed_width_child(values, pending_subtrees)? - } else { - self.walk(values, pending_subtrees)? - }; + let values_spec = self.walk_child(values, pending_subtrees)?; let values_smem_byte_offset = self.push_smem_stage(values_spec, num_values); // Pass byte offsets and PTypeTags directly — the C ABI now uses @@ -764,13 +719,10 @@ impl FusedPlan { } /// Add a stage that decodes fully into shared memory before the output - /// stage runs. Returns the shared memory byte offset where the data starts. - /// - /// The smem region is sized at the stage's output ptype width — i.e. - /// the ptype after all scalar ops have run. For stages that go through - /// type-changing scalar ops (e.g. dict values with FoR→ALP), the final - /// smem footprint is `len × final_ptype_byte_width`. If there are no - /// scalar ops, the source_ptype determines the width. + /// stage runs. Returns the shared memory byte offset where the data + /// starts. Allocates `len × max(final_width, output_elem_bytes)` bytes + /// so that narrower stages widened to `T` by `bitunpack_typed` never + /// overflow. fn push_smem_stage(&mut self, spec: Stage, len: u32) -> u32 { let smem_byte_offset = self.smem_byte_cursor; // The kernel's execute_input_stage always writes T-wide elements From 56b07314f63dabac4c99351b335401b8ebc8d8fe Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:45:14 -0400 Subject: [PATCH 175/250] Revert #7603 (#7613) This reverts commit 09ae129b7dc6a6f26df5b591c4f88b24512a3c49. ## Summary Closes: #000 ## Testing --- vortex-cuda/kernels/src/dynamic_dispatch.cu | 116 ++-------- vortex-cuda/kernels/src/dynamic_dispatch.h | 21 -- vortex-cuda/src/dynamic_dispatch/mod.rs | 203 +----------------- .../src/dynamic_dispatch/plan_builder.rs | 180 ++++++++++------ 4 files changed, 133 insertions(+), 387 deletions(-) diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.cu b/vortex-cuda/kernels/src/dynamic_dispatch.cu index a0a1039900e..9a413af17c1 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.cu +++ b/vortex-cuda/kernels/src/dynamic_dispatch.cu @@ -39,16 +39,9 @@ // // ## Mixed-width support // -// Dict codes, RunEnd ends, and other child arrays may have a narrower -// element type than the output T. Two mechanisms handle this: -// -// LOAD load_element() dispatches on the per-stage PTypeTag to -// read at the source's native width and static_cast to T. -// BITUNPACK bitunpack_typed() unpacks at the source's native width, -// then widens to T in-place via a backward scan -// (widen_inplace). The smem region is pre-allocated at -// max(source_width, T) bytes per element by the Rust plan -// builder, so the widen never overflows. +// LOAD sources from pending subtrees may have a narrower type than the +// output (e.g. u8 dict codes in a u32 plan). load_element() widens +// to T via static_cast — no separate widen kernel or smem intermediate. #include #include @@ -210,22 +203,6 @@ scatter_patches_chunk(const GPUPatches &patches, T *__restrict out, uint32_t chu // Source ops // ═══════════════════════════════════════════════════════════════════════════ -/// Widen U-sized elements in shared memory to T-sized, in-place. -/// Backward scan ensures no unread element is overwritten since -/// sizeof(T) >= sizeof(U) guarantees the write at index i touches -/// only bytes beyond those of src[i]. -template -__device__ inline void widen_inplace(T *dst, uint32_t len) { - if constexpr (sizeof(T) <= sizeof(U)) { - return; - } - const U *src = reinterpret_cast(dst); - for (int32_t i = static_cast(len) - 1 - threadIdx.x; i >= 0; i -= blockDim.x) { - dst[i] = static_cast(src[i]); - } - __syncthreads(); -} - /// FastLanes cooperative unpack — all threads in the block scatter-write /// decoded elements into `dst`. Caller must issue __syncthreads() before /// any thread reads from `dst`. @@ -259,68 +236,6 @@ __device__ inline void bitunpack(const T *__restrict packed, } } -/// Dispatch bitunpack at the source's native element width, then widen -/// to T in-place so all downstream scalar ops and smem consumers see -/// T-sized elements. Falls back to the direct `bitunpack` path when -/// the source ptype already matches T. Issues __syncthreads() before -/// returning on all paths. -/// -/// Accepts explicit chunk_start / chunk_len so it works for both input -/// stages (full decode with chunk_start=0, chunk_len=stage.len) and -/// the output stage (tiled with varying chunk_start / chunk_len). -template -__device__ inline void bitunpack_typed(T *__restrict dst, - const void *__restrict packed, - uint64_t chunk_start, - uint32_t chunk_len, - const struct SourceOp &src, - PTypeTag source_ptype) { - // Fast path: source width matches T — no widening needed. - if (ptype_byte_width(source_ptype) == sizeof(T)) { - bitunpack(reinterpret_cast(packed), dst, chunk_start, chunk_len, src); - __syncthreads(); - return; - } - - // Compute total elements written by bitunpack (including alignment - // padding) so widen_inplace covers the full scratch region. - const uint32_t elem_off = src.params.bitunpack.element_offset; - const uint32_t dst_off = (chunk_start + elem_off) % FL_CHUNK; - const uint32_t n_chunks = (chunk_len + dst_off + FL_CHUNK - 1) / FL_CHUNK; - const uint32_t total_elems = n_chunks * FL_CHUNK; - - // Narrow source: unpack at native width, then widen to T. - switch (source_ptype) { - case PTYPE_U8: - case PTYPE_I8: { - auto *narrow = reinterpret_cast(dst); - bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); - __syncthreads(); - widen_inplace(dst, total_elems); - break; - } - case PTYPE_U16: - case PTYPE_I16: { - auto *narrow = reinterpret_cast(dst); - bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); - __syncthreads(); - widen_inplace(dst, total_elems); - break; - } - case PTYPE_U32: - case PTYPE_I32: - case PTYPE_F32: { - auto *narrow = reinterpret_cast(dst); - bitunpack(reinterpret_cast(packed), narrow, chunk_start, chunk_len, src); - __syncthreads(); - widen_inplace(dst, total_elems); - break; - } - default: - __builtin_unreachable(); - } -} - /// Read N values from a source op into `out`. /// /// Dispatches on `src.op_code` to handle each encoding: @@ -439,14 +354,16 @@ __device__ void execute_output_stage(T *__restrict output, if (src.op_code == SourceOp::BITUNPACK) { chunk_len = bitunpack_tile_len(stage, block_len, elem_idx); T *scratch = reinterpret_cast(smem + stage.smem_byte_offset); - bitunpack_typed(scratch, - reinterpret_cast(stage.input_ptr), - block_start + elem_idx, - chunk_len, - src, - ptype); + bitunpack(reinterpret_cast(stage.input_ptr), + scratch, + block_start + elem_idx, + chunk_len, + src); const uint32_t align = (block_start + elem_idx + src.params.bitunpack.element_offset) % FL_CHUNK; smem_src = scratch + align; + // Write barrier: all threads finished bitunpack (and any + // patches), safe to read from scratch. + __syncthreads(); } else { chunk_len = block_len; } @@ -521,12 +438,11 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) { const auto &src = stage.source; if (src.op_code == SourceOp::BITUNPACK) { - bitunpack_typed(smem_out, - reinterpret_cast(stage.input_ptr), - 0, - stage.len, - src, - stage.source_ptype); + T *raw_smem = smem_out; + bitunpack(reinterpret_cast(stage.input_ptr), smem_out, 0, stage.len, src); + // Write barrier: cooperative bitunpack finished, safe to read + // decoded elements below. + __syncthreads(); smem_out += src.params.bitunpack.element_offset % SMEM_TILE_SIZE; diff --git a/vortex-cuda/kernels/src/dynamic_dispatch.h b/vortex-cuda/kernels/src/dynamic_dispatch.h index 2e51240ed4f..9bba3f50a04 100644 --- a/vortex-cuda/kernels/src/dynamic_dispatch.h +++ b/vortex-cuda/kernels/src/dynamic_dispatch.h @@ -78,27 +78,6 @@ PTYPE_HOST_DEVICE constexpr PTypeTag ptype_to_unsigned(PTypeTag tag) { return tag; } } - -PTYPE_HOST_DEVICE constexpr uint8_t ptype_byte_width(PTypeTag tag) { - switch (tag) { - case PTYPE_U8: - case PTYPE_I8: - return 1; - case PTYPE_U16: - case PTYPE_I16: - return 2; - case PTYPE_U32: - case PTYPE_I32: - case PTYPE_F32: - return 4; - case PTYPE_U64: - case PTYPE_I64: - case PTYPE_F64: - return 8; - default: - return 0; - } -} #endif /// Number of threads per CUDA block. diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index d4d2a3c408b..56df2f21d45 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -1806,8 +1806,7 @@ mod tests { #[crate::test] async fn test_dict_bitpacked_u8_codes_u32_values() -> VortexResult<()> { // Dict with BitPacked u8 codes (narrower than u32 output) and u32 values. - // The kernel's bitunpack_typed decodes at the source's native width and - // widens to T, so this fuses into a single kernel launch. + // Codes become a pending subtree, values fuse. let dict_values: Vec = vec![100, 200, 300, 400]; let len = 2048; let codes: Vec = (0..len).map(|i| (i % dict_values.len()) as u8).collect(); @@ -1826,8 +1825,8 @@ mod tests { let plan = DispatchPlan::new(&array)?; assert!( - matches!(plan, DispatchPlan::Fused(..)), - "expected Fused for mixed-width Dict with BitPacked codes" + matches!(plan, DispatchPlan::PartiallyFused { .. }), + "expected PartiallyFused for mixed-width Dict with BitPacked codes" ); let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; @@ -1841,202 +1840,6 @@ mod tests { Ok(()) } - /// Dict with non-Primitive narrower codes (BitPacked) at various - /// code/value widths. These fuse into a single kernel because - /// bitunpack_typed decodes at the source's native width and widens to T. - #[rstest] - #[case::bp_u8_codes_u16_values(2u8, 3000usize, vec![100u16, 200, 300, 400])] - #[case::bp_u8_codes_u32_values(2u8, 3000usize, vec![100_000u32, 200_000, 300_000, 400_000])] - #[case::bp_u16_codes_u32_values(4u8, 2048usize, vec![1_000_000u32, 2_000_000, 3_000_000, 4_000_000, 5_000_000, 6_000_000, 7_000_000, 8_000_000])] - #[crate::test] - async fn test_dict_mixed_width_bitpacked_codes( - #[case] bit_width: u8, - #[case] len: usize, - #[case] dict_values: Vec, - ) -> VortexResult<()> { - let dict_size = dict_values.len(); - let codes: Vec = (0..len).map(|i| (i % dict_size) as u8).collect(); - - let codes_prim = PrimitiveArray::new(Buffer::from(codes.clone()), NonNullable); - let codes_bp = BitPacked::encode( - &codes_prim.into_array(), - bit_width, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("bitpack codes"); - let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); - let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; - let array = dict.into_array(); - - let plan = DispatchPlan::new(&array)?; - assert!( - matches!(plan, DispatchPlan::Fused(..)), - "expected Fused for mixed-width Dict with BitPacked codes" - ); - - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; - let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); - - let expected: Vec = codes.iter().map(|&c| dict_values[c as usize]).collect(); - let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); - vortex::array::assert_arrays_eq!(expected_arr, result); - - Ok(()) - } - - /// Dict with FoR(BitPacked) codes narrower than the value type. - /// The entire codes decode chain runs at the narrower width; the kernel - /// decodes at native width and widens to T, fusing everything. - #[rstest] - #[case::for_bp_u8_codes_u32_values(3u8, 3000usize, vec![100u32, 200, 300, 400, 500, 600, 700, 800])] - #[case::for_bp_u16_codes_u32_values(4u8, 2048usize, vec![10_000u32, 20_000, 30_000, 40_000, 50_000, 60_000, 70_000, 80_000])] - #[crate::test] - async fn test_dict_mixed_width_for_bp_codes( - #[case] bit_width: u8, - #[case] len: usize, - #[case] dict_values: Vec, - ) -> VortexResult<()> { - let dict_size = dict_values.len(); - let codes: Vec = (0..len).map(|i| (i % dict_size) as u8).collect(); - - let codes_prim = PrimitiveArray::new(Buffer::from(codes.clone()), NonNullable); - let codes_bp = BitPacked::encode( - &codes_prim.into_array(), - bit_width, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("bitpack codes"); - let codes_for = FoR::try_new(codes_bp.into_array(), Scalar::from(0u8))?; - let values_prim = PrimitiveArray::new(Buffer::from(dict_values.clone()), NonNullable); - let dict = DictArray::try_new(codes_for.into_array(), values_prim.into_array())?; - let array = dict.into_array(); - - let plan = DispatchPlan::new(&array)?; - assert!( - matches!(plan, DispatchPlan::Fused(..)), - "expected Fused for mixed-width Dict with FoR(BitPacked) codes" - ); - - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; - let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); - - let expected: Vec = codes.iter().map(|&c| dict_values[c as usize]).collect(); - let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); - vortex::array::assert_arrays_eq!(expected_arr, result); - - Ok(()) - } - - /// RunEnd with non-Primitive narrower ends (BitPacked) and wider output - /// values. The kernel decodes at the source's native width and widens - /// to T in shared memory, fusing everything into one launch. - #[rstest] - #[case::bp_u8_ends_u16_values( - vec![25u8, 50, 75, 100], - vec![10u16, 20, 30, 40], - )] - #[case::bp_u8_ends_u32_values( - vec![40u8, 80, 120], - vec![1000u32, 2000, 3000], - )] - #[case::bp_u16_ends_u32_values( - vec![500u16, 1000, 1500, 2000], - vec![100_000u32, 200_000, 300_000, 400_000], - )] - #[crate::test] - async fn test_runend_mixed_width_bitpacked_ends< - E: vortex::dtype::NativePType + Into, - V: vortex::dtype::NativePType, - >( - #[case] ends: Vec, - #[case] values: Vec, - ) -> VortexResult<()> { - let ends_u64: Vec = ends.iter().map(|e| (*e).into()).collect(); - let len = *ends_u64.last().unwrap() as usize; - let bit_width = 64 - ends_u64.iter().max().unwrap().leading_zeros() as u8; - let ends_prim = PrimitiveArray::new(Buffer::from(ends.clone()), NonNullable); - let ends_bp = BitPacked::encode( - &ends_prim.into_array(), - bit_width, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("bitpack ends"); - - let values_prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let re = RunEnd::new(ends_bp.into_array(), values_prim.into_array(), &mut ctx); - let array = re.into_array(); - - let plan = DispatchPlan::new(&array)?; - assert!( - matches!(plan, DispatchPlan::Fused(..)), - "expected Fused for mixed-width RunEnd with BitPacked ends" - ); - - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; - let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); - - let mut expected: Vec = Vec::with_capacity(len); - let mut prev: u64 = 0; - for (end, val) in ends_u64.iter().zip(values.iter()) { - let run_len = (*end - prev) as usize; - expected.extend(std::iter::repeat_n(*val, run_len)); - prev = *end; - } - let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); - vortex::array::assert_arrays_eq!(expected_arr, result); - - Ok(()) - } - - /// RunEnd with FoR(BitPacked) narrower ends: the full decode chain for - /// ends runs at a narrower width; the kernel's bitunpack_typed decodes - /// at native width and widens to T, fusing everything. - #[crate::test] - async fn test_runend_mixed_width_for_bp_u16_ends_u32_values() -> VortexResult<()> { - let ends: Vec = vec![500, 1000, 1500, 2000]; - let values: Vec = vec![100, 200, 300, 400]; - let len = 2000usize; - - let ends_prim = PrimitiveArray::new(Buffer::from(ends.clone()), NonNullable); - let ends_bp = BitPacked::encode( - &ends_prim.into_array(), - 11, - &mut LEGACY_SESSION.create_execution_ctx(), - ) - .vortex_expect("bitpack ends"); - let ends_for = FoR::try_new(ends_bp.into_array(), Scalar::from(0u16))?; - - let values_prim = PrimitiveArray::new(Buffer::from(values.clone()), NonNullable); - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let re = RunEnd::new(ends_for.into_array(), values_prim.into_array(), &mut ctx); - let array = re.into_array(); - - let plan = DispatchPlan::new(&array)?; - assert!( - matches!(plan, DispatchPlan::Fused(..)), - "expected Fused for mixed-width RunEnd with FoR(BitPacked) ends" - ); - - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; - let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; - let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); - - let mut expected: Vec = Vec::with_capacity(len); - let mut prev = 0u16; - for (&end, &val) in ends.iter().zip(values.iter()) { - expected.extend(std::iter::repeat_n(val, (end - prev) as usize)); - prev = end; - } - let expected_arr = PrimitiveArray::new(Buffer::from(expected), NonNullable).into_array(); - vortex::array::assert_arrays_eq!(expected_arr, result); - - Ok(()) - } - #[crate::test] async fn test_sliced_dict_mixed_width() -> VortexResult<()> { // Sliced Dict with u8 codes and u32 values — combines PartiallyFused + slice handling. diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index 88a5afc69be..1b63630906c 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -178,28 +178,55 @@ pub enum DispatchPlan { /// /// Stages are stored in kernel execution order. There are two phases: /// -/// 1. All stages except the last decode into shared memory (dict values, -/// run-end endpoints). The kernel writes `T`-wide elements even when -/// a stage's source ptype is narrower, widening in-place as needed. +/// 1. All stages except the last run first and decode their output +/// into shared memory (e.g. all dict values, all run-end endpoints). +/// This data stays resident for the output stage to index into. /// -/// 2. The last stage (the output stage) tiles at `SMEM_TILE_SIZE` (1024) -/// elements, decoding each tile into a scratch region, applying scalar -/// ops (which may reference earlier stages), and streaming to global -/// memory. +/// 2. The last stage (the output stage) iterates over the input in tiles +/// of `SMEM_TILE_SIZE` (1024) elements, decoding each tile into a +/// scratch region of shared memory, applying scalar ops (which may +/// reference data from the earlier stages), and writing the result to +/// global memory. +/// +/// # Per-stage PType tracking +/// +/// Each stage carries a `source_ptype` (`PTypeTag`) that identifies the +/// primitive type produced by its source op (LOAD, BITUNPACK, etc.). +/// Scalar ops may change the type (e.g. DICT transforms codes → values, +/// ALP transforms encoded ints → floats); each `ScalarOp` declares its +/// `output_ptype`. The kernel uses these tags to dispatch typed memory +/// operations and cross-stage references at the correct element width. /// /// # Shared memory allocation /// -/// Total = `smem_byte_cursor` + `SMEM_TILE_SIZE × output_elem_bytes`. +/// Total shared memory = `smem_byte_cursor` + `SMEM_TILE_SIZE` × `output_elem_bytes`. +/// +/// `smem_byte_cursor` is tracked in bytes and covers the preceding +/// fully-decoded stages (dict values, run-end endpoints). Each stage's +/// shared memory footprint is `len × final_ptype_byte_width`, where the +/// final ptype is determined by the last scalar op's `output_ptype` (or +/// `source_ptype` if there are no scalar ops). +/// +/// All shared memory offsets are byte offsets — the C ABI uses byte +/// offsets and per-field `PTypeTag` values so that stages with different +/// element widths can coexist in the same shared memory pool. +/// +/// This is sufficient because: /// -/// Each input stage occupies `len × max(final_width, output_elem_bytes)` -/// bytes, where `final_width` is the byte width of the last scalar op's -/// `output_ptype` (or `source_ptype` if none). The `max` is necessary -/// because `execute_input_stage` writes `T`-wide elements even when -/// the stage's logical type is narrower. +/// - Earlier stages only originate from dict (values) and run-end (ends, +/// values). `push_smem_stage` reserves the appropriate number of bytes +/// in `smem_byte_cursor`, so each stage's source op has room to decode +/// the complete input. /// -/// `BITUNPACK` writes full FastLanes blocks (1024 elements) which may -/// exceed `stage.len` by up to 1023 elements; this overflow is absorbed -/// by the scratch region (`SMEM_TILE_SIZE` ≥ `FL_CHUNK_SIZE`). +/// - The output stage (last) tiles at `SMEM_TILE_SIZE` (1024 elements), +/// so its source op never writes more than 1024 elements into the +/// scratch region, even though each block is responsible for +/// `ELEMENTS_PER_BLOCK` (2048) output elements — it processes them in +/// two passes through the scratch. +/// +/// Note: `BITUNPACK` writes full FastLanes blocks (1024 elements), which can +/// exceed `stage.len` by up to 1023 elements. This overflow is absorbed by +/// the scratch region (`SMEM_TILE_SIZE` ≥ `FL_CHUNK_SIZE`). pub struct FusedPlan { /// Stages in kernel execution order; all but the last decode into /// shared memory, the last decodes into global memory. @@ -418,7 +445,22 @@ impl FusedPlan { pending_subtrees: &mut Vec, ) -> VortexResult { if !is_dyn_dispatch_compatible(&array) { - return self.push_subtree(array, pending_subtrees); + // Subtree can't be fused — record it as a deferred LOAD source. + // Bail if dtype is non-primitive (can't become a LOAD stage). + let ptype = PType::try_from(array.dtype()).map_err(|_| { + vortex_err!( + "unfusable subtree has non-primitive dtype {:?}, cannot partially fuse", + array.dtype() + ) + })?; + let buf_idx = self.source_buffers.len(); + self.source_buffers.push(None); // placeholder, filled at materialize time + pending_subtrees.push(array); + return Ok(Stage::new( + SourceOp::load(), + Some(buf_idx), + ptype_to_tag(ptype), + )); } let id = array.encoding_id(); @@ -597,53 +639,24 @@ impl FusedPlan { Ok(pipeline) } - /// Walk a child that may have a different element width than the output. + /// Handle a child array whose element width differs from the output type. /// - /// Primitives are always handled directly (`load_element()` widens - /// in-kernel). Non-primitive children are recursively walked; the kernel's - /// `bitunpack_typed` decodes at the source's native width and widens to - /// `T` in shared memory, and `push_smem_stage` allocates accordingly. - fn walk_child( + /// If the child is a `Primitive`, its buffer is grabbed directly as a LOAD + /// source — no separate kernel launch needed, since `load_element()` + /// handles the widening in-kernel. Otherwise, the child is recorded as a + /// pending subtree for separate execution. + fn walk_mixed_width_child( &mut self, - array: ArrayRef, + child: ArrayRef, pending_subtrees: &mut Vec, ) -> VortexResult { - if array.encoding_id() == Primitive.id() { - return self.walk_primitive(array); + let ptype = PType::try_from(child.dtype())?; + if child.encoding_id() == Primitive.id() { + return self.walk_primitive(child); } - self.walk(array, pending_subtrees) - } - - /// Reserve a placeholder buffer slot and record the array as a pending subtree. - /// - /// Called from [`walk`] when [`is_dyn_dispatch_compatible`] rejects a child. - /// Cases that require a separate kernel dispatch: - /// - /// - **F16 / F64 primitives** — no reinterpret path in the kernel. - /// - **ALP with non-F32 dtype** — only F32 ALP is supported. - /// - **Dict with nullable codes** — garbage at null positions could OOB - /// the DICT gather in shared memory. - /// - **Dict with codes wider than values** — `load_element()` would - /// truncate the code indices. - /// - **RunEnd with nullable ends** — garbage values break the binary - /// search / forward-scan. - /// - **RunEnd with ends wider than values** — same truncation issue. - /// - **Unrecognized encoding** — anything outside the kernel's allow-list - /// (e.g. FSST, Pco, Zstd). - fn push_subtree( - &mut self, - array: ArrayRef, - pending_subtrees: &mut Vec, - ) -> VortexResult { - let ptype = PType::try_from(array.dtype()).map_err(|_| { - vortex_err!( - "unfusable subtree has non-primitive dtype {:?}, cannot partially fuse", - array.dtype() - ) - })?; let buf_idx = self.source_buffers.len(); self.source_buffers.push(None); - pending_subtrees.push(array); + pending_subtrees.push(child); Ok(Stage::new( SourceOp::load(), Some(buf_idx), @@ -661,12 +674,28 @@ impl FusedPlan { let codes = dict.codes().clone(); let values_ptype = PType::try_from(values.dtype())?; - + let values_elem_bytes = values_ptype.byte_width() as u32; + let codes_ptype = PType::try_from(codes.dtype())?; + let codes_elem_bytes = codes_ptype.byte_width() as u32; + + // If values have a different width than the output type, they + // can't be fused into the same kernel instantiation. Primitives + // are handled directly (just grab the buffer); other encodings + // become pending subtrees executed by a separate kernel. let values_len = values.len() as u32; - let values_spec = self.walk_child(values, pending_subtrees)?; + let values_spec = if values_elem_bytes != self.output_elem_bytes { + self.walk_mixed_width_child(values, pending_subtrees)? + } else { + self.walk(values, pending_subtrees)? + }; let values_smem_byte_offset = self.push_smem_stage(values_spec, values_len); - let mut pipeline = self.walk_child(codes, pending_subtrees)?; + // Same for codes. + let mut pipeline = if codes_elem_bytes != self.output_elem_bytes { + self.walk_mixed_width_child(codes, pending_subtrees)? + } else { + self.walk(codes, pending_subtrees)? + }; // DICT scalar op: pass byte offset directly (C ABI uses byte offsets). // output_ptype is the values' ptype — DICT transforms codes → values. pipeline.scalar_ops.push(( @@ -698,10 +727,26 @@ impl FusedPlan { let num_runs = ends.len() as u32; let num_values = values.len() as u32; - let ends_spec = self.walk_child(ends, pending_subtrees)?; + let ends_ptype = PType::try_from(ends.dtype())?; + let ends_elem_bytes = ends_ptype.byte_width() as u32; + let values_ptype = PType::try_from(values.dtype())?; + let values_elem_bytes = values_ptype.byte_width() as u32; + + // If ends or values have a different width than the output type, + // they can't be fused into the same kernel instantiation. + // Primitives are handled directly; others become pending subtrees. + let ends_spec = if ends_elem_bytes != self.output_elem_bytes { + self.walk_mixed_width_child(ends, pending_subtrees)? + } else { + self.walk(ends, pending_subtrees)? + }; let ends_smem_byte_offset = self.push_smem_stage(ends_spec, num_runs); - let values_spec = self.walk_child(values, pending_subtrees)?; + let values_spec = if values_elem_bytes != self.output_elem_bytes { + self.walk_mixed_width_child(values, pending_subtrees)? + } else { + self.walk(values, pending_subtrees)? + }; let values_smem_byte_offset = self.push_smem_stage(values_spec, num_values); // Pass byte offsets and PTypeTags directly — the C ABI now uses @@ -719,10 +764,13 @@ impl FusedPlan { } /// Add a stage that decodes fully into shared memory before the output - /// stage runs. Returns the shared memory byte offset where the data - /// starts. Allocates `len × max(final_width, output_elem_bytes)` bytes - /// so that narrower stages widened to `T` by `bitunpack_typed` never - /// overflow. + /// stage runs. Returns the shared memory byte offset where the data starts. + /// + /// The smem region is sized at the stage's output ptype width — i.e. + /// the ptype after all scalar ops have run. For stages that go through + /// type-changing scalar ops (e.g. dict values with FoR→ALP), the final + /// smem footprint is `len × final_ptype_byte_width`. If there are no + /// scalar ops, the source_ptype determines the width. fn push_smem_stage(&mut self, spec: Stage, len: u32) -> u32 { let smem_byte_offset = self.smem_byte_cursor; // The kernel's execute_input_stage always writes T-wide elements From b6ee27ba82f4de5a53beeee76bfa09890f3055b4 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:52:04 -0400 Subject: [PATCH 176/250] Clean up `vortex-tensor` even more (#7610) ## Summary Tracking issue: https://github.com/vortex-data/vortex/issues/7297 This is mostly just cosmetic changes that will help with some future changes incoming. - Extract validate_vector_storage_dtype from Vector::validate_dtype into types/vector/mod.rs. - Move unit_norm_tolerance and BinaryTensorOpMetadata into utils.rs; switch unit_norm_tolerance to the c*sqrt(d)*epsilon bound with a dimensions parameter. - Drop op_name from validate_binary_tensor_float_inputs. - TurboQuantConfig::seed Option -> u64. - SorfOptions::dimension -> SorfOptions::dimensions and matching SorfMatrix::try_new parameter. - centroids::get_centroids -> compute_or_get_centroids. - Crate-root #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::unwrap_in_result))]. - Extract validate_vector_storage_dtype from Vector::validate_dtype into types/vector/mod.rs. - Move unit_norm_tolerance and BinaryTensorOpMetadata into utils.rs; switch unit_norm_tolerance to the c*sqrt(d)*epsilon bound with a dimensions parameter. - Drop op_name from validate_binary_tensor_float_inputs. - TurboQuantConfig::seed Option -> u64. - SorfOptions::dimension -> SorfOptions::dimensions and matching SorfMatrix::try_new parameter. - centroids::get_centroids -> compute_or_get_centroids. - Crate-root #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::unwrap_in_result))]. ## API Changes Some renames ## Testing N/A Signed-off-by: Connor Tsui --- vortex-tensor/public-api.lock | 8 +- .../src/encodings/turboquant/centroids.rs | 24 ++-- .../src/encodings/turboquant/compress.rs | 16 +-- vortex-tensor/src/encodings/turboquant/mod.rs | 2 +- .../src/encodings/turboquant/scheme.rs | 18 ++- .../src/encodings/turboquant/tests/compute.rs | 10 +- .../encodings/turboquant/tests/nullable.rs | 8 +- .../encodings/turboquant/tests/roundtrip.rs | 24 ++-- .../encodings/turboquant/tests/structural.rs | 20 +-- vortex-tensor/src/lib.rs | 5 + .../src/scalar_fns/cosine_similarity.rs | 13 +- vortex-tensor/src/scalar_fns/inner_product.rs | 84 +---------- vortex-tensor/src/scalar_fns/l2_denorm.rs | 30 ++-- .../src/scalar_fns/sorf_transform/mod.rs | 4 +- .../src/scalar_fns/sorf_transform/rotation.rs | 6 +- .../src/scalar_fns/sorf_transform/tests.rs | 20 +-- .../src/scalar_fns/sorf_transform/vtable.rs | 66 +++++---- vortex-tensor/src/types/vector/mod.rs | 24 ++++ vortex-tensor/src/types/vector/vtable.rs | 19 +-- vortex-tensor/src/utils.rs | 131 ++++++++++++++---- vortex/benches/single_encoding_throughput.rs | 2 +- 21 files changed, 289 insertions(+), 245 deletions(-) diff --git a/vortex-tensor/public-api.lock b/vortex-tensor/public-api.lock index 42f03e73c88..7300bb399e6 100644 --- a/vortex-tensor/public-api.lock +++ b/vortex-tensor/public-api.lock @@ -28,7 +28,7 @@ pub vortex_tensor::encodings::turboquant::TurboQuantConfig::bit_width: u8 pub vortex_tensor::encodings::turboquant::TurboQuantConfig::num_rounds: u8 -pub vortex_tensor::encodings::turboquant::TurboQuantConfig::seed: core::option::Option +pub vortex_tensor::encodings::turboquant::TurboQuantConfig::seed: u64 impl core::clone::Clone for vortex_tensor::encodings::turboquant::TurboQuantConfig @@ -440,11 +440,11 @@ pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::padded_dim(&self) pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::rotate(&self, input: &[f32], output: &mut [f32]) -pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::try_new(seed: u64, dimension: usize, num_rounds: usize) -> vortex_error::VortexResult +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfMatrix::try_new(seed: u64, dimensions: usize, num_rounds: usize) -> vortex_error::VortexResult pub struct vortex_tensor::scalar_fns::sorf_transform::SorfOptions -pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::dimension: u32 +pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::dimensions: u32 pub vortex_tensor::scalar_fns::sorf_transform::SorfOptions::element_ptype: vortex_array::dtype::ptype::PType @@ -490,7 +490,7 @@ pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::clone(&self) -> impl vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable for vortex_tensor::scalar_fns::sorf_transform::SorfTransform -pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult> +pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult> pub fn vortex_tensor::scalar_fns::sorf_transform::SorfTransform::serialize(&self, view: &vortex_array::arrays::scalar_fn::vtable::ScalarFnArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult>> diff --git a/vortex-tensor/src/encodings/turboquant/centroids.rs b/vortex-tensor/src/encodings/turboquant/centroids.rs index 3111034e68f..1af86c79d85 100644 --- a/vortex-tensor/src/encodings/turboquant/centroids.rs +++ b/vortex-tensor/src/encodings/turboquant/centroids.rs @@ -36,7 +36,7 @@ static CENTROID_CACHE: LazyLock>> = LazyLock::new /// Returns `2^bit_width` centroids sorted in ascending order, representing optimal scalar /// quantization levels for the coordinate distribution after random rotation in /// `dimension`-dimensional space. -pub fn get_centroids(dimension: u32, bit_width: u8) -> VortexResult> { +pub fn compute_or_get_centroids(dimension: u32, bit_width: u8) -> VortexResult> { vortex_ensure!( (1..=MAX_BIT_WIDTH).contains(&bit_width), "TurboQuant bit_width must be 1-{}, got {bit_width}", @@ -239,7 +239,7 @@ mod tests { #[case] bits: u8, #[case] expected: usize, ) -> VortexResult<()> { - let centroids = get_centroids(dim, bits)?; + let centroids = compute_or_get_centroids(dim, bits)?; assert_eq!(centroids.len(), expected); Ok(()) } @@ -251,7 +251,7 @@ mod tests { #[case(128, 4)] #[case(768, 2)] fn centroids_are_sorted(#[case] dim: u32, #[case] bits: u8) -> VortexResult<()> { - let centroids = get_centroids(dim, bits)?; + let centroids = compute_or_get_centroids(dim, bits)?; for window in centroids.windows(2) { assert!( window[0] < window[1], @@ -268,7 +268,7 @@ mod tests { #[case(256, 2)] #[case(768, 2)] fn centroids_are_symmetric(#[case] dim: u32, #[case] bits: u8) -> VortexResult<()> { - let centroids = get_centroids(dim, bits)?; + let centroids = compute_or_get_centroids(dim, bits)?; let count = centroids.len(); for idx in 0..count / 2 { let diff = (centroids[idx] + centroids[count - 1 - idx]).abs(); @@ -287,7 +287,7 @@ mod tests { #[case(128, 1)] #[case(128, 4)] fn centroids_within_bounds(#[case] dim: u32, #[case] bits: u8) -> VortexResult<()> { - let centroids = get_centroids(dim, bits)?; + let centroids = compute_or_get_centroids(dim, bits)?; for &val in centroids.iter() { assert!( (-1.0..=1.0).contains(&val), @@ -299,15 +299,15 @@ mod tests { #[test] fn centroids_cached() -> VortexResult<()> { - let c1 = get_centroids(128, 2)?; - let c2 = get_centroids(128, 2)?; + let c1 = compute_or_get_centroids(128, 2)?; + let c2 = compute_or_get_centroids(128, 2)?; assert_eq!(c1, c2); Ok(()) } #[test] fn find_nearest_basic() -> VortexResult<()> { - let centroids = get_centroids(128, 2)?; + let centroids = compute_or_get_centroids(128, 2)?; let boundaries = compute_centroid_boundaries(¢roids); assert_eq!(find_nearest_centroid(-1.0, &boundaries), 0); @@ -324,9 +324,9 @@ mod tests { #[test] fn rejects_invalid_params() { - assert!(get_centroids(128, 0).is_err()); - assert!(get_centroids(128, 9).is_err()); - assert!(get_centroids(1, 2).is_err()); - assert!(get_centroids(127, 2).is_err()); + assert!(compute_or_get_centroids(128, 0).is_err()); + assert!(compute_or_get_centroids(128, 9).is_err()); + assert!(compute_or_get_centroids(1, 2).is_err()); + assert!(compute_or_get_centroids(127, 2).is_err()); } } diff --git a/vortex-tensor/src/encodings/turboquant/compress.rs b/vortex-tensor/src/encodings/turboquant/compress.rs index 58c5666525a..ca32faa6ec9 100644 --- a/vortex-tensor/src/encodings/turboquant/compress.rs +++ b/vortex-tensor/src/encodings/turboquant/compress.rs @@ -32,8 +32,8 @@ use vortex_error::vortex_ensure; use crate::encodings::turboquant::MAX_BIT_WIDTH; use crate::encodings::turboquant::MIN_DIMENSION; use crate::encodings::turboquant::centroids::compute_centroid_boundaries; +use crate::encodings::turboquant::centroids::compute_or_get_centroids; use crate::encodings::turboquant::centroids::find_nearest_centroid; -use crate::encodings::turboquant::centroids::get_centroids; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::scalar_fns::l2_denorm::normalize_as_l2_denorm; use crate::scalar_fns::sorf_transform::SorfMatrix; @@ -48,8 +48,8 @@ use crate::utils::cast_to_f32; pub struct TurboQuantConfig { /// Bits per coordinate (1-8). pub bit_width: u8, - /// Optional seed for the rotation matrix. If None, the default seed is used. - pub seed: Option, + /// Seed for the rotation matrix. + pub seed: u64, /// Number of sign-diagonal + WHT rounds in the structured rotation (default 3). pub num_rounds: u8, } @@ -58,7 +58,7 @@ impl Default for TurboQuantConfig { fn default() -> Self { Self { bit_width: MAX_BIT_WIDTH, - seed: Some(42), + seed: 42, num_rounds: 3, } } @@ -141,7 +141,7 @@ pub unsafe fn turboquant_encode_unchecked( let vector_metadata = ext_dtype.as_extension().metadata::(); let element_ptype = vector_metadata.element_ptype(); - let seed = config.seed.unwrap_or(42); + let seed = config.seed; let num_rows = fsl.len(); if fsl.is_empty() { @@ -161,7 +161,7 @@ pub unsafe fn turboquant_encode_unchecked( let sorf_options = SorfOptions { seed, num_rounds: config.num_rounds, - dimension, + dimensions: dimension, element_ptype, }; return Ok( @@ -177,7 +177,7 @@ pub unsafe fn turboquant_encode_unchecked( let sorf_options = SorfOptions { seed, num_rounds: config.num_rounds, - dimension, + dimensions: dimension, element_ptype, }; Ok(SorfTransform::try_new_array(&sorf_options, padded_vector, num_rows)?.into_array()) @@ -213,7 +213,7 @@ fn turboquant_quantize_core( let elements_prim: PrimitiveArray = fsl.elements().clone().execute(ctx)?; let f32_elements = cast_to_f32(elements_prim)?; - let centroids = get_centroids(padded_dim_u32, bit_width)?; + let centroids = compute_or_get_centroids(padded_dim_u32, bit_width)?; let boundaries = compute_centroid_boundaries(¢roids); let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); diff --git a/vortex-tensor/src/encodings/turboquant/mod.rs b/vortex-tensor/src/encodings/turboquant/mod.rs index 463e8bc7b25..50cef7b721e 100644 --- a/vortex-tensor/src/encodings/turboquant/mod.rs +++ b/vortex-tensor/src/encodings/turboquant/mod.rs @@ -121,7 +121,7 @@ //! // Normalize and quantize at 2 bits per coordinate in one pass. //! let session = VortexSession::empty().with::(); //! let mut ctx = session.create_execution_ctx(); -//! let config = TurboQuantConfig { bit_width: 2, seed: Some(42), num_rounds: 3 }; +//! let config = TurboQuantConfig { bit_width: 2, seed: 42, num_rounds: 3 }; //! let tq = turboquant_encode(vector, &config, &mut ctx).unwrap(); //! //! // Verify compression: 100 vectors x 128 dims x 4 bytes = 51200 bytes input. diff --git a/vortex-tensor/src/encodings/turboquant/scheme.rs b/vortex-tensor/src/encodings/turboquant/scheme.rs index 19007664abe..d4362096bd2 100644 --- a/vortex-tensor/src/encodings/turboquant/scheme.rs +++ b/vortex-tensor/src/encodings/turboquant/scheme.rs @@ -111,21 +111,25 @@ impl Scheme for TurboQuantScheme { fn estimate_compression_ratio(element_bit_width: u8, dimensions: u32, num_vectors: usize) -> f64 { let config = TurboQuantConfig::default(); let padded_dim = dimensions.next_power_of_two() as usize; + let element_bits = usize::from(element_bit_width); - // Per-vector: MSE codes per padded coordinate, plus one stored norm in the input element - // float width. - let compressed_bits_per_vector = - usize::from(element_bit_width) + usize::from(config.bit_width) * padded_dim; + // Get the size of the fully uncompressed vector data. + let uncompressed_size_bits = element_bits * dimensions as usize * num_vectors; + + // Per-vector: MSE codes per padded coordinate, plus one stored norm in the input element float + // width. + let norm_bits = element_bits; + let compressed_bits_per_vector = usize::from(config.bit_width) * padded_dim; + let total_bits_per_vector = norm_bits + compressed_bits_per_vector; // Shared overhead: codebook centroids (2^bit_width f32 values). - // Note: rotation signs are no longer stored — rotation is deterministic from seed. let num_centroids = 1usize << config.bit_width; debug_assert!(num_centroids <= MAX_CENTROIDS); let overhead_bits = num_centroids * 32; // centroids are always f32 - let compressed_size_bits = compressed_bits_per_vector * num_vectors + overhead_bits; + // This includes the quantized vectors, norms, and centroid codebook. + let compressed_size_bits = total_bits_per_vector * num_vectors + overhead_bits; - let uncompressed_size_bits = usize::from(element_bit_width) * dimensions as usize * num_vectors; uncompressed_size_bits as f64 / compressed_size_bits as f64 } diff --git a/vortex-tensor/src/encodings/turboquant/tests/compute.rs b/vortex-tensor/src/encodings/turboquant/tests/compute.rs index a26f0c54cfc..4d670695eaf 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/compute.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/compute.rs @@ -40,7 +40,7 @@ fn slice_preserves_data() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 4, }; let mut ctx = SESSION.create_execution_ctx(); @@ -85,7 +85,7 @@ fn scalar_at_matches_decompress() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); @@ -108,7 +108,7 @@ fn l2_norm_readthrough() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 5, }; let mut ctx = SESSION.create_execution_ctx(); @@ -146,7 +146,7 @@ fn l2_norm_readthrough_is_authoritative_for_lossy_storage() -> VortexResult<()> let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 1, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -183,7 +183,7 @@ fn cosine_similarity_readthrough_is_authoritative_for_lossy_storage() -> VortexR let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 1, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs index 41124c27c80..e3febbaab7b 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/nullable.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/nullable.rs @@ -23,7 +23,7 @@ fn nullable_vectors_roundtrip() -> VortexResult<()> { let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 4, }; let mut ctx = SESSION.create_execution_ctx(); @@ -84,7 +84,7 @@ fn nullable_norms_match_validity() -> VortexResult<()> { let config = TurboQuantConfig { bit_width: 2, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -114,7 +114,7 @@ fn nullable_l2_norm_readthrough() -> VortexResult<()> { let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -156,7 +156,7 @@ fn nullable_slice_preserves_validity() -> VortexResult<()> { let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs index db013cdfd37..d82be3cf714 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/roundtrip.rs @@ -29,7 +29,7 @@ fn roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> VortexResult<()> { let fsl = make_fsl(10, dim, 42); let config = TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -49,7 +49,7 @@ fn mse_within_theoretical_bound(#[case] dim: usize, #[case] bit_width: u8) -> Vo let fsl = make_fsl(num_rows, dim, 42); let config = TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -76,7 +76,7 @@ fn high_bitwidth_mse_is_small(#[case] dim: usize, #[case] bit_width: u8) -> Vort let config_4bit = TurboQuantConfig { bit_width: 4, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original_4, decoded_4) = encode_decode(&fsl, &config_4bit)?; @@ -84,7 +84,7 @@ fn high_bitwidth_mse_is_small(#[case] dim: usize, #[case] bit_width: u8) -> Vort let config = TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -108,7 +108,7 @@ fn mse_decreases_with_bits() -> VortexResult<()> { for bit_width in 1..=8u8 { let config = TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -130,7 +130,7 @@ fn roundtrip_edge_cases(#[case] num_rows: usize) -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 2, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -159,7 +159,7 @@ fn rejects_dimension_below_128(#[case] dim: usize) { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 2, - seed: Some(0), + seed: 0, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -174,7 +174,7 @@ fn rejects_invalid_bit_width(#[case] bit_width: u8) { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width, - seed: Some(0), + seed: 0, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -203,7 +203,7 @@ fn all_zero_vectors_roundtrip() -> VortexResult<()> { let config = TurboQuantConfig { bit_width: 3, - seed: Some(42), + seed: 42, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -223,7 +223,7 @@ fn large_dimension_roundtrip(#[case] dim: usize, #[case] bit_width: u8) -> Vorte let fsl = make_fsl(num_rows, dim, 42); let config = TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, }; let (original, decoded) = encode_decode(&fsl, &config)?; @@ -262,7 +262,7 @@ fn f64_input_encodes_successfully() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(42), + seed: 42, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -295,7 +295,7 @@ fn f16_input_encodes_successfully() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(42), + seed: 42, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-tensor/src/encodings/turboquant/tests/structural.rs b/vortex-tensor/src/encodings/turboquant/tests/structural.rs index 193d340285a..3913cf3d8fe 100644 --- a/vortex-tensor/src/encodings/turboquant/tests/structural.rs +++ b/vortex-tensor/src/encodings/turboquant/tests/structural.rs @@ -13,14 +13,14 @@ use vortex_error::VortexResult; use super::*; -/// Verify that the centroids stored in the DictArray match what `get_centroids()` computes. +/// Verify that the centroids stored in the DictArray match what `compute_or_get_centroids()` computes. #[test] fn stored_centroids_match_computed() -> VortexResult<()> { let fsl = make_fsl(10, 128, 42); let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -30,7 +30,7 @@ fn stored_centroids_match_computed() -> VortexResult<()> { let stored = centroids.as_slice::(); // padded_dim for dim=128 is 128. - let computed = crate::encodings::turboquant::centroids::get_centroids(128, 3)?; + let computed = crate::encodings::turboquant::centroids::compute_or_get_centroids(128, 3)?; assert_eq!(stored.len(), computed.len()); for i in 0..stored.len() { @@ -46,7 +46,7 @@ fn seed_deterministic_rotation_produces_correct_decode() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 4, }; @@ -90,7 +90,7 @@ fn encoded_dtype_is_vector_extension() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 3, - seed: Some(123), + seed: 123, num_rounds: 2, }; let mut ctx = SESSION.create_execution_ctx(); @@ -115,7 +115,7 @@ fn cosine_similarity_quantized_accuracy() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 4, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -172,7 +172,7 @@ fn dot_product_quantized_accuracy() -> VortexResult<()> { let ext = make_vector_ext(&fsl); let config = TurboQuantConfig { bit_width: 8, - seed: Some(123), + seed: 123, num_rounds: 3, }; let mut ctx = SESSION.create_execution_ctx(); @@ -231,8 +231,8 @@ fn sorf_transform_roundtrip_isolation() -> VortexResult<()> { use vortex_buffer::BufferMut; use crate::encodings::turboquant::centroids::compute_centroid_boundaries; + use crate::encodings::turboquant::centroids::compute_or_get_centroids; use crate::encodings::turboquant::centroids::find_nearest_centroid; - use crate::encodings::turboquant::centroids::get_centroids; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfOptions; use crate::scalar_fns::sorf_transform::SorfTransform; @@ -261,7 +261,7 @@ fn sorf_transform_roundtrip_isolation() -> VortexResult<()> { // Forward transform + quantize (mimicking what turboquant_quantize_core does). let rotation = SorfMatrix::try_new(seed, dim, num_rounds as usize)?; let padded_dim = rotation.padded_dim(); - let centroids = get_centroids(padded_dim as u32, 8)?; + let centroids = compute_or_get_centroids(padded_dim as u32, 8)?; let boundaries = compute_centroid_boundaries(¢roids); let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); @@ -299,7 +299,7 @@ fn sorf_transform_roundtrip_isolation() -> VortexResult<()> { let sorf_options = SorfOptions { seed, num_rounds, - dimension: dim as u32, + dimensions: dim as u32, element_ptype: vortex_array::dtype::PType::F32, }; let sorf_array = diff --git a/vortex-tensor/src/lib.rs b/vortex-tensor/src/lib.rs index c748bdd9f43..7beadc02e93 100644 --- a/vortex-tensor/src/lib.rs +++ b/vortex-tensor/src/lib.rs @@ -5,6 +5,11 @@ //! including unit vectors, spherical coordinates, and similarity measures such as cosine //! similarity. +#![cfg_attr( + test, + allow(clippy::unwrap_used, clippy::expect_used, clippy::unwrap_in_result) +)] + use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayPlugin; use vortex_array::dtype::session::DTypeSessionExt; use vortex_array::scalar_fn::session::ScalarFnSessionExt; diff --git a/vortex-tensor/src/scalar_fns/cosine_similarity.rs b/vortex-tensor/src/scalar_fns/cosine_similarity.rs index b39b010082c..7819e0b46f0 100644 --- a/vortex-tensor/src/scalar_fns/cosine_similarity.rs +++ b/vortex-tensor/src/scalar_fns/cosine_similarity.rs @@ -33,11 +33,11 @@ use vortex_buffer::Buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; -use crate::scalar_fns::inner_product::BinaryTensorOpMetadata; use crate::scalar_fns::inner_product::InnerProduct; use crate::scalar_fns::l2_denorm::DenormOrientation; use crate::scalar_fns::l2_denorm::try_build_constant_l2_denorm; use crate::scalar_fns::l2_norm::L2Norm; +use crate::utils::BinaryTensorOpMetadata; use crate::utils::extract_l2_denorm_children; use crate::utils::validate_binary_tensor_float_inputs; @@ -115,7 +115,7 @@ impl ScalarFnVTable for CosineSimilarity { let lhs = &arg_dtypes[0]; let rhs = &arg_dtypes[1]; - let tensor_match = validate_binary_tensor_float_inputs("CosineSimilarity", lhs, rhs)?; + let tensor_match = validate_binary_tensor_float_inputs(lhs, rhs)?; let ptype = tensor_match.element_ptype(); let nullability = Nullability::from(lhs.is_nullable() || rhs.is_nullable()); Ok(DType::Primitive(ptype, nullability)) @@ -227,13 +227,8 @@ impl ScalarFnArrayVTable for CosineSimilarity { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult> { - let reconstructed = BinaryTensorOpMetadata::decode_children( - metadata, - len, - children, - session, - "CosineSimilarity", - )?; + let reconstructed = + BinaryTensorOpMetadata::decode_children(metadata, len, children, session)?; Ok(ScalarFnArrayParts { options: EmptyOptions, children: reconstructed, diff --git a/vortex-tensor/src/scalar_fns/inner_product.rs b/vortex-tensor/src/scalar_fns/inner_product.rs index 5b04f2b0bd6..d60938dfbd8 100644 --- a/vortex-tensor/src/scalar_fns/inner_product.rs +++ b/vortex-tensor/src/scalar_fns/inner_product.rs @@ -6,7 +6,6 @@ use std::fmt::Formatter; use num_traits::Float; -use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -17,13 +16,11 @@ use vortex_array::arrays::Extension; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeList; use vortex_array::arrays::PrimitiveArray; -use vortex_array::arrays::ScalarFn as ScalarFnArrayEncoding; use vortex_array::arrays::ScalarFnArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; -use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; @@ -31,7 +28,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::expr::and; use vortex_array::match_each_float_ptype; @@ -47,8 +43,6 @@ use vortex_buffer::Buffer; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_ensure; -use vortex_error::vortex_err; use vortex_session::VortexSession; use crate::matcher::AnyTensor; @@ -56,6 +50,7 @@ use crate::scalar_fns::l2_denorm::DenormOrientation; use crate::scalar_fns::sorf_transform::SorfMatrix; use crate::scalar_fns::sorf_transform::SorfTransform; use crate::types::vector::Vector; +use crate::utils::BinaryTensorOpMetadata; use crate::utils::extract_constant_flat_row; use crate::utils::extract_flat_elements; use crate::utils::extract_l2_denorm_children; @@ -130,7 +125,7 @@ impl ScalarFnVTable for InnerProduct { let rhs = &arg_dtypes[1]; // TODO(connor): relax the float-only gate once integer tensors are supported. - let tensor_match = validate_binary_tensor_float_inputs("InnerProduct", lhs, rhs)?; + let tensor_match = validate_binary_tensor_float_inputs(lhs, rhs)?; let ptype = tensor_match.element_ptype(); let nullability = Nullability::from(lhs.is_nullable() || rhs.is_nullable()); Ok(DType::Primitive(ptype, nullability)) @@ -225,66 +220,6 @@ impl ScalarFnVTable for InnerProduct { } } -/// Metadata for a serialized binary tensor-op array (shared by [`InnerProduct`] and -/// [`CosineSimilarity`]). Both operands share the same extension dtype up to nullability -/// (enforced by their `return_dtype` checks), but their individual nullabilities are lost in the -/// parent's unioned output, so both are persisted. -/// -/// [`CosineSimilarity`]: crate::scalar_fns::cosine_similarity::CosineSimilarity -#[derive(Clone, prost::Message)] -pub(crate) struct BinaryTensorOpMetadata { - #[prost(message, optional, tag = "1")] - pub(crate) lhs_dtype: Option, - #[prost(message, optional, tag = "2")] - pub(crate) rhs_dtype: Option, -} - -impl BinaryTensorOpMetadata { - /// Encodes the two children of `view` into a [`BinaryTensorOpMetadata`] byte blob. - pub(crate) fn encode_from_view( - view: &ScalarFnArrayView, - ) -> VortexResult> { - let scalar_fn_array = view.as_::(); - let lhs_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); - let rhs_dtype = Some(scalar_fn_array.child_at(1).dtype().try_into()?); - Ok(Self { - lhs_dtype, - rhs_dtype, - } - .encode_to_vec()) - } - - /// Decodes `metadata` and fetches both children from `children` using the decoded dtypes, - /// validating that `lhs` and `rhs` agree modulo nullability. - pub(crate) fn decode_children( - metadata: &[u8], - len: usize, - children: &dyn ArrayChildren, - session: &VortexSession, - scalar_fn_name: &str, - ) -> VortexResult> { - let metadata = Self::decode(metadata) - .map_err(|e| vortex_err!("Failed to decode BinaryTensorOpMetadata: {e}"))?; - let lhs_pb = metadata - .lhs_dtype - .as_ref() - .ok_or_else(|| vortex_err!("{scalar_fn_name} metadata missing lhs_dtype"))?; - let rhs_pb = metadata - .rhs_dtype - .as_ref() - .ok_or_else(|| vortex_err!("{scalar_fn_name} metadata missing rhs_dtype"))?; - let lhs_dtype = DType::from_proto(lhs_pb, session)?; - let rhs_dtype = DType::from_proto(rhs_pb, session)?; - vortex_ensure!( - lhs_dtype.eq_ignore_nullability(&rhs_dtype), - "{scalar_fn_name} operand dtype mismatch: {lhs_dtype} vs {rhs_dtype}" - ); - let lhs = children.get(0, &lhs_dtype, len)?; - let rhs = children.get(1, &rhs_dtype, len)?; - Ok(vec![lhs, rhs]) - } -} - impl ScalarFnArrayVTable for InnerProduct { fn serialize( &self, @@ -302,13 +237,8 @@ impl ScalarFnArrayVTable for InnerProduct { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult> { - let reconstructed = BinaryTensorOpMetadata::decode_children( - metadata, - len, - children, - session, - "InnerProduct", - )?; + let reconstructed = + BinaryTensorOpMetadata::decode_children(metadata, len, children, session)?; Ok(ScalarFnArrayParts { options: EmptyOptions, children: reconstructed, @@ -421,7 +351,7 @@ impl InnerProduct { return Ok(None); }; - let dim = sorf_view.options.dimension as usize; + let dim = sorf_view.options.dimensions as usize; let num_rounds = sorf_view.options.num_rounds as usize; let seed = sorf_view.options.seed; let padded_dim = dim.next_power_of_two(); @@ -980,7 +910,7 @@ mod tests { let sorf_options = SorfOptions { seed, num_rounds, - dimension: dim, + dimensions: dim, element_ptype: PType::F32, }; let sorf = @@ -1556,7 +1486,7 @@ mod tests { let sorf_options = SorfOptions { seed, num_rounds, - dimension: dim, + dimensions: dim, element_ptype: PType::F32, }; let sorf = diff --git a/vortex-tensor/src/scalar_fns/l2_denorm.rs b/vortex-tensor/src/scalar_fns/l2_denorm.rs index 9da97959075..1bdd81833d9 100644 --- a/vortex-tensor/src/scalar_fns/l2_denorm.rs +++ b/vortex-tensor/src/scalar_fns/l2_denorm.rs @@ -31,7 +31,6 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; -use vortex_array::dtype::PType; use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::expr::and; @@ -62,6 +61,7 @@ use crate::matcher::AnyTensor; use crate::scalar_fns::l2_norm::L2Norm; use crate::utils::extract_constant_flat_row; use crate::utils::extract_flat_elements; +use crate::utils::unit_norm_tolerance; use crate::utils::validate_tensor_float_input; /// Re-applies authoritative L2 norms to a normalized tensor column. @@ -360,7 +360,21 @@ fn execute_l2_denorm_constant_norms( .vortex_expect("we know that this is a float, so it must fit in f64") - 1.0f64; - let tolerance = unit_norm_tolerance(norm_scalar.dtype().as_ptype()); + let tensor_match = normalized_ref + .dtype() + .as_extension_opt() + .and_then(|ext| ext.metadata_opt::()) + .ok_or_else(|| { + vortex_err!( + "L2Denorm normalized child must be a tensor-like extension, got {}", + normalized_ref.dtype(), + ) + })?; + + let tolerance = unit_norm_tolerance( + norm_scalar.dtype().as_ptype(), + tensor_match.list_size() as usize, + ); if err.abs() < tolerance { return Ok(normalized_ref); } @@ -594,16 +608,6 @@ fn build_tensor_array( Ok(ExtensionArray::new(dtype.as_extension().clone(), storage.into_array()).into_array()) } -/// Returns the acceptable unit-norm drift for the given element precision. -fn unit_norm_tolerance(element_ptype: PType) -> f64 { - match element_ptype { - PType::F16 => 2e-3, - PType::F32 => 2e-6, - PType::F64 => 1e-10, - _ => unreachable!("L2Denorm requires float elements, got {element_ptype:?}"), - } -} - /// Validates that `normalized` and (when supplied) the matching `norms` jointly satisfy the /// [`L2Denorm`] invariants: /// @@ -623,8 +627,8 @@ pub fn validate_l2_normalized_rows_against_norms( let tensor_match = validate_tensor_float_input(normalized.dtype())?; let element_ptype = tensor_match.element_ptype(); - let tolerance = unit_norm_tolerance(element_ptype); let tensor_flat_size = tensor_match.list_size() as usize; + let tolerance = unit_norm_tolerance(element_ptype, tensor_flat_size); if let Some(norms) = norms { vortex_ensure_eq!( diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs index c974d9a9f09..26d38e87a1e 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/mod.rs @@ -81,7 +81,7 @@ pub struct SorfOptions { pub num_rounds: u8, /// Original vector dimension (before power-of-2 padding). The output /// [`Vector`](crate::vector::Vector) has this dimension. - pub dimension: u32, + pub dimensions: u32, /// Element type of the output [`Vector`](crate::vector::Vector). The child input must always /// be `f32`, but the output can be any float type (`F16`, `F32`, `F64`); the final /// `f32 -> element_ptype` cast happens while building the output. @@ -137,7 +137,7 @@ impl fmt::Display for SorfOptions { write!( f, "SorfOptions(seed={}, rounds={}, dim={}, ptype={})", - self.seed, self.num_rounds, self.dimension, self.element_ptype + self.seed, self.num_rounds, self.dimensions, self.element_ptype ) } } diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs b/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs index 9279a35259d..ff8aebd0f11 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/rotation.rs @@ -76,15 +76,15 @@ impl SorfMatrix { /// The seed is expanded using Vortex's frozen local SplitMix64 stream. Signs are generated in /// round-major, block-major order, with each `u64` contributing 64 sign bits in /// least-significant-bit-first order. - pub fn try_new(seed: u64, dimension: usize, num_rounds: usize) -> VortexResult { + pub fn try_new(seed: u64, dimensions: usize, num_rounds: usize) -> VortexResult { vortex_ensure!(num_rounds >= 1, "num_rounds must be >= 1, got {num_rounds}"); - let padded_dim = dimension.next_power_of_two(); + let padded_dim = dimensions.next_power_of_two(); let sign_masks = gen_sign_masks_from_seed(seed, padded_dim, num_rounds); // Compute in f64 for precision, then store as f32 since the WHT operates on f32 buffers. // The result is always in (0, 1] for any valid padded_dim >= 2 and num_rounds >= 1, so - // the f64 -> f32 cast is a precision loss only -- it cannot overflow to infinity. + // the f64 -> f32 cast is a precision loss only (it cannot overflow to infinity). #[expect( clippy::cast_possible_truncation, reason = "the norm factor is in (0, 1] so the f64 -> f32 cast cannot overflow" diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs index 4bc93871594..46abc66db71 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/tests.rs @@ -33,8 +33,8 @@ use super::SorfOptions; use super::SorfTransform; use super::rotation::SorfMatrix; use crate::encodings::turboquant::centroids::compute_centroid_boundaries; +use crate::encodings::turboquant::centroids::compute_or_get_centroids; use crate::encodings::turboquant::centroids::find_nearest_centroid; -use crate::encodings::turboquant::centroids::get_centroids; use crate::tests::SESSION; use crate::types::vector::Vector; @@ -67,7 +67,7 @@ fn forward_rotate_and_quantize( let rotation = SorfMatrix::try_new(seed, dim, num_rounds)?; let padded_dim = rotation.padded_dim(); - let centroids = get_centroids(padded_dim as u32, bit_width)?; + let centroids = compute_or_get_centroids(padded_dim as u32, bit_width)?; let boundaries = compute_centroid_boundaries(¢roids); let mut all_indices = BufferMut::::with_capacity(num_rows * padded_dim); @@ -115,7 +115,7 @@ fn default_options(dim: u32, seed: u64) -> SorfOptions { SorfOptions { seed, num_rounds: 3, - dimension: dim, + dimensions: dim, element_ptype: PType::F32, } } @@ -319,7 +319,7 @@ fn rejects_zero_rounds_at_construction() { let options = SorfOptions { seed: 42, num_rounds: 0, - dimension: 128, + dimensions: 128, element_ptype: PType::F32, }; let elements = PrimitiveArray::from_iter([0.0f32; 128]).into_array(); @@ -336,7 +336,7 @@ fn rejects_non_float_output_ptype_at_construction() { let options = SorfOptions { seed: 42, num_rounds: 3, - dimension: 128, + dimensions: 128, element_ptype: PType::U8, }; let elements = PrimitiveArray::from_iter([0.0f32; 128]).into_array(); @@ -400,7 +400,7 @@ fn f16_output_type() -> VortexResult<()> { let options = SorfOptions { seed, num_rounds: 3, - dimension: dim as u32, + dimensions: dim as u32, element_ptype: PType::F16, }; let sorf = SorfTransform::try_new_array(&options, padded_vector.into_array(), num_rows)?; @@ -425,7 +425,7 @@ fn f64_output_type() -> VortexResult<()> { let options = SorfOptions { seed, num_rounds: 3, - dimension: dim as u32, + dimensions: dim as u32, element_ptype: PType::F64, }; let sorf = SorfTransform::try_new_array(&options, padded_vector.into_array(), num_rows)?; @@ -461,13 +461,13 @@ fn trivial_padded_vector(padded_dim: u32, num_rows: usize, validity: Validity) - #[case::non_power_of_two_dim(100, Validity::NonNullable)] // Nullable top-level Vector to verify child nullability is reconstructed from the parent output. #[case::nullable_child(100, Validity::AllValid)] -fn serde_round_trip(#[case] dimension: u32, #[case] validity: Validity) -> VortexResult<()> { - let padded_dim = dimension.next_power_of_two(); +fn serde_round_trip(#[case] dimensions: u32, #[case] validity: Validity) -> VortexResult<()> { + let padded_dim = dimensions.next_power_of_two(); let num_rows = 4; let options = SorfOptions { seed: 42, num_rounds: 3, - dimension, + dimensions, element_ptype: PType::F32, }; let child = trivial_padded_vector(padded_dim, num_rows, validity); diff --git a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs index 81e17f665fa..827f8e6a796 100644 --- a/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs +++ b/vortex-tensor/src/scalar_fns/sorf_transform/vtable.rs @@ -16,8 +16,10 @@ use vortex_array::IntoArray; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn as ScalarFnArrayEncoding; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayParts; use vortex_array::arrays::scalar_fn::plugin::ScalarFnArrayVTable; @@ -26,6 +28,7 @@ use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::extension::ExtDType; +use vortex_array::dtype::proto::dtype as pb; use vortex_array::expr::Expression; use vortex_array::extension::EmptyMetadata; use vortex_array::match_each_float_ptype; @@ -90,13 +93,13 @@ impl ScalarFnVTable for SorfTransform { vortex_err!("SorfTransform child must be a Vector extension, got {child_dtype}") })?; - let expected_padded = options.dimension.next_power_of_two(); + let expected_padded = options.dimensions.next_power_of_two(); vortex_ensure_eq!( vector_metadata.dimensions(), expected_padded, "SorfTransform child Vector must have dimension {expected_padded} (next power of two \ for dimension {})", - options.dimension, + options.dimensions, ); // For now, the child Vector storage must be f32. TurboQuant stores its centroids as f32, @@ -113,11 +116,13 @@ impl ScalarFnVTable for SorfTransform { let output_elem_dtype = DType::Primitive(options.element_ptype, Nullability::NonNullable); let storage_dtype = DType::FixedSizeList( Arc::new(output_elem_dtype), - options.dimension, + options.dimensions, child_dtype.nullability(), ); + let _ = vector_metadata; let ext_dtype = ExtDType::::try_new(EmptyMetadata, storage_dtype)?.erased(); + Ok(DType::Extension(ext_dtype)) } @@ -127,19 +132,18 @@ impl ScalarFnVTable for SorfTransform { args: &dyn ExecutionArgs, ctx: &mut ExecutionCtx, ) -> VortexResult { - validate_sorf_options(options)?; - let dim = options.dimension as usize; + let dim = options.dimensions as usize; let num_rows = args.row_count(); if num_rows == 0 { - let child_nullability = args.get(0)?.dtype().nullability(); - let validity = Validity::from(child_nullability); + let child_dtype = args.get(0)?.dtype().clone(); + let validity = Validity::from(child_dtype.nullability()); return match_each_float_ptype!(options.element_ptype, |T| { let elements = PrimitiveArray::empty::(Nullability::NonNullable); let fsl = FixedSizeListArray::try_new( elements.into_array(), - options.dimension, + options.dimensions, validity, 0, )?; @@ -194,11 +198,9 @@ impl ScalarFnVTable for SorfTransform { /// Metadata for a serialized [`SorfTransform`] array. /// -/// Stores the full [`SorfOptions`] inline. The child [`DType`] is not serialized because it is -/// fully determined by the options: the child is always a [`Vector`] extension wrapping -/// `FSL`. The child's nullability is recovered from the -/// parent output dtype at deserialize time, since `SorfTransform::return_dtype` propagates child -/// nullability into the output FSL (see `return_dtype` above). +/// Stores the full [`SorfOptions`] inline along with the child [`DType`]. Older metadata omitted +/// this field; deserialization derives the legacy plain-`Vector` child dtype from the parent dtype +/// in that case. #[derive(Clone, prost::Message)] pub(super) struct SorfTransformMetadata { #[prost(uint64, tag = "1")] @@ -210,6 +212,8 @@ pub(super) struct SorfTransformMetadata { dimension: u32, #[prost(enumeration = "PType", tag = "4")] element_ptype: i32, + #[prost(message, optional, tag = "5")] + child_dtype: Option, } impl ScalarFnArrayVTable for SorfTransform { @@ -218,9 +222,13 @@ impl ScalarFnArrayVTable for SorfTransform { view: &ScalarFnArrayView, _session: &VortexSession, ) -> VortexResult>> { - Ok(Some( - SorfTransformMetadata::from(view.options).encode_to_vec(), - )) + let scalar_fn_array = view.as_::(); + let child_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); + let metadata = SorfTransformMetadata { + child_dtype, + ..SorfTransformMetadata::from(view.options) + }; + Ok(Some(metadata.encode_to_vec())) } fn deserialize( @@ -229,11 +237,11 @@ impl ScalarFnArrayVTable for SorfTransform { len: usize, metadata: &[u8], children: &dyn ArrayChildren, - _session: &VortexSession, + session: &VortexSession, ) -> VortexResult> { - let options = SorfTransformMetadata::decode(metadata) - .map_err(|e| vortex_err!("Failed to decode SorfTransformMetadata: {e}"))? - .to_options()?; + let metadata = SorfTransformMetadata::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode SorfTransformMetadata: {e}"))?; + let options = metadata.to_options()?; // `return_dtype` sets the output FSL's nullability to the child's nullability (see // `return_dtype` above), so we read the child nullability back from the parent dtype. @@ -244,14 +252,19 @@ impl ScalarFnArrayVTable for SorfTransform { })? .storage_dtype() .nullability(); - let padded_dim = options.dimension.next_power_of_two(); + let padded_dim = options.dimensions.next_power_of_two(); let child_storage = DType::FixedSizeList( Arc::new(DType::Primitive(PType::F32, Nullability::NonNullable)), padded_dim, child_nullability, ); - let child_ext = ExtDType::::try_new(EmptyMetadata, child_storage)?.erased(); - let child_dtype = DType::Extension(child_ext); + let child_dtype = match metadata.child_dtype.as_ref() { + Some(dtype) => DType::from_proto(dtype, session)?, + None => { + let child_ext = ExtDType::::try_new(EmptyMetadata, child_storage)?.erased(); + DType::Extension(child_ext) + } + }; let child = children.get(0, &child_dtype, len)?; Ok(ScalarFnArrayParts { @@ -270,7 +283,7 @@ fn float_from_f32(v: f32) -> T { } /// Apply the inverse SORF transform on f32 data, truncate to the original dimension, cast each -/// element to `T`, and build the output [`Vector`] extension array. +/// element to `T`, and build a plain [`Vector`](crate::vector::Vector) extension array. fn inverse_rotate_typed( f32_elements: &[f32], rotation: &SorfMatrix, @@ -304,8 +317,9 @@ impl From<&SorfOptions> for SorfTransformMetadata { Self { seed: options.seed, num_rounds: u32::from(options.num_rounds), - dimension: options.dimension, + dimension: options.dimensions, element_ptype: options.element_ptype as i32, + child_dtype: None, } } } @@ -323,7 +337,7 @@ impl SorfTransformMetadata { let options = SorfOptions { seed: self.seed, num_rounds, - dimension: self.dimension, + dimensions: self.dimension, element_ptype: self.element_ptype(), }; validate_sorf_options(&options)?; diff --git a/vortex-tensor/src/types/vector/mod.rs b/vortex-tensor/src/types/vector/mod.rs index d077a183713..3763220fc82 100644 --- a/vortex-tensor/src/types/vector/mod.rs +++ b/vortex-tensor/src/types/vector/mod.rs @@ -14,6 +14,30 @@ use vortex_array::extension::EmptyMetadata; use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; + +/// Validates that `storage` is a valid storage dtype for a [`Vector`] or +/// [`NormalizedVector`](crate::normalized_vector::NormalizedVector) extension type. +/// +/// The storage must be a `FixedSizeList` with non-nullable float +/// elements. The outer nullability is not constrained. +pub(crate) fn validate_vector_storage_dtype(storage: &DType) -> VortexResult<()> { + let DType::FixedSizeList(element_dtype, _list_size, _nullability) = storage else { + vortex_bail!("Vector storage dtype must be a FixedSizeList, got {storage}"); + }; + + vortex_ensure!( + element_dtype.is_float(), + "Vector element dtype must be a float, got {element_dtype}" + ); + vortex_ensure!( + !element_dtype.is_nullable(), + "Vector element dtype must be non-nullable" + ); + + Ok(()) +} /// The Vector extension type. #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] diff --git a/vortex-tensor/src/types/vector/vtable.rs b/vortex-tensor/src/types/vector/vtable.rs index a8b3c10994a..c80f17665f2 100644 --- a/vortex-tensor/src/types/vector/vtable.rs +++ b/vortex-tensor/src/types/vector/vtable.rs @@ -8,10 +8,9 @@ use vortex_array::dtype::extension::ExtVTable; use vortex_array::extension::EmptyMetadata; use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; -use vortex_error::vortex_bail; -use vortex_error::vortex_ensure; use crate::types::vector::Vector; +use crate::types::vector::validate_vector_storage_dtype; impl ExtVTable for Vector { type Metadata = EmptyMetadata; @@ -46,21 +45,7 @@ impl ExtVTable for Vector { } fn validate_dtype(ext_dtype: &ExtDType) -> VortexResult<()> { - let storage_dtype = ext_dtype.storage_dtype(); - let DType::FixedSizeList(element_dtype, _list_size, _nullability) = storage_dtype else { - vortex_bail!("Vector storage dtype must be a FixedSizeList, got {storage_dtype}"); - }; - - vortex_ensure!( - element_dtype.is_float(), - "Vector element dtype must be a float, got {element_dtype}" - ); - vortex_ensure!( - !element_dtype.is_nullable(), - "Vector element dtype must be non-nullable" - ); - - Ok(()) + validate_vector_storage_dtype(ext_dtype.storage_dtype()) } fn unpack_native<'a>( diff --git a/vortex-tensor/src/utils.rs b/vortex-tensor/src/utils.rs index 97bc91d3759..9dc097e11e0 100644 --- a/vortex-tensor/src/utils.rs +++ b/vortex-tensor/src/utils.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use half::f16; +use prost::Message; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -8,23 +10,70 @@ use vortex_array::arrays::Constant; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::ScalarFn; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex_array::arrays::primitive::PrimitiveArrayExt; use vortex_array::arrays::scalar_fn::ExactScalarFn; +use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; +use vortex_array::arrays::scalar_fn::ScalarFnArrayView; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::PType; +use vortex_array::dtype::proto::dtype as pb; +use vortex_array::scalar_fn::ScalarFnVTable; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; +use vortex_session::VortexSession; use crate::matcher::AnyTensor; use crate::matcher::TensorMatch; use crate::scalar_fns::l2_denorm::L2Denorm; +/// Safety factor for unit-norm tolerance. Applied as a constant multiplier on the probabilistic +/// `√d · ε` bound so that legitimate round-off noise clears the check with headroom. +pub(crate) const SAFETY_FACTOR: usize = 10; + +/// Returns the acceptable unit-norm drift for the given element precision and dimension count. +/// +/// Uses the `c · √d · ε` bound where ε is machine epsilon and d is the vector dimension. Under +/// IEEE 754 round-to-nearest the probabilistic (RMS-case) forward error for computing ‖x‖₂ grows +/// as `O(√d · ε)` rather than the worst-case `O(d · ε)` from the classical Wilkinson bound, +/// assuming near-independent rounding errors across the d-term summation. +/// +/// Reference: Croci, Fasi, Higham, Mary, Mikaitis (2022). "Stochastic rounding: implementation, +/// error analysis and applications." Royal Society Open Science, 9: 211631, §6.1 "Probabilistic +/// error analysis." https://doi.org/10.1098/rsos.211631 +pub fn unit_norm_tolerance(element_ptype: PType, dimensions: usize) -> f64 { + let machine_epsilon: f64 = match element_ptype { + PType::F64 => f64::EPSILON, + PType::F32 => f32::EPSILON as f64, + PType::F16 => f16::EPSILON.to_f64_const(), + _ => unreachable!("unit_norm_tolerance requires a float ptype, got {element_ptype:?}"), + }; + + let dimensions_root = (dimensions as f64).sqrt(); + + SAFETY_FACTOR as f64 * machine_epsilon * dimensions_root +} + +/// Extracts the `(normalized, norms)` children from an [`L2Denorm`] scalar function array. +/// +/// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm +pub fn extract_l2_denorm_children(array: &ArrayRef) -> (ArrayRef, ArrayRef) { + let sfn = array + .as_opt::>() + .vortex_expect("expected ScalarFnArray wrapping L2Denorm"); + ( + sfn.nth_child(0) + .vortex_expect("L2Denorm missing normalized array"), + sfn.nth_child(1).vortex_expect("L2Denorm missing norms"), + ) +} + /// Validates that `input_dtype` is a float-valued tensor-like extension dtype. pub fn validate_tensor_float_input(input_dtype: &DType) -> VortexResult> { let ext = input_dtype @@ -46,17 +95,13 @@ pub fn validate_tensor_float_input(input_dtype: &DType) -> VortexResult( - op_name: &str, lhs: &'a DType, rhs: &DType, ) -> VortexResult> { vortex_ensure!( lhs.eq_ignore_nullability(rhs), - "{op_name} requires both inputs to have the same dtype, got {lhs} and {rhs}" + "binary tensor expression expects inputs to have the same dtype, got {lhs} and {rhs}" ); validate_tensor_float_input(lhs) } @@ -73,7 +118,7 @@ pub fn validate_binary_tensor_float_inputs<'a>( pub fn cast_to_f32(prim: PrimitiveArray) -> VortexResult> { match prim.ptype() { PType::F16 => Ok(prim - .as_slice::() + .as_slice::() .iter() .map(|&v| f32::from(v)) .collect()), @@ -209,18 +254,62 @@ pub fn extract_constant_flat_row( Ok(FlatRow { elems }) } -/// Extracts the `(normalized, norms)` children from an [`L2Denorm`] scalar function array. +/// Metadata for a serialized binary tensor-op array (shared by [`InnerProduct`] and +/// [`CosineSimilarity`]). Both operands share the same extension dtype up to nullability +/// (enforced by their `return_dtype` checks), but their individual nullabilities are lost in the +/// parent's unioned output, so both are persisted. /// -/// [`L2Denorm`]: crate::scalar_fns::l2_denorm::L2Denorm -pub fn extract_l2_denorm_children(array: &ArrayRef) -> (ArrayRef, ArrayRef) { - let sfn = array - .as_opt::>() - .vortex_expect("expected ScalarFnArray wrapping L2Denorm"); - ( - sfn.nth_child(0) - .vortex_expect("L2Denorm missing normalized array"), - sfn.nth_child(1).vortex_expect("L2Denorm missing norms"), - ) +/// [`CosineSimilarity`]: crate::scalar_fns::cosine_similarity::CosineSimilarity +#[derive(Clone, prost::Message)] +pub(crate) struct BinaryTensorOpMetadata { + #[prost(message, optional, tag = "1")] + pub(crate) lhs_dtype: Option, + #[prost(message, optional, tag = "2")] + pub(crate) rhs_dtype: Option, +} + +impl BinaryTensorOpMetadata { + /// Encodes the two children of `view` into a [`BinaryTensorOpMetadata`] byte blob. + pub(crate) fn encode_from_view( + view: &ScalarFnArrayView, + ) -> VortexResult> { + let scalar_fn_array = view.as_::(); + let lhs_dtype = Some(scalar_fn_array.child_at(0).dtype().try_into()?); + let rhs_dtype = Some(scalar_fn_array.child_at(1).dtype().try_into()?); + Ok(Self { + lhs_dtype, + rhs_dtype, + } + .encode_to_vec()) + } + + /// Decodes `metadata` and fetches both children from `children` using the decoded dtypes, + /// validating that `lhs` and `rhs` are compatible tensor operands. + pub(crate) fn decode_children( + metadata: &[u8], + len: usize, + children: &dyn vortex_array::serde::ArrayChildren, + session: &VortexSession, + ) -> VortexResult> { + let metadata = Self::decode(metadata) + .map_err(|e| vortex_err!("Failed to decode BinaryTensorOpMetadata: {e}"))?; + let lhs_pb = metadata + .lhs_dtype + .as_ref() + .ok_or_else(|| vortex_err!("metadata missing lhs_dtype"))?; + let rhs_pb = metadata + .rhs_dtype + .as_ref() + .ok_or_else(|| vortex_err!("metadata missing rhs_dtype"))?; + + let lhs_dtype = DType::from_proto(lhs_pb, session)?; + let rhs_dtype = DType::from_proto(rhs_pb, session)?; + validate_binary_tensor_float_inputs(&lhs_dtype, &rhs_dtype)?; + + let lhs = children.get(0, &lhs_dtype, len)?; + let rhs = children.get(1, &rhs_dtype, len)?; + Ok(vec![lhs, rhs]) + } } #[cfg(test)] @@ -241,7 +330,6 @@ pub mod test_helpers { use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; - use vortex_error::vortex_err; use crate::scalar_fns::l2_denorm::L2Denorm; use crate::types::fixed_shape::FixedShapeTensor; @@ -271,12 +359,7 @@ pub mod test_helpers { /// The number of rows is inferred from the total element count divided by the product of the /// shape dimensions. For 0-dimensional tensors (scalar), each element is one row. pub fn tensor_array(shape: &[usize], elements: &[T]) -> VortexResult { - let list_size: u32 = shape - .iter() - .product::() - .max(1) - .try_into() - .map_err(|e| vortex_err!("{e}"))?; + let list_size: u32 = shape.iter().product::().max(1).try_into().unwrap(); let storage = flat_fsl(elements, list_size); let metadata = FixedShapeTensorMetadata::new(shape.to_vec()); let ext_dtype = diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index e0caed0d84f..be253187956 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -544,7 +544,7 @@ mod turboquant_benches { fn turboquant_config(bit_width: u8) -> TurboQuantConfig { TurboQuantConfig { bit_width, - seed: Some(123), + seed: 123, num_rounds: 3, } } From aece3ad446fc4eda05d1b84bad916b882117c1bc Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 23 Apr 2026 11:53:54 -0400 Subject: [PATCH 177/250] Make java spark tests more resilient (#7612) Depending on your network spark's automatic detection of address to bind might fail therefore we fix it to localhost for tests Signed-off-by: Robert Kruszewski --- .../spark/VortexDataSourceS3MockTest.java | 1 + .../spark/VortexDataSourceWriteTest.java | 43 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java index 04c94546013..be3d2e5da59 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java @@ -45,6 +45,7 @@ public void setUp() { .master("local[2]") .config("spark.sql.shuffle.partitions", "2") .config("spark.sql.adaptive.enabled", "false") + .config("spark.driver.host", "127.0.0.1") .config("spark.ui.enabled", "false") // S3A configuration for S3Mock. // This should be propagated into our reader diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java index fa2833af0da..8d2f64e3bdb 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceWriteTest.java @@ -3,9 +3,7 @@ package dev.vortex.spark; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; import java.nio.file.Files; @@ -45,6 +43,7 @@ public void setUp() { spark = SparkSession.builder() .appName("VortexWriteTest") .master("local[2]") // Use 2 threads + .config("spark.driver.host", "127.0.0.1") .config("spark.sql.shuffle.partitions", "2") .config("spark.sql.adaptive.enabled", "false") // Disable AQE for predictable partitioning .config("spark.ui.enabled", "false") // Disable UI for tests @@ -213,8 +212,8 @@ public void testPartitionedWrite() throws IOException { // Verify vortex files inside partition directories List filesA = findVortexFiles(outputPath.resolve("group=A")); List filesB = findVortexFiles(outputPath.resolve("group=B")); - assertTrue(!filesA.isEmpty(), "Partition A should have vortex files"); - assertTrue(!filesB.isEmpty(), "Partition B should have vortex files"); + assertFalse(filesA.isEmpty(), "Partition A should have vortex files"); + assertFalse(filesB.isEmpty(), "Partition B should have vortex files"); // When: read back Dataset readDf = spark.read() @@ -333,15 +332,15 @@ public void testWriteAndReadTemporalAndStructColumns() throws IOException { "cast(id as int) as id", "CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END AS event_date", """ - CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) - ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END AS event_ts""", + CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END AS event_ts""", """ - named_struct( - 'event_date', CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END, - 'event_ts', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) - ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, - 'label', CASE WHEN id = 0 THEN 'alpha' ELSE 'beta' END - ) AS payload"""); + named_struct( + 'event_date', CASE WHEN id = 0 THEN CAST('2024-01-02' AS DATE) ELSE CAST('2024-02-03' AS DATE) END, + 'event_ts', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, + 'label', CASE WHEN id = 0 THEN 'alpha' ELSE 'beta' END + ) AS payload"""); Path outputPath = tempDir.resolve("temporal_struct_output"); originalDf @@ -366,7 +365,7 @@ ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, assertEquals(DataTypes.DateType, readDf.schema().fields()[1].dataType()); assertEquals(DataTypes.TimestampType, readDf.schema().fields()[2].dataType()); - assertTrue(readDf.schema().fields()[3].dataType() instanceof StructType); + assertInstanceOf(StructType.class, readDf.schema().fields()[3].dataType()); assertEquals(expectedRows, projectTemporalAndStructRows(readDf)); } @@ -374,12 +373,12 @@ ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP) END, @DisplayName("Write TimestampNTZ columns and nested structs") public void testWriteTimestampNtzColumns() throws IOException { Dataset timestampNtzDf = spark.range(0, 2).selectExpr("cast(id as int) as id", """ - CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) - ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz""", """ - named_struct( - 'event_ntz', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) - ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END - ) AS payload"""); + CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) + ELSE CAST(NULL AS TIMESTAMP_NTZ) END AS event_ntz""", """ + named_struct( + 'event_ntz', CASE WHEN id = 0 THEN CAST('2024-01-02 03:04:05.123456' AS TIMESTAMP_NTZ) + ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END + ) AS payload"""); Path outputPath = tempDir.resolve("timestamp_ntz_output"); assertDoesNotThrow(() -> timestampNtzDf @@ -389,7 +388,7 @@ ELSE CAST('2024-02-03 04:05:06.654321' AS TIMESTAMP_NTZ) END .mode(SaveMode.Overwrite) .save()); - assertTrue(!findVortexFiles(outputPath).isEmpty(), "TimestampNTZ write should create Vortex files"); + assertFalse(findVortexFiles(outputPath).isEmpty(), "TimestampNTZ write should create Vortex files"); } /** @@ -424,7 +423,7 @@ private List projectTemporalAndStructRows(Dataset df) { */ private List findVortexFiles(Path directory) throws IOException { if (!Files.exists(directory)) { - return Arrays.asList(); + return List.of(); } try (Stream paths = Files.walk(directory)) { From e0bbe32e8468d32b4e862abd4ba2912e6da5e9e0 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Thu, 23 Apr 2026 17:00:37 -0400 Subject: [PATCH 178/250] New Java Scan API (#7527) Not yet convinced this is right. Tried to make spark behave nicely, need to look again to see if non distributed case is reasonable --------- Signed-off-by: Robert Kruszewski --- Cargo.lock | 76 +- Cargo.toml | 1 + java/build.gradle.kts | 1 + java/gradle/wrapper/gradle-wrapper.properties | 1 + java/vortex-jni/build.gradle.kts | 5 + .../main/java/dev/vortex/VortexCleaner.java | 14 + .../src/main/java/dev/vortex/api/Array.java | 191 --- .../java/dev/vortex/api/ArrayIterator.java | 69 -- .../src/main/java/dev/vortex/api/DType.java | 549 --------- .../main/java/dev/vortex/api/DataSource.java | 151 +++ .../main/java/dev/vortex/api/Expression.java | 233 ++-- .../src/main/java/dev/vortex/api/File.java | 106 -- .../src/main/java/dev/vortex/api/Files.java | 91 -- .../main/java/dev/vortex/api/Partition.java | 75 ++ .../src/main/java/dev/vortex/api/Scan.java | 72 ++ .../main/java/dev/vortex/api/ScanOptions.java | 81 +- .../src/main/java/dev/vortex/api/Session.java | 39 + .../java/dev/vortex/api/VortexWriter.java | 124 +- .../dev/vortex/api/expressions/Binary.java | 333 ------ .../dev/vortex/api/expressions/GetItem.java | 111 -- .../dev/vortex/api/expressions/IsNotNull.java | 99 -- .../dev/vortex/api/expressions/IsNull.java | 98 -- .../dev/vortex/api/expressions/Literal.java | 1061 ----------------- .../java/dev/vortex/api/expressions/Not.java | 97 -- .../java/dev/vortex/api/expressions/Root.java | 66 - .../dev/vortex/api/expressions/Unknown.java | 47 - .../java/dev/vortex/api/proto/DTypes.java | 208 ---- .../dev/vortex/api/proto/EndianUtils.java | 78 -- .../dev/vortex/api/proto/Expressions.java | 70 -- .../java/dev/vortex/api/proto/Scalars.java | 443 ------- .../vortex/api/proto/TemporalMetadatas.java | 114 -- .../main/java/dev/vortex/jni/JNIArray.java | 151 --- .../java/dev/vortex/jni/JNIArrayIterator.java | 59 - .../main/java/dev/vortex/jni/JNIDType.java | 127 -- .../src/main/java/dev/vortex/jni/JNIFile.java | 55 - .../main/java/dev/vortex/jni/JNIWriter.java | 87 -- .../jni/NativeArrayIteratorMethods.java | 30 - .../dev/vortex/jni/NativeArrayMethods.java | 65 - .../dev/vortex/jni/NativeDTypeMethods.java | 184 --- .../java/dev/vortex/jni/NativeDataSource.java | 36 + .../java/dev/vortex/jni/NativeExpression.java | 47 + .../dev/vortex/jni/NativeFileMethods.java | 68 -- .../main/java/dev/vortex/jni/NativeFiles.java | 34 + .../java/dev/vortex/jni/NativeLoader.java | 41 +- .../java/dev/vortex/jni/NativePartition.java | 29 + .../java/dev/vortex/jni/NativeRuntime.java | 33 + .../main/java/dev/vortex/jni/NativeScan.java | 51 + .../java/dev/vortex/jni/NativeSession.java | 19 + .../java/dev/vortex/jni/NativeWriter.java | 35 + .../dev/vortex/jni/NativeWriterMethods.java | 55 - .../test/java/dev/vortex/api/DTypeTest.java | 73 -- .../test/java/dev/vortex/api/TestMinimal.java | 231 ++-- .../vortex/api/expressions/LiteralTest.java | 23 - .../proto/TestExpressionProtos.java | 41 - .../api/expressions/proto/TestOpenErrors.java | 20 - .../java/dev/vortex/jni/JNIWriterTest.java | 132 +- .../java/dev/vortex/spark/ArrowUtils.java | 6 +- .../java/dev/vortex/spark/SparkTypes.java | 174 --- .../dev/vortex/spark/VortexDataSourceV2.java | 30 +- .../dev/vortex/spark/VortexFilePartition.java | 88 +- .../vortex/spark/VortexSessionProvider.java | 25 + .../dev/vortex/spark/VortexSparkSession.java | 94 ++ .../java/dev/vortex/spark/VortexTable.java | 19 +- .../vortex/spark/read/PartitionPathUtils.java | 20 +- .../dev/vortex/spark/read/ReaderFactory.java | 67 -- .../spark/read/VortexArrowColumnVector.java | 22 +- .../vortex/spark/read/VortexBatchExec.java | 87 +- .../spark/read/VortexColumnarBatch.java | 59 - .../read/VortexColumnarBatchIterator.java | 108 -- .../spark/read/VortexPartitionReader.java | 227 ++-- .../read/VortexPartitionReaderFactory.java | 52 + .../dev/vortex/spark/read/VortexScan.java | 20 +- .../vortex/spark/read/VortexScanBuilder.java | 7 +- .../vortex/spark/write/VortexBatchWrite.java | 8 +- .../vortex/spark/write/VortexDataWriter.java | 27 +- .../java/dev/vortex/spark/SparkTypesTest.java | 44 - .../spark/VortexDataSourceBasicTest.java | 8 +- .../spark/VortexDataSourceS3MockTest.java | 1 + vortex-duckdb/cpp/file_system.cpp | 16 + .../cpp/include/duckdb_vx/file_system.h | 3 + vortex-duckdb/src/filesystem.rs | 21 + vortex-io/Cargo.toml | 1 - vortex-io/public-api.lock | 44 + vortex-io/src/compat/filesystem.rs | 4 + vortex-io/src/compat/mod.rs | 2 + vortex-io/src/compat/obj_store.rs | 111 ++ vortex-io/src/compat/read_at.rs | 2 +- vortex-io/src/filesystem/glob.rs | 4 + vortex-io/src/filesystem/mod.rs | 2 + vortex-io/src/filesystem/prefix.rs | 6 + vortex-io/src/object_store/filesystem.rs | 12 + vortex-jni/Cargo.toml | 9 +- vortex-jni/src/array.rs | 525 -------- vortex-jni/src/array_iter.rs | 122 -- vortex-jni/src/data_source.rs | 267 +++++ vortex-jni/src/dtype.rs | 708 +---------- vortex-jni/src/errors.rs | 87 +- vortex-jni/src/expression.rs | 261 ++++ vortex-jni/src/file.rs | 334 ++---- vortex-jni/src/lib.rs | 48 +- vortex-jni/src/logging.rs | 4 +- vortex-jni/src/object_store.rs | 27 +- vortex-jni/src/runtime.rs | 49 + vortex-jni/src/scan.rs | 353 ++++++ vortex-jni/src/session.rs | 44 + vortex-jni/src/writer.rs | 219 ++-- 106 files changed, 3072 insertions(+), 7836 deletions(-) create mode 100644 java/vortex-jni/src/main/java/dev/vortex/VortexCleaner.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/Array.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/ArrayIterator.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/DType.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/File.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/Files.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/Partition.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/Scan.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/Session.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/GetItem.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/Literal.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/Not.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/Root.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/expressions/Unknown.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/proto/DTypes.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/proto/EndianUtils.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/proto/Scalars.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/api/proto/TemporalMetadatas.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/JNIArray.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/JNIArrayIterator.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/JNIDType.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/JNIFile.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/JNIWriter.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayIteratorMethods.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayMethods.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeDTypeMethods.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeFileMethods.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativePartition.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeRuntime.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeSession.java create mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java delete mode 100644 java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriterMethods.java delete mode 100644 java/vortex-jni/src/test/java/dev/vortex/api/DTypeTest.java delete mode 100644 java/vortex-jni/src/test/java/dev/vortex/api/expressions/LiteralTest.java delete mode 100644 java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java delete mode 100644 java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestOpenErrors.java delete mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/SparkTypes.java create mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/VortexSessionProvider.java create mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/VortexSparkSession.java delete mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/read/ReaderFactory.java delete mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatch.java delete mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatchIterator.java create mode 100644 java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java delete mode 100644 java/vortex-spark/src/test/java/dev/vortex/spark/SparkTypesTest.java create mode 100644 vortex-io/src/compat/obj_store.rs delete mode 100644 vortex-jni/src/array.rs delete mode 100644 vortex-jni/src/array_iter.rs create mode 100644 vortex-jni/src/data_source.rs create mode 100644 vortex-jni/src/expression.rs create mode 100644 vortex-jni/src/runtime.rs create mode 100644 vortex-jni/src/scan.rs create mode 100644 vortex-jni/src/session.rs diff --git a/Cargo.lock b/Cargo.lock index c9b74f2ff69..069aa68ebab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,7 +1420,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.9", + "libloading", ] [[package]] @@ -1978,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3aa12038120eb13347a6ae2ffab1d34efe78150125108627fd85044dd4d6ff1e" dependencies = [ "half", - "libloading 0.8.9", + "libloading", ] [[package]] @@ -4274,12 +4274,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "handle" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ed72152641d513a6084a3907bfcba3f35ae2d3335c22ce2242969c25ff8e46" - [[package]] name = "hashbag" version = "0.1.13" @@ -4882,15 +4876,45 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "java-locator", "jni-sys 0.3.1", - "libloading 0.7.4", "log", "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "java-locator", + "jni-macros", + "jni-sys 0.4.1", + "libloading", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" version = "0.3.1" @@ -5600,16 +5624,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libloading" version = "0.8.9" @@ -8052,7 +8066,7 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.21.1", "log", "once_cell", "rustls", @@ -8468,6 +8482,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -10223,7 +10247,7 @@ name = "vortex-cub" version = "0.1.0" dependencies = [ "bindgen", - "libloading 0.8.9", + "libloading", "paste", "vortex-array", "vortex-cuda-macros", @@ -10542,7 +10566,6 @@ dependencies = [ "custom-labels", "futures", "glob", - "handle", "itertools 0.14.0", "kanal", "object_store 0.13.2", @@ -10584,15 +10607,12 @@ name = "vortex-jni" version = "0.1.0" dependencies = [ "arrow-array 58.1.0", - "arrow-ipc 58.1.0", "arrow-schema 58.1.0", "futures", - "jni", + "jni 0.22.4", "object_store 0.13.2", "parking_lot", - "prost 0.14.3", "thiserror 2.0.18", - "tokio", "tracing", "tracing-subscriber", "url", @@ -10667,7 +10687,7 @@ name = "vortex-nvcomp" version = "0.1.0" dependencies = [ "bindgen", - "libloading 0.8.9", + "libloading", "liblzma", "reqwest 0.13.2", "tar", diff --git a/Cargo.toml b/Cargo.toml index c9b7814af34..92e08f1338f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,6 +164,7 @@ inventory = "0.3.20" itertools = "0.14.0" jetscii = "0.5.3" jiff = "0.2.0" +jni = { version = "0.22.0" } kanal = "0.1.1" lasso = { version = "0.7", features = ["multi-threaded"] } lending-iterator = "0.1.7" diff --git a/java/build.gradle.kts b/java/build.gradle.kts index 128ec72e963..1b3bb74376d 100644 --- a/java/build.gradle.kts +++ b/java/build.gradle.kts @@ -52,6 +52,7 @@ allprojects { // ignore protobuf generated files options.errorprone.excludedPaths = ".*/build/generated/.*" options.release = 17 + options.compilerArgs.add("-Werror") options.generatedSourceOutputDirectory = projectDir.resolve("generated_src") } diff --git a/java/gradle/wrapper/gradle-wrapper.properties b/java/gradle/wrapper/gradle-wrapper.properties index c61a118f7dd..cbcbf0b9fae 100644 --- a/java/gradle/wrapper/gradle-wrapper.properties +++ b/java/gradle/wrapper/gradle-wrapper.properties @@ -5,3 +5,4 @@ networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists + diff --git a/java/vortex-jni/build.gradle.kts b/java/vortex-jni/build.gradle.kts index 148d0cf18af..aca95ec703c 100644 --- a/java/vortex-jni/build.gradle.kts +++ b/java/vortex-jni/build.gradle.kts @@ -138,6 +138,11 @@ tasks.register("makeTestFiles") { into("$projectDir/src/main/resources/native/linux-amd64") } + copy { + from("${rootProject.projectDir.absoluteFile.parentFile}/target/debug/libvortex_jni.so") + into("$projectDir/src/main/resources/native/linux-aarch64") + } + copy { from("${rootProject.projectDir.absoluteFile.parentFile}/target/debug/libvortex_jni.dylib") into("$projectDir/src/main/resources/native/darwin-aarch64") diff --git a/java/vortex-jni/src/main/java/dev/vortex/VortexCleaner.java b/java/vortex-jni/src/main/java/dev/vortex/VortexCleaner.java new file mode 100644 index 00000000000..2e11f90197d --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/VortexCleaner.java @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex; + +import java.lang.ref.Cleaner; + +public final class VortexCleaner { + private static final Cleaner cleaner = Cleaner.create(); + + public static Cleaner.Cleanable register(Object obj, Runnable r) { + return VortexCleaner.cleaner.register(obj, r); + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Array.java b/java/vortex-jni/src/main/java/dev/vortex/api/Array.java deleted file mode 100644 index 06069c611de..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/Array.java +++ /dev/null @@ -1,191 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -import java.math.BigDecimal; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.VectorSchemaRoot; - -public interface Array extends AutoCloseable { - /** - * Returns the number of elements in this array. - * - * @return the length of the array - */ - long getLen(); - - /** - * Returns the total size in bytes of this array including all buffers. - * - * @return the size in bytes - */ - long nbytes(); - - /** - * Export to an ArrowVector. The data will now be owned by the VectorSchemaRoot after this operation. - */ - VectorSchemaRoot exportToArrow(BufferAllocator allocator, VectorSchemaRoot reuse); - - /** - * Returns the data type of this array. - * - * @return the DType describing the logical type of this array - */ - DType getDataType(); - - /** - * Returns a child array at the given field index. - * - *

This is used for accessing fields in struct arrays or elements in list arrays.

- * - * @param index the field index - * @return the child array at the specified index - * @throws IndexOutOfBoundsException if index is out of bounds - */ - Array getField(int index); - - /** - * Returns a slice of this array from start (inclusive) to stop (exclusive). - * - * @param start the starting index (inclusive) - * @param stop the ending index (exclusive) - * @return a new Array containing the sliced elements - * @throws IndexOutOfBoundsException if start or stop are out of bounds - */ - Array slice(int start, int stop); - - /** - * Returns true if the value at the given index is null. - * - * @param index the element index - * @return true if the value is null, false otherwise - * @throws IndexOutOfBoundsException if index is out of bounds - */ - boolean getNull(int index); - - /** - * Returns the total number of null values in this array. - * - * @return the null count - */ - int getNullCount(); - - /** - * Returns the byte value at the given index. - * - * @param index the element index - * @return the byte value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with byte - */ - byte getByte(int index); - - /** - * Returns the short value at the given index. - * - * @param index the element index - * @return the short value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with short - */ - short getShort(int index); - - /** - * Returns the int value at the given index. - * - * @param index the element index - * @return the int value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with int - */ - int getInt(int index); - - /** - * Returns the long value at the given index. - * - * @param index the element index - * @return the long value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with long - */ - long getLong(int index); - - /** - * Returns the boolean value at the given index. - * - * @param index the element index - * @return the boolean value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with boolean - */ - boolean getBool(int index); - - /** - * Returns the float value at the given index. - * - * @param index the element index - * @return the float value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with float - */ - float getFloat(int index); - - /** - * Returns the double value at the given index. - * - * @param index the element index - * @return the double value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with double - */ - double getDouble(int index); - - /** - * Returns the BigDecimal value at the given index. - * - * @param index the element index - * @return the BigDecimal value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with decimal - */ - BigDecimal getBigDecimal(int index); - - /** - * Returns the UTF-8 string value at the given index. - * - * @param index the element index - * @return the string value - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with string - */ - String getUTF8(int index); - - /** - * Returns the UTF-8 string value at the given index as a pointer and length. - * - *

This is a low-level method that provides direct access to the underlying - * string data without copying. The pointer and length are written to the - * provided arrays.

- * - * @param index the element index - * @param ptr array to store the pointer (first element will be set) - * @param len array to store the length (first element will be set) - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with string - */ - void getUTF8_ptr_len(int index, long[] ptr, int[] len); - - /** - * Returns the binary value at the given index as a byte array. - * - * @param index the element index - * @return the binary value as a byte array - * @throws IndexOutOfBoundsException if index is out of bounds - * @throws ClassCastException if the array type is not compatible with binary - */ - byte[] getBinary(int index); - - @Override - void close(); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ArrayIterator.java b/java/vortex-jni/src/main/java/dev/vortex/api/ArrayIterator.java deleted file mode 100644 index b6d29acbc62..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ArrayIterator.java +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -import java.util.Iterator; - -/** - * An iterator interface for traversing Vortex arrays while providing type information - * and resource management capabilities. - * - *

This interface extends both {@link Iterator} and {@link AutoCloseable}, allowing - * for efficient iteration over array elements while ensuring proper cleanup of any - * underlying resources. The iterator provides access to the data type of the arrays - * being iterated over, which is useful for type-safe processing.

- * - *

Example usage:

- *
{@code
- * try (ArrayIterator iterator = array.getIterator()) {
- *     DType dataType = iterator.getDataType();
- *     while (iterator.hasNext()) {
- *         Array element = iterator.next();
- *         // Process element based on dataType
- *     }
- * }
- * }
- * - * @see Array - * @see DType - * @see Iterator - * @see AutoCloseable - */ -public interface ArrayIterator extends AutoCloseable, Iterator { - /** - * Returns the data type of the arrays that this iterator produces. - * - *

This method provides type information about the arrays that will be - * returned by subsequent calls to {@link #next()}. The data type remains - * constant throughout the lifetime of the iterator and can be used for - * type-safe processing and validation.

- * - * @return the {@link DType} representing the data type of arrays produced by this iterator - */ - DType getDataType(); - - /** - * Closes this iterator and releases any underlying resources. - * - *

This method should be called when the iterator is no longer needed to - * ensure proper cleanup of any native resources or memory allocations. - * After calling this method, the iterator should not be used for further - * operations.

- * - *

It is recommended to use this iterator within a try-with-resources - * statement to ensure automatic cleanup:

- *
{@code
-     * try (ArrayIterator iterator = array.getIterator()) {
-     *     // Use iterator
-     * } // close() is called automatically
-     * }
- * - *

This method overrides {@link AutoCloseable#close()} and does not - * throw any checked exceptions, making it safe to use in any context.

- * - * @see AutoCloseable#close() - */ - @Override - void close(); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DType.java b/java/vortex-jni/src/main/java/dev/vortex/api/DType.java deleted file mode 100644 index 2c190b07be8..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/DType.java +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -import dev.vortex.jni.JNIDType; -import dev.vortex.jni.NativeDTypeMethods; -import java.util.List; -import java.util.Optional; - -/** - * Vortex logical type interface representing the schema and metadata for array data. - * - *

DType defines the logical type system used by Vortex arrays, including primitive types, - * complex types like structs and lists, and temporal types with associated metadata. - * This interface provides methods to inspect type variants, nullability, temporal properties, - * decimal precision, and structural information for complex types. - * - *

Implementations of this interface are typically obtained from Vortex arrays and - * should be properly closed when no longer needed to free native resources. - */ -public interface DType extends AutoCloseable { - - /** - * Returns the variant of this data type. - * - * @return the {@link Variant} enum value representing the specific type category - */ - Variant getVariant(); - - /** - * Checks if this data type allows null values. - * - * @return {@code true} if the type is nullable, {@code false} otherwise - */ - boolean isNullable(); - - /** - * Get the field names for a STRUCT type. - */ - List getFieldNames(); - - /** - * Get the field types for a STRUCT type. - */ - List getFieldTypes(); - - /** - * Get the element type for a LIST or FIXED_SIZE_LIST type. - */ - DType getElementType(); - - /** - * Get the fixed size for a FIXED_SIZE_LIST type. - */ - int getFixedSizeListSize(); - - /** - * Checks if this data type represents a date. - * - * @return {@code true} if this is a date type, {@code false} otherwise - */ - boolean isDate(); - - /** - * Checks if this data type represents a time. - * - * @return {@code true} if this is a time type, {@code false} otherwise - */ - boolean isTime(); - - /** - * Checks if this data type represents a timestamp. - * - * @return {@code true} if this is a timestamp type, {@code false} otherwise - */ - boolean isTimestamp(); - - /** - * Returns the time unit for temporal data types. - * - * @return the {@link TimeUnit} for this temporal type - * @throws IllegalStateException if this is not a temporal type - */ - TimeUnit getTimeUnit(); - - /** - * Returns the timezone for timestamp data types. - * - * @return an {@link Optional} containing the timezone string if present, - * or empty if no timezone is specified - */ - Optional getTimeZone(); - - /** - * Checks if this data type represents a decimal number. - * - * @return {@code true} if this is a decimal type, {@code false} otherwise - */ - boolean isDecimal(); - - /** - * Returns the precision for decimal data types. - * - * @return the precision (total number of digits) for decimal types - * @throws IllegalStateException if this is not a decimal type - */ - int getPrecision(); - - /** - * Returns the scale for decimal data types. - * - * @return the scale (number of digits after the decimal point) for decimal types - * @throws IllegalStateException if this is not a decimal type - */ - byte getScale(); - - /** - * Closes this DType and releases any associated native resources. - * - *

After calling this method, the DType should not be used again. - * This method is idempotent and can be called multiple times safely. - */ - @Override - void close(); - - /** - * Create a new signed INT8 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newByte(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newInt(isNullable), true); - } - - /** - * Create a new signed INT16 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newShort(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newShort(isNullable), true); - } - - /** - * Create a new signed INT32 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newInt(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newInt(isNullable), true); - } - - /** - * Create a new signed INT64 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newLong(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newLong(isNullable), true); - } - - /** - * Create a new signed FLOAT32 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newFloat(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newFloat(isNullable), true); - } - - /** - * Create a new signed FLOAT64 data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newDouble(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newDouble(isNullable), true); - } - - /** - * Create a new Decimal data type. - * - * @param precision Decimal values precision - * @param scale Decimal values scale - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newDecimal(int precision, int scale, boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newDecimal(precision, scale, isNullable), true); - } - - /** - * Create a new UTF-8 string data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newUtf8(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newUtf8(isNullable), true); - } - - /** - * Create a new Binary data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newBinary(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newBinary(isNullable), true); - } - - /** - * Create a new Binary data type. - * - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newBool(boolean isNullable) { - return new JNIDType(NativeDTypeMethods.newBool(isNullable), true); - } - - /** - * Create a new List data type. - * - * @param element DType of the list elements - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newList(DType element, boolean isNullable) { - // Get the pointer - JNIDType jniType = (JNIDType) element; - return new JNIDType(NativeDTypeMethods.newList(jniType.getPointer(), isNullable), true); - } - - /** - * Create a new FixedSizeList data type. - * - * @param element DType of the list elements - * @param size The fixed size of each list - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newFixedSizeList(DType element, int size, boolean isNullable) { - JNIDType jniType = (JNIDType) element; - return new JNIDType(NativeDTypeMethods.newFixedSizeList(jniType.getPointer(), size, isNullable), true); - } - - /** - * Create a new Struct data type. - * - * @param fieldNames Name of each of the fields - * @param fieldTypes DType for each field - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newStruct(String[] fieldNames, DType[] fieldTypes, boolean isNullable) { - long[] ptrs = new long[fieldTypes.length]; - for (int i = 0; i < fieldTypes.length; i++) { - ptrs[i] = ((JNIDType) fieldTypes[i]).getPointer(); - } - return new JNIDType(NativeDTypeMethods.newStruct(fieldNames, ptrs, isNullable), true); - } - - /** - * Create a new Timestamp data type. - * - * @param unit Time unit for the timestamp values - * @param timeZone Optional time zone for the timestamp values - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newTimestamp(TimeUnit unit, Optional timeZone, boolean isNullable) { - byte timeUnit = unit.asByte(); - return new JNIDType(NativeDTypeMethods.newTimestamp(timeUnit, timeZone.orElse(null), isNullable), true); - } - - /** - * Create a new Date data type. - * - * @param unit Time unit for the value - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newDate(TimeUnit unit, boolean isNullable) { - byte timeUnit = unit.asByte(); - return new JNIDType(NativeDTypeMethods.newDate(timeUnit, isNullable), true); - } - - /** - * Create a new Time data type. - * - * @param unit Time unit for the value - * @param isNullable True if the values can be null - * @return The new DType instance, allocated in native heap memory - */ - static DType newTime(TimeUnit unit, boolean isNullable) { - byte timeUnit = unit.asByte(); - return new JNIDType(NativeDTypeMethods.newTime(timeUnit, isNullable), true); - } - - /** - * Enumeration of time units supported by Vortex temporal data types. - * - *

Time units define the granularity of temporal values and are used - * by date, time, and timestamp data types to specify their precision. - */ - enum TimeUnit { - /** - * Nanosecond precision (10^-9 seconds) - */ - NANOSECONDS, - - /** - * Microsecond precision (10^-6 seconds) - */ - MICROSECONDS, - - /** - * Millisecond precision (10^-3 seconds) - */ - MILLISECONDS, - - /** - * Second precision - */ - SECONDS, - - /** - * Day precision (24-hour periods) - */ - DAYS, - ; - - /** - * Converts a byte value to the corresponding TimeUnit enum. - * - * @param unit the byte value representing the time unit (0-4) - * @return the corresponding {@link TimeUnit} enum value - * @throws RuntimeException if the unit value is not recognized - */ - public static TimeUnit from(byte unit) { - switch (unit) { - case 0: - return NANOSECONDS; - case 1: - return MICROSECONDS; - case 2: - return MILLISECONDS; - case 3: - return SECONDS; - case 4: - return DAYS; - default: - throw new IllegalArgumentException("Unknown TimeUnit: " + unit); - } - } - - /** - * Get the byte value of this TimeUnit. - */ - public byte asByte() { - switch (this) { - case NANOSECONDS: - return 0; - case MICROSECONDS: - return 1; - case MILLISECONDS: - return 2; - case SECONDS: - return 3; - case DAYS: - return 4; - default: - throw new IllegalArgumentException("Unknown TimeUnit: " + this); - } - } - } - - /** - * Enumeration of all supported data type variants in Vortex. - * - *

Each variant represents a different category of data type, from primitive - * numeric types to complex structured types. This enum provides a way to - * categorize and identify the specific type of data stored in a Vortex array. - */ - enum Variant { - /** - * Null type representing absence of value - */ - NULL, - - /** - * Boolean type for true/false values - */ - BOOL, - - /** - * Unsigned 8-bit integer type - */ - PRIMITIVE_U8, - - /** - * Unsigned 16-bit integer type - */ - PRIMITIVE_U16, - - /** - * Unsigned 32-bit integer type - */ - PRIMITIVE_U32, - - /** - * Unsigned 64-bit integer type - */ - PRIMITIVE_U64, - - /** - * Signed 8-bit integer type - */ - PRIMITIVE_I8, - - /** - * Signed 16-bit integer type - */ - PRIMITIVE_I16, - - /** - * Signed 32-bit integer type - */ - PRIMITIVE_I32, - - /** - * Signed 64-bit integer type - */ - PRIMITIVE_I64, - - /** - * 16-bit floating point type - */ - PRIMITIVE_F16, - - /** - * 32-bit floating point type - */ - PRIMITIVE_F32, - - /** - * 64-bit floating point type - */ - PRIMITIVE_F64, - - /** - * UTF-8 encoded string type - */ - UTF8, - - /** - * Binary data type for arbitrary byte sequences - */ - BINARY, - - /** - * Structured type containing named fields - */ - STRUCT, - - /** - * List type containing elements of a single type - */ - LIST, - - /** - * Extension type for custom or domain-specific types - */ - EXTENSION, - - /** - * Decimal type for precise numeric values - */ - DECIMAL, - - /** - * Fixed-size list type containing a fixed number of elements of a single type - */ - FIXED_SIZE_LIST, - ; - - /** - * Converts a byte value to the corresponding Variant enum. - * - * @param variant the byte value representing the variant (0-19) - * @return the corresponding {@link Variant} enum value - * @throws RuntimeException if the variant value is not recognized - */ - public static Variant from(byte variant) { - switch (variant) { - case 0: - return NULL; - case 1: - return BOOL; - case 2: - return PRIMITIVE_U8; - case 3: - return PRIMITIVE_U16; - case 4: - return PRIMITIVE_U32; - case 5: - return PRIMITIVE_U64; - case 6: - return PRIMITIVE_I8; - case 7: - return PRIMITIVE_I16; - case 8: - return PRIMITIVE_I32; - case 9: - return PRIMITIVE_I64; - case 10: - return PRIMITIVE_F16; - case 11: - return PRIMITIVE_F32; - case 12: - return PRIMITIVE_F64; - case 13: - return UTF8; - case 14: - return BINARY; - case 15: - return STRUCT; - case 16: - return LIST; - case 17: - return EXTENSION; - case 18: - return DECIMAL; - case 19: - return FIXED_SIZE_LIST; - default: - throw new IllegalArgumentException("Unknown DType variant: " + variant); - } - } - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java new file mode 100644 index 00000000000..6d2885e8fa6 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.api; + +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativeDataSource; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.OptionalLong; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * A set of Vortex files opened through a {@link Session}. Data sources are cheap to open + * (only the first file is read eagerly, to determine the schema) and can be scanned + * multiple times. + * + *

Native resources are released automatically via {@link VortexCleaner} when the + * data source becomes unreachable. + */ +public final class DataSource { + private final Session session; + private final long pointer; + + private DataSource(Session session, long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid data source pointer"); + this.session = Objects.requireNonNull(session, "session"); + this.pointer = pointer; + VortexCleaner.register(this, () -> NativeDataSource.free(pointer)); + } + + /** Open a single URI. */ + public static DataSource open(Session session, String uri) { + return open(session, uri, Collections.emptyMap()); + } + + /** + * Open one or more URIs or globs. When a glob is used, the first match is opened eagerly; + * subsequent matches are opened lazily on scan. + * + * @param session open session + * @param uri single URI or glob + * @param properties object-store credentials / options + */ + public static DataSource open(Session session, String uri, Map properties) { + return open(session, List.of(uri), properties); + } + + /** + * Open one or more URIs or globs. When a glob is used, the first match is opened eagerly; + * subsequent matches are opened lazily on scan. + * + * @param session open session + * @param uris URIs or globs to scan + * @param properties object-store credentials / options + */ + public static DataSource open(Session session, List uris, Map properties) { + Objects.requireNonNull(session, "session"); + Objects.requireNonNull(uris, "uris"); + Preconditions.checkArgument(!uris.isEmpty(), "at least one uri is required"); + String[] uriArray = uris.toArray(String[]::new); + Preconditions.checkArgument( + Arrays.stream(uriArray).allMatch(Objects::nonNull), "uris must not contain null values"); + long sessionPointer = session.nativePointer(); + long pointer = NativeDataSource.open(sessionPointer, uriArray, properties); + return new DataSource(session, pointer); + } + + /** Arrow schema of the data source (and of scans produced from it). */ + public Schema arrowSchema(BufferAllocator allocator) { + try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) { + NativeDataSource.arrowSchema(pointer, schema.memoryAddress()); + return Data.importSchema(allocator, schema, null); + } + } + + /** + * Row count along with the precision of that estimate. Mirrors the Rust + * {@code Option>} returned by {@code DataSource::row_count}: + * {@link RowCount.Unknown} when no estimate is available, {@link RowCount.Estimate} + * for an inexact hint, {@link RowCount.Exact} when the count is authoritative. + */ + public RowCount rowCount() { + long[] out = new long[2]; + NativeDataSource.rowCount(pointer, out); + return switch ((int) out[1]) { + case 1 -> new RowCount.Estimate(out[0]); + case 2 -> new RowCount.Exact(out[0]); + default -> RowCount.Unknown.INSTANCE; + }; + } + + /** Precision-aware row count. See {@link #rowCount()}. */ + public sealed interface RowCount { + /** Returns the row count as a long, or {@code OptionalLong.empty()} when unknown. */ + OptionalLong asOptional(); + + /** Row count is not known. */ + final class Unknown implements RowCount { + public static final Unknown INSTANCE = new Unknown(); + + private Unknown() {} + + @Override + public OptionalLong asOptional() { + return OptionalLong.empty(); + } + } + + /** Estimated row count; the actual value may differ. */ + record Estimate(long value) implements RowCount { + @Override + public OptionalLong asOptional() { + return OptionalLong.of(value); + } + } + + /** Exact row count. */ + record Exact(long value) implements RowCount { + @Override + public OptionalLong asOptional() { + return OptionalLong.of(value); + } + } + } + + /** Submit a scan. */ + public Scan scan(ScanOptions options) { + Objects.requireNonNull(options, "options"); + + long projectionPtr = options.projection().map(Expression::nativePointer).orElse(0L); + long filterPtr = options.filter().map(Expression::nativePointer).orElse(0L); + long begin = options.rowRangeBegin().orElse(0L); + long end = options.rowRangeEnd().orElse(0L); + long[] selectionIndices = options.selectionIndices().orElse(null); + byte selectionMode = options.selectionMode().code(); + long limit = options.limit().orElse(0L); + boolean ordered = options.ordered(); + + long scanPtr = dev.vortex.jni.NativeScan.create( + pointer, projectionPtr, filterPtr, begin, end, selectionIndices, selectionMode, limit, ordered); + return Scan.fromPointer(session, scanPtr); + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java index e2fca7be023..4765d3dca7e 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Expression.java @@ -3,109 +3,140 @@ package dev.vortex.api; -import dev.vortex.api.expressions.*; -import java.util.List; -import java.util.Optional; +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativeExpression; +import java.util.Arrays; /** - * Vortex expression language. + * A Vortex expression node backed by a native pointer. + * + *

Expressions are composed via the static factories ({@link #root()}, {@link + * #getItem(String, Expression)}, etc.). Each returned {@code Expression} owns its native + * pointer; the pointer is released automatically when the {@code Expression} is no longer + * reachable. Passing an expression as an input to a builder does not transfer + * ownership — the resulting expression is an independent copy on the native side. */ -public interface Expression { - /** - * The globally unique identifier for this type of expression. - */ - String id(); - - /** - * Returns the children of this expression. - */ - List children(); - - /** - * Returns the serialized metadata for this expression, or empty if serialization is not supported. - */ - Optional metadata(); - - /** - * Accepts a visitor and dispatches to the appropriate visit method based on the expression type. - * This method implements the visitor pattern, allowing different operations to be performed - * on expressions without modifying the expression classes themselves. - * - * @param the return type of the visitor - * @param visitor the visitor to accept - * @return the result of the visitor's operation on this expression - */ - default T accept(Visitor visitor) { - return visitor.visitOther(this); - } - - /** - * Visitor interface for implementing the visitor pattern on expressions. - * This interface defines methods for visiting different types of expressions, - * allowing for type-safe operations across the expression hierarchy. - * - * @param the return type of the visitor methods - */ - interface Visitor { - /** - * Visits a literal expression. - * - * @param literal the literal expression to visit - * @return the result of visiting the literal expression - */ - T visitLiteral(Literal literal); - - /** - * Visits a root expression. - * - * @param root the root expression to visit - * @return the result of visiting the root expression - */ - T visitRoot(Root root); - - /** - * Visits a binary expression. - * - * @param binary the binary expression to visit - * @return the result of visiting the binary expression - */ - T visitBinary(Binary binary); - - /** - * Visits a not expression (logical negation). - * - * @param not the not expression to visit - * @return the result of visiting the not expression - */ - T visitNot(Not not); - - /** - * Visits a get item expression (array/object indexing). - * - * @param getItem the get item expression to visit - * @return the result of visiting the get item expression - */ - T visitGetItem(GetItem getItem); - - /** - * Visits an is null expression (null check). - * - * @param isNull the is null expression to visit - * @return the result of visiting the is null expression - */ - T visitIsNull(IsNull isNull); - - /** - * Visits an is not null expression (non-null check). - * - * @param isNotNull the is not null expression to visit - * @return the result of visiting the is not null expression - */ - T visitIsNotNull(IsNotNull isNotNull); - - /** - * For expressions that do not have a specific visitor method. - */ - T visitOther(Expression expression); +public final class Expression { + private final long pointer; + + private Expression(long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid expression pointer"); + this.pointer = pointer; + VortexCleaner.register(this, () -> NativeExpression.free(pointer)); + } + + long nativePointer() { + return pointer; + } + + /** The root expression: applying it to an array yields the array itself. */ + public static Expression root() { + return new Expression(NativeExpression.root()); + } + + /** Access a named field from a struct expression. */ + public static Expression getItem(String fieldName, Expression child) { + return new Expression(NativeExpression.getItem(fieldName, child.nativePointer())); + } + + /** Shortcut for {@code getItem(fieldName, root())}. */ + public static Expression column(String fieldName) { + return getItem(fieldName, root()); + } + + /** Project a subset of fields out of a struct expression. */ + public static Expression select(String[] fieldNames, Expression child) { + return new Expression(NativeExpression.select(fieldNames, child.nativePointer())); + } + + /** Logical AND. Requires at least one operand. */ + public static Expression and(Expression... operands) { + Preconditions.checkArgument(operands.length > 0, "and requires at least one operand"); + return new Expression(NativeExpression.and(nativePointers(operands))); + } + + /** Logical OR. Requires at least one operand. */ + public static Expression or(Expression... operands) { + Preconditions.checkArgument(operands.length > 0, "or requires at least one operand"); + return new Expression(NativeExpression.or(nativePointers(operands))); + } + + public static Expression binary(BinaryOp op, Expression lhs, Expression rhs) { + return new Expression(NativeExpression.binary(op.code(), lhs.nativePointer(), rhs.nativePointer())); + } + + public static Expression not(Expression child) { + return new Expression(NativeExpression.not(child.nativePointer())); + } + + public static Expression isNull(Expression child) { + return new Expression(NativeExpression.isNull(child.nativePointer())); + } + + public static Expression literal(boolean value) { + return new Expression(NativeExpression.literalBool(value, false)); + } + + public static Expression nullLiteralBool() { + return new Expression(NativeExpression.literalBool(false, true)); + } + + public static Expression literal(byte value) { + return new Expression(NativeExpression.literalI8(value, false)); + } + + public static Expression literal(short value) { + return new Expression(NativeExpression.literalI16(value, false)); + } + + public static Expression literal(int value) { + return new Expression(NativeExpression.literalI32(value, false)); + } + + public static Expression literal(long value) { + return new Expression(NativeExpression.literalI64(value, false)); + } + + public static Expression literal(float value) { + return new Expression(NativeExpression.literalF32(value, false)); + } + + public static Expression literal(double value) { + return new Expression(NativeExpression.literalF64(value, false)); + } + + public static Expression literal(String value) { + return new Expression(NativeExpression.literalString(value)); + } + + private static long[] nativePointers(Expression[] exprs) { + return Arrays.stream(exprs).mapToLong(Expression::nativePointer).toArray(); + } + + /** Binary operator codes; must match the Rust {@code parse_op} table. */ + public enum BinaryOp { + EQ((byte) 0), + NOT_EQ((byte) 1), + GT((byte) 2), + GTE((byte) 3), + LT((byte) 4), + LTE((byte) 5), + AND((byte) 6), + OR((byte) 7), + ADD((byte) 8), + SUB((byte) 9), + MUL((byte) 10), + DIV((byte) 11); + + private final byte code; + + BinaryOp(byte code) { + this.code = code; + } + + public byte code() { + return code; + } } } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/File.java b/java/vortex-jni/src/main/java/dev/vortex/api/File.java deleted file mode 100644 index 414f0a47de0..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/File.java +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -/** - * Interface for reading Vortex format files, providing access to schema information, - * row metadata, and configurable scanning capabilities. - * - *

A {@code File} represents a Vortex format file that has been opened for reading. - * It provides methods to inspect the file's schema, count rows, and create iterators - * for scanning the data with various filtering and projection options. This interface - * extends {@link AutoCloseable} to ensure proper resource cleanup when the file - * is no longer needed.

- * - *

Example usage:

- *
{@code
- * try (File file = VortexReader.open(path)) {
- *     DType schema = file.getDType();
- *     long totalRows = file.rowCount();
- *
- *     ScanOptions options = ScanOptions.builder()
- *         .columns(List.of("name", "age"))
- *         .build();
- *
- *     try (ArrayIterator iterator = file.newScan(options)) {
- *         while (iterator.hasNext()) {
- *             Array batch = iterator.next();
- *             // Process batch
- *         }
- *     }
- * }
- * }
- * - * @see ScanOptions - * @see ArrayIterator - * @see DType - * @see AutoCloseable - */ -public interface File extends AutoCloseable { - /** - * Returns the data type (schema) of this Vortex file. - * - *

The returned {@link DType} describes the logical structure and types - * of the data contained in this file. For structured data, this will typically - * be a {@link DType.Variant#STRUCT} containing field names and their corresponding - * data types. The schema remains constant for the lifetime of the file.

- * - * @return the {@link DType} representing the schema of this file - */ - DType getDType(); - - /** - * Returns the total number of rows in this Vortex file. - * - *

This method provides the count of logical rows contained in the file, - * which represents the number of records or tuples that can be read. This - * count is independent of any filtering or projection that may be applied - * during scanning operations.

- * - * @return the total number of rows as a non-negative long value - */ - long rowCount(); - - /** - * Creates a new iterator for scanning this file with the specified options. - * - *

This method returns an {@link ArrayIterator} that can be used to traverse - * the data in this file according to the provided {@link ScanOptions}. The - * scan options allow for column projection, row filtering via predicates, - * and row range or index selection. Each call to this method creates a new - * independent iterator.

- * - *

The returned iterator must be properly closed when no longer needed to - * release any underlying resources. It is recommended to use the iterator - * within a try-with-resources statement.

- * - * @param options the {@link ScanOptions} configuring the scan behavior, - * including column selection, filtering, and row selection - * @return a new {@link ArrayIterator} for scanning the file data - * @throws RuntimeException if the scan options contain invalid - * column names or conflicting row selection criteria - * @see ScanOptions - * @see ArrayIterator - */ - ArrayIterator newScan(ScanOptions options); - - /** - * Closes this file and releases any associated resources. - * - *

This method should be called when the file is no longer needed to ensure - * proper cleanup of any underlying file handles, native memory, or other resources. - * After calling this method, the file should not be used for any further operations. - * This method is idempotent and can be called multiple times safely.

- * - *

It is recommended to use this file within a try-with-resources statement - * to ensure automatic cleanup:

- *
{@code
-     * try (File file = VortexReader.open(path)) {
-     *     // Use file
-     * } // close() is called automatically
-     * }
- */ - @Override - void close(); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Files.java b/java/vortex-jni/src/main/java/dev/vortex/api/Files.java deleted file mode 100644 index 8483c26be56..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/Files.java +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -import com.google.common.base.Preconditions; -import dev.vortex.jni.JNIFile; -import dev.vortex.jni.NativeFileMethods; -import java.net.URI; -import java.nio.file.Paths; -import java.util.Map; - -/** - * Utility class for opening and accessing Vortex files. - * - *

This class provides static methods to open Vortex files from various sources, - * including local file system paths and URIs. It supports both simple path-based - * access and more advanced configuration through properties. - * - *

All methods in this class are static and the class cannot be instantiated. - * - * @since 1.0 - */ -public final class Files { - - /** - * Private constructor to prevent instantiation of this utility class. - */ - private Files() {} - - /** - * Opens a Vortex file from the specified path string. - * - * @see #open(String, Map) - */ - public static File open(String path) { - return open(path, Map.of()); - } - - /** - * Opens a Vortex file from the specified path string and format parameters. - * - *

This method provides a convenient way to open Vortex files using either - * absolute file system paths or URI strings. If the path starts with "/", - * it is treated as an absolute file system path and converted to a file URI. - * Otherwise, it is parsed as a URI directly. - * - *

This is equivalent to calling {@code open(uri, Map.of())} with no additional - * properties. - * - * @param path the path to the Vortex file, either as an absolute file system path - * (starting with "/") or as a URI string - * @return a {@link File} instance representing the opened Vortex file - * @throws RuntimeException if the file cannot be opened or the path is invalid - * @throws NullPointerException if path is null - * @see #open(URI, Map) - */ - public static File open(String path, Map properties) { - if (path.startsWith("/")) { - return open(Paths.get(path).toUri(), properties); - } - return open(URI.create(path), properties); - } - - /** - * Opens a Vortex file from the specified URI with additional properties. - * - *

This method provides the most flexible way to open Vortex files, allowing - * you to specify both the location via a URI and additional configuration - * properties. The URI can point to local files, remote resources, or other - * supported storage systems. - * - *

The properties map can be used to pass configuration options to the - * underlying file system or storage layer. The specific properties supported - * depend on the URI scheme and storage backend being used. - * - * @param uri the URI pointing to the Vortex file to open - * @param properties a map of configuration properties for opening the file; - * may be empty but must not be null - * @return a {@link File} instance representing the opened Vortex file - * @throws RuntimeException if the file cannot be opened, the URI is invalid, - * or the returned native pointer is invalid - * @throws NullPointerException if uri or properties is null - * @see #open(String) - */ - public static File open(URI uri, Map properties) { - long ptr = NativeFileMethods.open(uri.toString(), properties); - Preconditions.checkArgument(ptr > 0, "Failed to open file: %s", uri); - return new JNIFile(ptr); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Partition.java b/java/vortex-jni/src/main/java/dev/vortex/api/Partition.java new file mode 100644 index 00000000000..db8964228fa --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Partition.java @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.api; + +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativePartition; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; + +/** + * A unit of scan work that materializes into an Arrow stream. Partitions are single-pass: + * calling {@link #scanArrow(BufferAllocator)} consumes the partition and transfers + * ownership of its native memory to the returned {@link ArrowReader}. If the partition + * is never consumed, its native memory is released automatically via {@link VortexCleaner}. + */ +public final class Partition { + private final Session session; + private final long pointer; + private final AtomicBoolean consumed = new AtomicBoolean(false); + + private Partition(Session session, long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid partition pointer"); + this.session = session; + this.pointer = pointer; + AtomicBoolean consumedRef = this.consumed; + VortexCleaner.register(this, () -> { + if (!consumedRef.get()) { + NativePartition.free(pointer); + } + }); + } + + static Partition fromPointer(Session session, long pointer) { + return new Partition(session, pointer); + } + + /** Estimated row count of the partition. Empty when unknown. */ + public OptionalLong rowCount() { + Preconditions.checkState(!consumed.get(), "partition already consumed"); + long[] out = new long[2]; + NativePartition.rowCount(pointer, out); + if (out[1] == 0) { + return OptionalLong.empty(); + } + return OptionalLong.of(out[0]); + } + + /** + * Consume the partition and return an {@link ArrowReader} that yields record batches. + * The caller must close the reader when finished; doing so releases the native partition + * resources as well. + */ + public ArrowReader scanArrow(BufferAllocator allocator) { + if (!consumed.compareAndSet(false, true)) { + throw new IllegalStateException("partition already consumed"); + } + // Native side unconditionally takes ownership of the partition, regardless of + // whether the call subsequently throws, so it is correct to flip `consumed` + // before invoking JNI. The cleaner then always skips free for this handle. + ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator); + try { + NativePartition.scanArrow(session.nativePointer(), pointer, stream.memoryAddress()); + } catch (RuntimeException ex) { + stream.close(); + throw ex; + } + return Data.importArrayStream(allocator, stream); + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Scan.java b/java/vortex-jni/src/main/java/dev/vortex/api/Scan.java new file mode 100644 index 00000000000..a1f2a2752ef --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Scan.java @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.api; + +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativeScan; +import java.util.Iterator; +import java.util.NoSuchElementException; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Schema; + +/** + * A lazy handle to a set of {@link Partition partitions}. + * + *

Once a scan has produced its last partition it is effectively exhausted; native + * resources are released automatically via {@link VortexCleaner} when the scan becomes + * unreachable. + */ +public final class Scan implements Iterator { + private final Session session; + private final long pointer; + + private long nextPartitionPointer; + private boolean primed; + + private Scan(Session session, long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid scan pointer"); + this.session = session; + this.pointer = pointer; + VortexCleaner.register(this, () -> NativeScan.free(pointer)); + } + + static Scan fromPointer(Session session, long pointer) { + return new Scan(session, pointer); + } + + /** + * Arrow schema produced by this scan. Must be called before the first call to + * {@link #hasNext()}/{@link #next()}. + */ + public Schema arrowSchema(BufferAllocator allocator) { + try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) { + NativeScan.arrowSchema(pointer, schema.memoryAddress()); + return Data.importSchema(allocator, schema, null); + } + } + + @Override + public boolean hasNext() { + if (primed) { + return nextPartitionPointer != 0; + } + nextPartitionPointer = NativeScan.nextPartition(pointer); + primed = true; + return nextPartitionPointer != 0; + } + + @Override + public Partition next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + long ptr = nextPartitionPointer; + nextPartitionPointer = 0; + primed = false; + return Partition.fromPointer(session, ptr); + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java index 68a6e7f9c9a..800bc54831d 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/ScanOptions.java @@ -3,52 +3,77 @@ package dev.vortex.api; -import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import org.immutables.value.Value; /** - * Create a new set of options for configuring the scan. + * Scan configuration passed to {@link DataSource#scan(ScanOptions)}. + * + *

All fields are optional. A call to {@link #of()} returns a default that reads every + * row and column. */ @Value.Immutable public interface ScanOptions { - /** - * Columns to project out. - */ - List columns(); - /** - * Optional pruning expression that is pushed down to the scan. - */ - Optional predicate(); + /** Projection expression. If empty, all columns are returned. */ + Optional projection(); - /** - * Optional start (inclusive) and end (exclusive) row indices to select a range of rows - * in the scan. - */ - Optional rowRange(); + /** Filter expression applied before returning rows. */ + Optional filter(); - /** - * Optional row indices to select specific rows. - * These must be sorted in ascending order. - */ - Optional rowIndices(); + /** Inclusive start of the row range to read. */ + OptionalLong rowRangeBegin(); + + /** Exclusive end of the row range to read. */ + OptionalLong rowRangeEnd(); /** - * Creates a new ScanOptions instance with default values. - * - * @return a ScanOptions instance with empty columns list, no predicate, no row range, and no row indices + * Sorted ascending row indices that should be included in (or excluded from) the scan, + * depending on {@link #selectionMode()}. */ + Optional selectionIndices(); + + /** Meaning of {@link #selectionIndices()}. */ + @Value.Default + default SelectionMode selectionMode() { + return SelectionMode.INCLUDE_ALL; + } + + /** Maximum row count to return. Absent means "no limit". */ + OptionalLong limit(); + + /** If {@code true}, the scan preserves the original row order across partitions. */ + @Value.Default + default boolean ordered() { + return false; + } + static ScanOptions of() { return ImmutableScanOptions.builder().build(); } - /** - * Creates a new builder for constructing ScanOptions instances. - * - * @return a new builder instance that can be used to configure and build ScanOptions - */ static ImmutableScanOptions.Builder builder() { return ImmutableScanOptions.builder(); } + + /** How to interpret {@link #selectionIndices()}. */ + enum SelectionMode { + /** Ignore {@link #selectionIndices()}. */ + INCLUDE_ALL((byte) 0), + /** Return only rows at the indices. */ + INCLUDE((byte) 1), + /** Return rows except those at the indices. */ + EXCLUDE((byte) 2); + + private final byte code; + + SelectionMode(byte code) { + this.code = code; + } + + public byte code() { + return code; + } + } } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/Session.java b/java/vortex-jni/src/main/java/dev/vortex/api/Session.java new file mode 100644 index 00000000000..7140c703c26 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/api/Session.java @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.api; + +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativeSession; + +/** + * Handle to a native Vortex session. The session owns a current-thread async runtime and + * is the entry point for opening {@link DataSource data sources} and {@link VortexWriter + * writers}. + * + *

Sessions are safe to share across threads within a process, though concrete + * operations (scans, writes) remain single-threaded on the session's runtime thread. + * + *

Native resources are released automatically when the session becomes unreachable, + * via {@link VortexCleaner}. + */ +public final class Session { + private final long pointer; + + private Session(long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid session pointer"); + this.pointer = pointer; + VortexCleaner.register(this, () -> NativeSession.free(pointer)); + } + + /** Create a new session. */ + public static Session create() { + return new Session(NativeSession.newSession()); + } + + /** Internal: returns the native pointer. Do not free directly. */ + public long nativePointer() { + return pointer; + } +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java b/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java index 9bc5dfd0cf3..1a2bb8a19ce 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/VortexWriter.java @@ -3,65 +3,93 @@ package dev.vortex.api; -import dev.vortex.jni.JNIDType; -import dev.vortex.jni.JNIWriter; -import dev.vortex.jni.NativeWriterMethods; +import com.google.common.base.Preconditions; +import dev.vortex.VortexCleaner; +import dev.vortex.jni.NativeWriter; import java.io.IOException; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.types.pojo.Schema; /** - * Writer for creating Vortex files from Arrow data. - *

- * This class provides methods to write Arrow VectorSchemaRoot batches - * to Vortex format files. + * Writer for Vortex files. + * + *

Batches are accepted via the Arrow C Data Interface: callers export an Arrow record + * batch to an {@code ArrowArray} / {@code ArrowSchema} pair and pass the addresses to + * {@link #writeBatch(long, long)}. The writer accepts up to four in-flight batches + * on the session's runtime thread before back-pressuring the caller. + * + *

Call {@link #close()} to flush remaining batches and finalize the file. If the writer + * becomes unreachable without an explicit {@code close()}, {@link VortexCleaner} will flush + * and release native resources as a backstop — but callers should always finalize + * explicitly so that I/O errors surface through the normal call path. */ -public interface VortexWriter extends AutoCloseable { +public final class VortexWriter implements AutoCloseable { + private final long pointer; + private final AtomicBoolean closed = new AtomicBoolean(false); - /** - * Creates a new VortexWriter for writing to the specified file path. - * - * @param uri The URI for where the file is opened - * @param dtype The Vortex DType for data that gets written - * @param options additional writer options - * @return a new VortexWriter instance - * @throws IOException if the writer cannot be created - */ - static VortexWriter create(String uri, DType dtype, Map options) throws IOException { - long ptr = NativeWriterMethods.create(uri, ((JNIDType) dtype).getPointer(), options); - if (ptr <= 0) { - throw new IOException("Failed to create Vortex writer for: " + uri + " (got ptr=" + ptr + ")"); - } - return new JNIWriter(ptr); + private VortexWriter(long pointer) { + Preconditions.checkArgument(pointer != 0, "invalid writer pointer"); + this.pointer = pointer; + AtomicBoolean closedRef = this.closed; + VortexCleaner.register(this, () -> { + if (closedRef.compareAndSet(false, true)) { + NativeWriter.close(pointer); + } + }); } /** - * Writes a batch of Arrow data to the Vortex file. - * - * @param arrowData the Arrow data in IPC format as byte array - * @throws IOException if writing fails + * Create a writer that streams records into the file at {@code uri}. The Arrow schema + * describes the exact layout of every batch written. */ - void writeBatch(byte[] arrowData) throws IOException; + public static VortexWriter create( + Session session, String uri, Schema arrowSchema, Map options, BufferAllocator allocator) + throws IOException { + Objects.requireNonNull(session, "session"); + Objects.requireNonNull(uri, "uri"); + Objects.requireNonNull(arrowSchema, "arrowSchema"); + Objects.requireNonNull(allocator, "allocator"); + ArrowSchema ffi = ArrowSchema.allocateNew(allocator); + try { + Data.exportSchema(allocator, arrowSchema, null, ffi); + long ptr = NativeWriter.create(session.nativePointer(), uri, ffi.memoryAddress(), options); + if (ptr <= 0) { + throw new IOException("failed to create writer for uri " + uri + " (ptr=" + ptr + ")"); + } + return new VortexWriter(ptr); + } finally { + ffi.close(); + } + } - /** - * Writes a batch of Arrow data directly from Arrow C Data Interface pointers. - *

- * This avoids the IPC serialization overhead by accepting raw memory addresses - * of ArrowArray and ArrowSchema structs. - * - * @param arrowArrayAddr memory address of the ArrowArray struct - * @param arrowSchemaAddr memory address of the ArrowSchema struct - * @throws IOException if writing fails - */ - void writeBatchFfi(long arrowArrayAddr, long arrowSchemaAddr) throws IOException; + /** Write a batch directly from Arrow C Data Interface addresses. */ + public void writeBatch(long arrowArrayAddr, long arrowSchemaAddr) throws IOException { + Preconditions.checkState(!closed.get(), "writer already closed"); + final boolean ok; + try { + ok = NativeWriter.writeBatch(pointer, arrowArrayAddr, arrowSchemaAddr); + } catch (RuntimeException e) { + throw new IOException("failed to write batch", e); + } + if (!ok) { + throw new IOException("failed to write batch"); + } + } - /** - * Closes the writer and finalizes the Vortex file. - *

- * This method must be called to ensure the file is properly written - * with all necessary metadata and footers. - * - * @throws IOException if closing fails - */ + /** Flush any pending batches and finalize the file. Idempotent. */ @Override - void close() throws IOException; + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + try { + NativeWriter.close(pointer); + } catch (RuntimeException e) { + throw new IOException("failed to close writer", e); + } + } + } } diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java deleted file mode 100644 index 54e3b09c7df..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Binary.java +++ /dev/null @@ -1,333 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import com.google.protobuf.InvalidProtocolBufferException; -import dev.vortex.api.Expression; -import dev.vortex.proto.ExprProtos; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Stream; - -/** - * Represents a binary expression that operates on two child expressions using a binary operator. - * Binary expressions support comparison operations (equality, inequality, relational comparisons) - * and boolean algebra operations (AND, OR). - * - *

This class is immutable and implements the {@link Expression} interface, making it suitable - * for use in expression trees and query processing pipelines.

- * - *

Example usage:

- *
- * // Create equality comparison: left == right
- * Binary equalExpr = Binary.eq(leftExpr, rightExpr);
- *
- * // Create logical AND: expr1 && expr2 && expr3
- * Binary andExpr = Binary.and(expr1, expr2, expr3);
- *
- * // Create greater than comparison: left > right
- * Binary gtExpr = Binary.gt(leftExpr, rightExpr);
- * 
- */ -public final class Binary implements Expression { - private final BinaryOp operator; - private final Expression left; - private final Expression right; - - private Binary(BinaryOp operator, Expression left, Expression right) { - this.operator = operator; - this.left = left; - this.right = right; - } - - /** - * Parses a Binary expression from protobuf metadata and child expressions. - * - * @param metadata the serialized protobuf metadata containing the binary operator - * @param children the list of child expressions, must contain exactly 2 expressions - * @return a new Binary expression instance - * @throws RuntimeException if children size is not 2 or if metadata parsing fails - */ - public static Binary parse(byte[] metadata, List children) { - if (children.size() != 2) { - throw new IllegalArgumentException( - "Binary expression must have exactly two children, found: " + children.size()); - } - try { - ExprProtos.BinaryOpts opts = ExprProtos.BinaryOpts.parseFrom(metadata); - BinaryOp operator = BinaryOp.fromProto(opts.getOp()); - return new Binary(operator, children.get(0), children.get(1)); - } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to parse Binary metadata", e); - } - } - - /** - * Creates a new Binary expression with the specified operator and operands. - * - * @param operator the binary operator to apply - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression instance - */ - public static Binary of(BinaryOp operator, Expression left, Expression right) { - return new Binary(operator, left, right); - } - - /** - * Creates a logical AND expression combining multiple expressions. - * If only one expression is provided, it returns an AND with a literal true. - * Multiple expressions are right-associated: {@code first && (rest[0] && rest[1] && ...)}. - * - * @param first the first expression (left operand) - * @param rest additional expressions to AND together (right operands) - * @return a new Binary expression representing the logical AND operation - */ - public static Binary and(Expression first, Expression... rest) { - Expression rhs = Stream.of(rest).reduce(Binary::and).orElse(Literal.bool(true)); - return new Binary(BinaryOp.AND, first, rhs); - } - - /** - * Creates a logical OR expression combining multiple expressions. - * If only one expression is provided, it returns an OR with a literal false. - * Multiple expressions are right-associated: {@code first || (rest[0] || rest[1] || ...)}. - * - * @param first the first expression (left operand) - * @param rest additional expressions to OR together (right operands) - * @return a new Binary expression representing the logical OR operation - */ - public static Binary or(Expression first, Expression... rest) { - Expression rhs = Stream.of(rest).reduce(Binary::or).orElse(Literal.bool(false)); - return new Binary(BinaryOp.OR, first, rhs); - } - - /** - * Creates an equality comparison expression (==). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left == right - */ - public static Binary eq(Expression left, Expression right) { - return new Binary(BinaryOp.EQ, left, right); - } - - /** - * Creates an inequality comparison expression (!=). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left != right - */ - public static Binary notEq(Expression left, Expression right) { - return new Binary(BinaryOp.NOT_EQ, left, right); - } - - /** - * Creates a greater-than comparison expression (>). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left > right - */ - public static Binary gt(Expression left, Expression right) { - return new Binary(BinaryOp.GT, left, right); - } - - /** - * Creates a greater-than-or-equal comparison expression (>=). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left >= right - */ - public static Binary gtEq(Expression left, Expression right) { - return new Binary(BinaryOp.GT_EQ, left, right); - } - - /** - * Creates a less-than comparison expression (<). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left < right - */ - public static Binary lt(Expression left, Expression right) { - return new Binary(BinaryOp.LT, left, right); - } - - /** - * Creates a less-than-or-equal comparison expression (<=). - * - * @param left the left operand expression - * @param right the right operand expression - * @return a new Binary expression representing left <= right - */ - public static Binary ltEq(Expression left, Expression right) { - return new Binary(BinaryOp.LT_EQ, left, right); - } - - @Override - public String id() { - return "vortex.binary"; - } - - @Override - public List children() { - return List.of(left, right); - } - - @Override - public Optional metadata() { - return Optional.of(ExprProtos.BinaryOpts.newBuilder() - .setOp(operator.toProto()) - .build() - .toByteArray()); - } - - @Override - public String toString() { - return "(" + left + " " + operator + " " + right + ")"; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - Binary binary = (Binary) o; - return operator == binary.operator && Objects.equals(left, binary.left) && Objects.equals(right, binary.right); - } - - @Override - public int hashCode() { - return Objects.hash(operator, left, right); - } - - @Override - public T accept(Visitor visitor) { - return visitor.visitBinary(this); - } - - /** - * Returns the binary operator used in this expression. - * - * @return the binary operator - */ - public BinaryOp getOperator() { - return operator; - } - - /** - * Returns the left operand expression. - * - * @return the left operand expression - */ - public Expression getLeft() { - return left; - } - - /** - * Returns the right operand expression. - * - * @return the right operand expression - */ - public Expression getRight() { - return right; - } - - /** - * Enumeration of binary operators supported by binary expressions. - * Includes comparison operators and boolean algebra operators. - */ - public enum BinaryOp { - /** Equality comparison operator (==) */ - EQ, - /** Inequality comparison operator (!=) */ - NOT_EQ, - /** Greater-than comparison operator (>) */ - GT, - /** Greater-than-or-equal comparison operator (>=) */ - GT_EQ, - /** Less-than comparison operator (<) */ - LT, - /** Less-than-or-equal comparison operator (<=) */ - LT_EQ, - /** Logical AND operator (&&) */ - AND, - /** Logical OR operator (||) */ - OR, - ; - - @Override - public String toString() { - switch (this) { - case EQ: - return "=="; - case NOT_EQ: - return "!="; - case GT: - return ">"; - case GT_EQ: - return ">="; - case LT: - return "<"; - case LT_EQ: - return "<="; - case AND: - return "&&"; - case OR: - return "||"; - default: - throw new IllegalStateException("Unknown Operator: " + this); - } - } - - static BinaryOp fromProto(ExprProtos.BinaryOpts.BinaryOp proto) { - switch (proto) { - case Eq: - return EQ; - case NotEq: - return NOT_EQ; - case Gt: - return GT; - case Gte: - return GT_EQ; - case Lt: - return LT; - case Lte: - return LT_EQ; - case And: - return AND; - case Or: - return OR; - default: - throw new IllegalArgumentException("Unsupported binary operator proto: " + proto); - } - } - - ExprProtos.BinaryOpts.BinaryOp toProto() { - switch (this) { - case EQ: - return ExprProtos.BinaryOpts.BinaryOp.Eq; - case NOT_EQ: - return ExprProtos.BinaryOpts.BinaryOp.NotEq; - case GT: - return ExprProtos.BinaryOpts.BinaryOp.Gt; - case GT_EQ: - return ExprProtos.BinaryOpts.BinaryOp.Gte; - case LT: - return ExprProtos.BinaryOpts.BinaryOp.Lt; - case LT_EQ: - return ExprProtos.BinaryOpts.BinaryOp.Lte; - case AND: - return ExprProtos.BinaryOpts.BinaryOp.And; - case OR: - return ExprProtos.BinaryOpts.BinaryOp.Or; - default: - throw new IllegalArgumentException("Unsupported binary operator: " + this); - } - } - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/GetItem.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/GetItem.java deleted file mode 100644 index b150b78a7e2..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/GetItem.java +++ /dev/null @@ -1,111 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import com.google.protobuf.InvalidProtocolBufferException; -import dev.vortex.api.Expression; -import dev.vortex.proto.ExprProtos; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents a "get item" expression that extracts a field or property from a parent expression. - * This expression is used to access nested fields in complex data structures such as structs, - * lists, or other composite types by specifying a path to the desired field. - */ -public final class GetItem implements Expression { - private final String path; - private final Expression child; - - private GetItem(Expression child, String path) { - this.child = child; - this.path = path; - } - - /** - * Creates a new GetItem expression that extracts the specified field from the given child expression. - * - * @param child the parent expression from which to extract the field - * @param path the path or name of the field to extract - * @return a new GetItem expression - */ - public static GetItem of(Expression child, String path) { - return new GetItem(child, path); - } - - /** - * Parses a GetItem expression from serialized metadata and child expressions. - * This method is used during deserialization of Vortex expressions. - * - * @param metadata the serialized metadata containing the field path information - * @param children the child expressions, must contain exactly one element - * @return a new GetItem expression parsed from the provided data - * @throws RuntimeException if the number of children is not exactly one, - * or if the metadata cannot be parsed - */ - public static GetItem parse(byte[] metadata, List children) { - if (children.size() != 1) { - throw new IllegalArgumentException( - "GetItem expression must have exactly one child, found: " + children.size()); - } - try { - ExprProtos.GetItemOpts opts = ExprProtos.GetItemOpts.parseFrom(metadata); - return new GetItem(children.get(0), opts.getPath()); - } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to parse GetItem metadata", e); - } - } - - /** - * Returns the child expression from which the field is being extracted. - * - * @return the child expression - */ - public Expression getChild() { - return child; - } - - /** - * Returns the path or name of the field being extracted from the child expression. - * - * @return the field path - */ - public String getPath() { - return path; - } - - @Override - public String id() { - return "vortex.get_item"; - } - - @Override - public List children() { - return List.of(child); - } - - @Override - public Optional metadata() { - return Optional.of( - ExprProtos.GetItemOpts.newBuilder().setPath(path).build().toByteArray()); - } - - @Override - public T accept(Visitor visitor) { - return visitor.visitGetItem(this); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof GetItem)) return false; - GetItem getItem = (GetItem) o; - return Objects.equals(path, getItem.path); - } - - @Override - public int hashCode() { - return Objects.hashCode(path); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java deleted file mode 100644 index 05c9e03f5d9..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import dev.vortex.api.Expression; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents an IS NOT NULL expression that checks whether values are non-null. - * This expression returns true for non-null values and false for null values. - */ -public final class IsNotNull implements Expression { - private final Expression child; - - private IsNotNull(Expression child) { - this.child = child; - } - - /** - * Parses an IsNotNull expression from serialized metadata and child expressions. - * This method is used during deserialization of Vortex expressions. - * - * @param metadata the serialized metadata, must be empty for IsNotNull expressions - * @param children the child expressions, must contain exactly one element - * @return a new IsNotNull expression parsed from the provided data - * @throws IllegalArgumentException if the number of children is not exactly one, - * or if metadata is not empty - */ - public static IsNotNull parse(byte[] metadata, List children) { - if (children.size() != 1) { - throw new IllegalArgumentException( - "IsNotNull expression must have exactly one child, found: " + children.size()); - } - if (metadata.length > 0) { - throw new IllegalArgumentException( - "IsNotNull expression must not have metadata, found: " + metadata.length); - } - return new IsNotNull(children.get(0)); - } - - /** - * Creates a new IsNotNull expression that checks non-nullity of the given child expression. - * - * @param child the expression to check for non-null values - * @return a new IsNotNull expression - */ - public static IsNotNull of(Expression child) { - return new IsNotNull(child); - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - IsNotNull other = (IsNotNull) o; - return Objects.equals(child, other.child); - } - - @Override - public int hashCode() { - return Objects.hash(child); - } - - @Override - public String id() { - return "vortex.is_not_null"; - } - - @Override - public List children() { - return List.of(child); - } - - @Override - public Optional metadata() { - return Optional.of(new byte[] {}); - } - - @Override - public String toString() { - return "vortex.is_not_null(" + child + ")"; - } - - /** - * Returns the child expression that will be checked for non-null values. - * - * @return the child expression - */ - public Expression getChild() { - return child; - } - - @Override - public T accept(Visitor visitor) { - return visitor.visitIsNotNull(this); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java deleted file mode 100644 index b000fe88f2c..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNull.java +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import dev.vortex.api.Expression; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents an IS NULL expression that checks whether values are null. - * This expression returns true for null values and false for non-null values. - */ -public final class IsNull implements Expression { - private final Expression child; - - private IsNull(Expression child) { - this.child = child; - } - - /** - * Parses an IsNull expression from serialized metadata and child expressions. - * This method is used during deserialization of Vortex expressions. - * - * @param metadata the serialized metadata, must be empty for IsNull expressions - * @param children the child expressions, must contain exactly one element - * @return a new IsNull expression parsed from the provided data - * @throws IllegalArgumentException if the number of children is not exactly one, - * or if metadata is not empty - */ - public static IsNull parse(byte[] metadata, List children) { - if (children.size() != 1) { - throw new IllegalArgumentException( - "IsNull expression must have exactly one child, found: " + children.size()); - } - if (metadata.length > 0) { - throw new IllegalArgumentException("IsNull expression must not have metadata, found: " + metadata.length); - } - return new IsNull(children.get(0)); - } - - /** - * Creates a new IsNull expression that checks nullity of the given child expression. - * - * @param child the expression to check for null values - * @return a new IsNull expression - */ - public static IsNull of(Expression child) { - return new IsNull(child); - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - IsNull other = (IsNull) o; - return Objects.equals(child, other.child); - } - - @Override - public int hashCode() { - return Objects.hash(child); - } - - @Override - public String id() { - return "vortex.is_null"; - } - - @Override - public List children() { - return List.of(child); - } - - @Override - public Optional metadata() { - return Optional.of(new byte[] {}); - } - - @Override - public String toString() { - return "vortex.is_null(" + child + ")"; - } - - /** - * Returns the child expression that will be checked for null values. - * - * @return the child expression - */ - public Expression getChild() { - return child; - } - - @Override - public T accept(Visitor visitor) { - return visitor.visitIsNull(this); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Literal.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Literal.java deleted file mode 100644 index dac9476b5a5..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Literal.java +++ /dev/null @@ -1,1061 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import com.google.common.base.Preconditions; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import dev.vortex.api.Expression; -import dev.vortex.api.proto.EndianUtils; -import dev.vortex.api.proto.Scalars; -import dev.vortex.api.proto.TemporalMetadatas; -import dev.vortex.proto.DTypeProtos; -import dev.vortex.proto.ExprProtos; -import dev.vortex.proto.ScalarProtos; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents a literal value expression in the Vortex query system. - *

- * A literal is a constant value of a specific type that can be used in expressions. - * This class provides factory methods for creating literals of various types including - * primitives (boolean, integers, floats), temporal types (dates, times, timestamps), - * and complex types (strings, bytes, decimals). - *

- * Literals are immutable and implement the visitor pattern for type-safe processing. - * - * @param the Java type of the literal value - */ -public abstract class Literal implements Expression { - private final T value; - - private Literal(T value) { - this.value = value; - } - - /** - * Parses a literal from serialized metadata and child expressions. - *

- * This method deserializes a literal expression from its protocol buffer - * representation. Literal expressions must have no children. - * - * @param metadata the serialized literal metadata as bytes - * @param children the list of child expressions (must be empty for literals) - * @return the parsed literal expression - * @throws RuntimeException if children is not empty or if metadata cannot be parsed - */ - public static Literal parse(byte[] metadata, List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("Literal expression must have no children, found: " + children.size()); - } - try { - ExprProtos.LiteralOpts opts = ExprProtos.LiteralOpts.parseFrom(metadata); - return deserializeLiteral(opts, children); - } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to parse literal metadata", e); - } - } - - /** - * Returns the value of this literal. - * - * @return the literal value, may be null for null literals - */ - public T getValue() { - return this.value; - } - - @Override - public String id() { - return "vortex.literal"; - } - - @Override - public List children() { - return List.of(); - } - - @Override - public Optional metadata() { - return Optional.of(ExprProtos.LiteralOpts.newBuilder() - .setValue(this.acceptLiteralVisitor(LiteralToScalar.INSTANCE)) - .build() - .toByteArray()); - } - - @Override - public int hashCode() { - return Objects.hashCode(getValue()); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof Literal)) return false; - Literal literal = (Literal) o; - return Objects.equals(value, literal.value); - } - - /** - * Creates a null literal. - * - * @return a literal representing null - */ - public static Literal nullLit() { - return NullLiteral.INSTANCE; - } - - /** - * Creates a boolean literal. - * - * @param value the boolean value, may be null - * @return a boolean literal - */ - public static Literal bool(Boolean value) { - return new BooleanLiteral(value); - } - - /** - * Creates an 8-bit signed integer literal. - * - * @param value the byte value, may be null - * @return an int8 literal - */ - public static Literal int8(Byte value) { - return new Int8Literal(value); - } - - /** - * Creates a 16-bit signed integer literal. - * - * @param value the short value, may be null - * @return an int16 literal - */ - public static Literal int16(Short value) { - return new Int16Literal(value); - } - - /** - * Creates a 32-bit signed integer literal. - * - * @param value the integer value, may be null - * @return an int32 literal - */ - public static Literal int32(Integer value) { - return new Int32Literal(value); - } - - /** - * Creates a 64-bit signed integer literal. - * - * @param value the long value, may be null - * @return an int64 literal - */ - public static Literal int64(Long value) { - return new Int64Literal(value); - } - - /** - * Creates a 32-bit floating-point literal. - * - * @param value the float value, may be null - * @return a float32 literal - */ - public static Literal float32(Float value) { - return new Float32Literal(value); - } - - /** - * Creates a 64-bit floating-point literal. - * - * @param value the double value, may be null - * @return a float64 literal - */ - public static Literal float64(Double value) { - return new Float64Literal(value); - } - - /** - * Creates a decimal literal with specified precision and scale. - * - * @param value the decimal value, may be null - * @param precision the total number of digits - * @param scale the number of digits after the decimal point - * @return a decimal literal - * @throws RuntimeException if the value's scale doesn't match the specified scale - */ - public static Literal decimal(BigDecimal value, int precision, int scale) { - return new DecimalLiteral(value, precision, scale); - } - - /** - * Creates a string literal. - * - * @param value the string value, may be null - * @return a string literal - */ - public static Literal string(String value) { - return new StringLiteral(value); - } - - /** - * Creates a byte array literal. - * - * @param value the byte array value, may be null - * @return a bytes literal - */ - public static Literal bytes(byte[] value) { - return new BytesLiteral(value); - } - - /** - * Creates a time literal representing time of day in seconds. - * - * @param value the time value in seconds since midnight, may be null - * @return a time literal with second precision - */ - public static Literal timeSeconds(Integer value) { - return new TimeSeconds(value); - } - - /** - * Creates a time literal representing time of day in milliseconds. - * - * @param value the time value in milliseconds since midnight, may be null - * @return a time literal with millisecond precision - */ - public static Literal timeMillis(Integer value) { - return new TimeMillis(value); - } - - /** - * Creates a time literal representing time of day in microseconds. - * - * @param value the time value in microseconds since midnight, may be null - * @return a time literal with microsecond precision - */ - public static Literal timeMicros(Long value) { - return new TimeMicros(value); - } - - /** - * Creates a time literal representing time of day in nanoseconds. - * - * @param value the time value in nanoseconds since midnight, may be null - * @return a time literal with nanosecond precision - */ - public static Literal timeNanos(Long value) { - return new TimeNanos(value); - } - - /** - * Creates a date literal representing date as days since epoch. - * - * @param value the number of days since Unix epoch (1970-01-01), may be null - * @return a date literal with day precision - */ - public static Literal dateDays(Integer value) { - return new DateDays(value); - } - - /** - * Creates a date literal representing date as milliseconds since epoch. - * - * @param value the number of milliseconds since Unix epoch, may be null - * @return a date literal with millisecond precision - */ - public static Literal dateMillis(Long value) { - return new DateMillis(value); - } - - /** - * Creates a timestamp literal with millisecond precision. - * - * @param value the timestamp value in milliseconds since Unix epoch, may be null - * @param timeZone the time zone identifier (e.g., "UTC", "America/New_York"), optional - * @return a timestamp literal with millisecond precision - */ - public static Literal timestampMillis(Long value, Optional timeZone) { - return new TimestampMillis(value, timeZone); - } - - /** - * Creates a timestamp literal with microsecond precision. - * - * @param value the timestamp value in microseconds since Unix epoch, may be null - * @param timeZone the time zone identifier (e.g., "UTC", "America/New_York"), optional - * @return a timestamp literal with microsecond precision - */ - public static Literal timestampMicros(Long value, Optional timeZone) { - return new TimestampMicros(value, timeZone); - } - - /** - * Creates a timestamp literal with nanosecond precision. - * - * @param value the timestamp value in nanoseconds since Unix epoch, may be null - * @param timeZone the time zone identifier (e.g., "UTC", "America/New_York"), optional - * @return a timestamp literal with nanosecond precision - */ - public static Literal timestampNanos(Long value, Optional timeZone) { - return new TimestampNanos(value, timeZone); - } - - @Override - public R accept(Expression.Visitor visitor) { - return visitor.visitLiteral(this); - } - - /** - * Accepts a literal visitor to process this literal in a type-safe manner. - *

- * This method implements the visitor pattern, allowing different operations - * to be performed on literals based on their specific type. - * - * @param the return type of the visitor operation - * @param visitor the visitor to accept - * @return the result of the visitor operation - */ - public abstract U acceptLiteralVisitor(LiteralVisitor visitor); - - /** - * Visitor interface for processing literals in a type-safe manner. - *

- * This interface defines methods for visiting each type of literal that can - * be represented in the Vortex system. Implementations can provide type-specific - * processing logic while maintaining type safety. - * - * @param the return type of visitor methods - */ - public interface LiteralVisitor { - /** - * Visits a null literal. - * - * @return the result of processing the null literal - */ - U visitNull(); - - /** - * Visits a boolean literal. - * - * @param literal the boolean value, may be null - * @return the result of processing the boolean literal - */ - U visitBoolean(Boolean literal); - - /** - * Visits an 8-bit signed integer literal. - * - * @param literal the byte value, may be null - * @return the result of processing the int8 literal - */ - U visitInt8(Byte literal); - - /** - * Visits a 16-bit signed integer literal. - * - * @param literal the short value, may be null - * @return the result of processing the int16 literal - */ - U visitInt16(Short literal); - - /** - * Visits a 32-bit signed integer literal. - * - * @param literal the integer value, may be null - * @return the result of processing the int32 literal - */ - U visitInt32(Integer literal); - - /** - * Visits a 64-bit signed integer literal. - * - * @param literal the long value, may be null - * @return the result of processing the int64 literal - */ - U visitInt64(Long literal); - - /** - * Visits a date literal with day precision. - * - * @param days the number of days since Unix epoch, may be null - * @return the result of processing the date literal - */ - U visitDateDays(Integer days); - - /** - * Visits a date literal with millisecond precision. - * - * @param millis the number of milliseconds since Unix epoch, may be null - * @return the result of processing the date literal - */ - U visitDateMillis(Long millis); - - /** - * Visits a time literal with second precision. - * - * @param seconds the time in seconds since midnight, may be null - * @return the result of processing the time literal - */ - U visitTimeSeconds(Integer seconds); - - /** - * Visits a time literal with millisecond precision. - * - * @param seconds the time in milliseconds since midnight, may be null - * @return the result of processing the time literal - */ - U visitTimeMillis(Integer seconds); - - /** - * Visits a time literal with microsecond precision. - * - * @param seconds the time in microseconds since midnight, may be null - * @return the result of processing the time literal - */ - U visitTimeMicros(Long seconds); - - /** - * Visits a time literal with nanosecond precision. - * - * @param seconds the time in nanoseconds since midnight, may be null - * @return the result of processing the time literal - */ - U visitTimeNanos(Long seconds); - - /** - * Visits a timestamp literal with millisecond precision. - * - * @param epochMillis the timestamp in milliseconds since Unix epoch, may be null - * @param timeZone the time zone identifier, optional - * @return the result of processing the timestamp literal - */ - U visitTimestampMillis(Long epochMillis, Optional timeZone); - - /** - * Visits a timestamp literal with microsecond precision. - * - * @param epochMicros the timestamp in microseconds since Unix epoch, may be null - * @param timeZone the time zone identifier, optional - * @return the result of processing the timestamp literal - */ - U visitTimestampMicros(Long epochMicros, Optional timeZone); - - /** - * Visits a timestamp literal with nanosecond precision. - * - * @param epochNanos the timestamp in nanoseconds since Unix epoch, may be null - * @param timeZone the time zone identifier, optional - * @return the result of processing the timestamp literal - */ - U visitTimestampNanos(Long epochNanos, Optional timeZone); - - /** - * Visits a 32-bit floating-point literal. - * - * @param literal the float value, may be null - * @return the result of processing the float32 literal - */ - U visitFloat32(Float literal); - - /** - * Visits a 64-bit floating-point literal. - * - * @param literal the double value, may be null - * @return the result of processing the float64 literal - */ - U visitFloat64(Double literal); - - /** - * Visits a decimal literal with specified precision and scale. - * - * @param decimal the decimal value, may be null - * @param precision the total number of digits - * @param scale the number of digits after the decimal point - * @return the result of processing the decimal literal - */ - U visitDecimal(BigDecimal decimal, int precision, int scale); - - /** - * Visits a string literal. - * - * @param literal the string value, may be null - * @return the result of processing the string literal - */ - U visitString(String literal); - - /** - * Visits a byte array literal. - * - * @param literal the byte array value, may be null - * @return the result of processing the bytes literal - */ - U visitBytes(byte[] literal); - } - - static final class NullLiteral extends Literal { - static final NullLiteral INSTANCE = new NullLiteral(); - - private NullLiteral() { - super(null); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitNull(); - } - } - - static final class BooleanLiteral extends Literal { - BooleanLiteral(Boolean value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitBoolean(getValue()); - } - } - - static final class Int8Literal extends Literal { - Int8Literal(Byte value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitInt8(getValue()); - } - } - - static final class Int16Literal extends Literal { - Int16Literal(Short value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitInt16(getValue()); - } - } - - static final class Int32Literal extends Literal { - Int32Literal(Integer value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitInt32(getValue()); - } - } - - static final class Int64Literal extends Literal { - Int64Literal(Long value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitInt64(getValue()); - } - } - - static final class Float32Literal extends Literal { - Float32Literal(Float value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitFloat32(getValue()); - } - } - - static final class Float64Literal extends Literal { - Float64Literal(Double value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitFloat64(getValue()); - } - } - - static final class DecimalLiteral extends Literal { - private final int precision; - private final int scale; - - DecimalLiteral(BigDecimal value, int precision, int scale) { - super(value); - if (!Objects.isNull(value)) { - Preconditions.checkArgument(scale == value.scale(), "scale %s ≠ value scale %s", scale, value.scale()); - } - this.precision = precision; - this.scale = scale; - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitDecimal(getValue(), precision, scale); - } - } - - static final class StringLiteral extends Literal { - StringLiteral(String value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitString(getValue()); - } - } - - static final class BytesLiteral extends Literal { - BytesLiteral(byte[] value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitBytes(getValue()); - } - } - - static final class TimeSeconds extends Literal { - TimeSeconds(Integer value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimeSeconds(getValue()); - } - } - - static final class TimeMillis extends Literal { - TimeMillis(Integer value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimeMillis(getValue()); - } - } - - static final class TimeMicros extends Literal { - TimeMicros(Long value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimeMicros(getValue()); - } - } - - static final class TimeNanos extends Literal { - TimeNanos(Long value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimeNanos(getValue()); - } - } - - static final class DateDays extends Literal { - DateDays(Integer value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitDateDays(getValue()); - } - } - - static final class DateMillis extends Literal { - DateMillis(Long value) { - super(value); - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitDateMillis(getValue()); - } - } - - static final class TimestampMillis extends Literal { - private final Optional timeZone; - - TimestampMillis(Long value, Optional timeZone) { - super(value); - this.timeZone = timeZone; - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimestampMillis(getValue(), timeZone); - } - } - - static final class TimestampMicros extends Literal { - private final Optional timeZone; - - TimestampMicros(Long value, Optional timeZone) { - super(value); - this.timeZone = timeZone; - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimestampMicros(getValue(), timeZone); - } - } - - static final class TimestampNanos extends Literal { - private final Optional timeZone; - - TimestampNanos(Long value, Optional timeZone) { - super(value); - this.timeZone = timeZone; - } - - @Override - public U acceptLiteralVisitor(LiteralVisitor visitor) { - return visitor.visitTimestampNanos(getValue(), timeZone); - } - } - - static final class LiteralToScalar implements LiteralVisitor { - static final LiteralToScalar INSTANCE = new LiteralToScalar(); - - private LiteralToScalar() {} - - @Override - public ScalarProtos.Scalar visitNull() { - return Scalars.nullNull(); - } - - @Override - public ScalarProtos.Scalar visitBoolean(Boolean literal) { - if (Objects.isNull(literal)) { - return Scalars.nullBool(); - } else { - return Scalars.bool(literal); - } - } - - @Override - public ScalarProtos.Scalar visitInt8(Byte literal) { - if (Objects.isNull(literal)) { - return Scalars.nullInt8(); - } else { - return Scalars.int8(literal); - } - } - - @Override - public ScalarProtos.Scalar visitInt16(Short literal) { - if (Objects.isNull(literal)) { - return Scalars.nullInt16(); - } else { - return Scalars.int16(literal); - } - } - - @Override - public ScalarProtos.Scalar visitInt32(Integer literal) { - if (Objects.isNull(literal)) { - return Scalars.nullInt32(); - } else { - return Scalars.int32(literal); - } - } - - @Override - public ScalarProtos.Scalar visitInt64(Long literal) { - if (Objects.isNull(literal)) { - return Scalars.nullInt64(); - } else { - return Scalars.int64(literal); - } - } - - @Override - public ScalarProtos.Scalar visitDateDays(Integer days) { - if (Objects.isNull(days)) { - return Scalars.nullDateDays(); - } else { - return Scalars.dateDays(days); - } - } - - @Override - public ScalarProtos.Scalar visitDateMillis(Long millis) { - if (Objects.isNull(millis)) { - return Scalars.nullDateMillis(); - } else { - return Scalars.dateMillis(millis); - } - } - - @Override - public ScalarProtos.Scalar visitTimeSeconds(Integer seconds) { - if (Objects.isNull(seconds)) { - return Scalars.nullTimeSeconds(); - } else { - return Scalars.timeSeconds(seconds); - } - } - - @Override - public ScalarProtos.Scalar visitTimeMillis(Integer seconds) { - if (Objects.isNull(seconds)) { - return Scalars.nullTimeMillis(); - } else { - return Scalars.timeMillis(seconds); - } - } - - @Override - public ScalarProtos.Scalar visitTimeMicros(Long seconds) { - if (Objects.isNull(seconds)) { - return Scalars.nullTimeMicros(); - } else { - return Scalars.timeMicros(seconds); - } - } - - @Override - public ScalarProtos.Scalar visitTimeNanos(Long seconds) { - if (Objects.isNull(seconds)) { - return Scalars.nullTimeNanos(); - } else { - return Scalars.timeNanos(seconds); - } - } - - @Override - public ScalarProtos.Scalar visitTimestampMillis(Long epochMillis, Optional timeZone) { - if (Objects.isNull(epochMillis)) { - return Scalars.nullTimestampMillis(timeZone); - } else { - return Scalars.timestampMillis(epochMillis, timeZone); - } - } - - @Override - public ScalarProtos.Scalar visitTimestampMicros(Long epochMicros, Optional timeZone) { - if (Objects.isNull(epochMicros)) { - return Scalars.nullTimestampMicros(timeZone); - } else { - return Scalars.timestampMicros(epochMicros, timeZone); - } - } - - @Override - public ScalarProtos.Scalar visitTimestampNanos(Long epochNanos, Optional timeZone) { - if (Objects.isNull(epochNanos)) { - return Scalars.nullTimestampNanos(timeZone); - } else { - return Scalars.timestampNanos(epochNanos, timeZone); - } - } - - @Override - public ScalarProtos.Scalar visitFloat32(Float literal) { - if (Objects.isNull(literal)) { - return Scalars.nullFloat32(); - } else { - return Scalars.float32(literal); - } - } - - @Override - public ScalarProtos.Scalar visitFloat64(Double literal) { - if (Objects.isNull(literal)) { - return Scalars.nullFloat64(); - } else { - return Scalars.float64(literal); - } - } - - @Override - public ScalarProtos.Scalar visitDecimal(BigDecimal decimal, int precision, int scale) { - if (Objects.isNull(decimal)) { - return Scalars.nullDecimal(precision, scale); - } else { - return Scalars.decimal(decimal, precision, scale); - } - } - - @Override - public ScalarProtos.Scalar visitString(String literal) { - if (Objects.isNull(literal)) { - return Scalars.nullString(); - } else { - return Scalars.string(literal); - } - } - - @Override - public ScalarProtos.Scalar visitBytes(byte[] literal) { - if (Objects.isNull(literal)) { - return Scalars.nullBytes(); - } else { - return Scalars.bytes(literal); - } - } - } - - private static Literal deserializeLiteral(ExprProtos.LiteralOpts literal, List children) { - ScalarProtos.Scalar literalScalar = literal.getValue(); - DTypeProtos.DType dtype = literalScalar.getDtype(); - - // Special handling of extension types - if (dtype.hasExtension()) { - return deserializeExtensionLiteral(literal); - } - - ScalarProtos.ScalarValue scalarValue = literalScalar.getValue(); - - switch (scalarValue.getKindCase()) { - case NULL_VALUE: - return nullLiteral(dtype); - case BOOL_VALUE: - return Literal.bool(scalarValue.getBoolValue()); - case INT64_VALUE: - return Literal.int64(scalarValue.getInt64Value()); - case UINT64_VALUE: - return Literal.int64(scalarValue.getUint64Value()); - case F32_VALUE: - return Literal.float32(scalarValue.getF32Value()); - case F64_VALUE: - return Literal.float64(scalarValue.getF64Value()); - case STRING_VALUE: - return Literal.string(scalarValue.getStringValue()); - case BYTES_VALUE: - if (dtype.hasDecimal()) { - ByteString littleEndian = scalarValue.getBytesValue(); - byte[] bigEndian = EndianUtils.reverse(littleEndian); - BigDecimal value = new BigDecimal( - new BigInteger(bigEndian), dtype.getDecimal().getScale()); - return Literal.decimal( - value, - dtype.getDecimal().getPrecision(), - dtype.getDecimal().getScale()); - } else { - return Literal.bytes(scalarValue.getBytesValue().toByteArray()); - } - default: - throw new UnsupportedOperationException("Unsupported ScalarValue type encountered: " + scalarValue); - } - } - - private static Literal deserializeExtensionLiteral(ExprProtos.LiteralOpts literal) { - ScalarProtos.Scalar scalar = literal.getValue(); - DTypeProtos.DType scalarType = scalar.getDtype(); - - Preconditions.checkArgument(scalarType.hasExtension()); - - DTypeProtos.Extension extType = scalarType.getExtension(); - String extId = scalarType.getExtension().getId(); - - switch (extId) { - case "vortex.time": { - byte timeUnit = - TemporalMetadatas.getTimeUnit(extType.getMetadata().toByteArray()); - if (timeUnit == TemporalMetadatas.TIME_UNIT_SECONDS) { - return Literal.timeSeconds(Math.toIntExact(scalar.getValue().getInt64Value())); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_MILLIS) { - return Literal.timeMillis(Math.toIntExact(scalar.getValue().getInt64Value())); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_MICROS) { - return Literal.timeMicros(scalar.getValue().getInt64Value()); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_NANOS) { - return Literal.timeNanos(scalar.getValue().getInt64Value()); - } else { - throw new UnsupportedOperationException("Unsupported TIME time unit: " + timeUnit); - } - } - case "vortex.date": { - byte timeUnit = - TemporalMetadatas.getTimeUnit(extType.getMetadata().toByteArray()); - if (timeUnit == TemporalMetadatas.TIME_UNIT_DAYS) { - return Literal.dateDays(Math.toIntExact(scalar.getValue().getInt64Value())); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_MILLIS) { - return Literal.dateMillis(scalar.getValue().getInt64Value()); - } else { - throw new UnsupportedOperationException("Unsupported DATE time unit: " + timeUnit); - } - } - case "vortex.timestamp": { - byte timeUnit = - TemporalMetadatas.getTimeUnit(extType.getMetadata().toByteArray()); - Optional timeZone = - TemporalMetadatas.getTimeZone(extType.getMetadata().toByteArray()); - if (timeUnit == TemporalMetadatas.TIME_UNIT_MILLIS) { - return Literal.timestampMillis(scalar.getValue().getInt64Value(), timeZone); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_MICROS) { - return Literal.timestampMicros(scalar.getValue().getInt64Value(), timeZone); - } else if (timeUnit == TemporalMetadatas.TIME_UNIT_NANOS) { - return Literal.timestampNanos(scalar.getValue().getInt64Value(), timeZone); - } else { - throw new UnsupportedOperationException("Unsupported TIMESTAMP time unit: " + timeUnit); - } - } - default: - throw new UnsupportedOperationException("Unsupported extension type: " + extId); - } - } - - private static Literal nullLiteral(DTypeProtos.DType type) { - switch (type.getDtypeTypeCase()) { - case NULL: - return Literal.nullLit(); - case BOOL: - return Literal.bool(null); - case PRIMITIVE: - switch (type.getPrimitive().getType()) { - case U8: - case I8: - return Literal.int8(null); - case U16: - case I16: - return Literal.int16(null); - case U32: - case I32: - return Literal.int32(null); - case U64: - case I64: - return Literal.int64(null); - case F32: - return Literal.float32(null); - case F64: - return Literal.float64(null); - default: - throw new UnsupportedOperationException("Unsupported ScalarValue type encountered: " + type); - } - case DECIMAL: - return Literal.decimal( - null, - type.getDecimal().getPrecision(), - type.getDecimal().getScale()); - case UTF8: - return Literal.string(null); - case BINARY: - return Literal.bytes(null); - default: - throw new UnsupportedOperationException("Unsupported ScalarValue type encountered: " + type); - } - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Not.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Not.java deleted file mode 100644 index 9d2783ff5af..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Not.java +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import dev.vortex.api.Expression; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Represents a logical NOT expression that negates the boolean result of its child expression. - * This expression applies the logical NOT operation to the result of evaluating its single child expression. - */ -public final class Not implements Expression { - private final Expression child; - - private Not(Expression child) { - this.child = child; - } - - /** - * Parses a Not expression from serialized metadata and child expressions. - * This method is used during deserialization of Vortex expressions. - * - * @param metadata the serialized metadata, must be empty for Not expressions - * @param children the child expressions, must contain exactly one element - * @return a new Not expression parsed from the provided data - * @throws RuntimeException if the number of children is not exactly one, - * or if metadata is not empty - */ - public static Not parse(byte[] metadata, List children) { - if (children.size() != 1) { - throw new IllegalArgumentException("Not expression must have exactly one child, found: " + children.size()); - } - if (metadata.length > 0) { - throw new IllegalArgumentException("Not expression must not have metadata, found: " + metadata.length); - } - return new Not(children.get(0)); - } - - /** - * Creates a new Not expression that negates the given child expression. - * - * @param child the expression to negate - * @return a new Not expression - */ - public static Not of(Expression child) { - return new Not(child); - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - Not other = (Not) o; - return Objects.equals(child, other.child); - } - - @Override - public int hashCode() { - return Objects.hash(child); - } - - @Override - public String id() { - return "vortex.not"; - } - - @Override - public List children() { - return List.of(child); - } - - @Override - public Optional metadata() { - return Optional.of(new byte[] {}); - } - - @Override - public String toString() { - return "not(" + child + ")"; - } - - /** - * Returns the child expression that will be negated by this Not expression. - * - * @return the child expression - */ - public Expression getChild() { - return child; - } - - @Override - public T accept(Visitor visitor) { - return visitor.visitNot(this); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Root.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Root.java deleted file mode 100644 index ba8c0a28a59..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Root.java +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import dev.vortex.api.Expression; -import java.util.List; -import java.util.Optional; - -/** - * Represents the root expression in a Vortex expression tree. - * This is a singleton expression that serves as the starting point for expression evaluation, - * typically representing the root context or input data from which other expressions derive values. - */ -public final class Root implements Expression { - /** - * The singleton instance of the Root expression. - * Since Root expressions have no state or parameters, a single instance is shared. - */ - public static final Root INSTANCE = new Root(); - - private Root() {} - - /** - * Parses a Root expression from serialized metadata and child expressions. - * This method is used during deserialization of Vortex expressions. - * - * @param _metadata the serialized metadata (ignored for Root expressions) - * @param children the child expressions, must be empty for Root expressions - * @return the singleton Root instance - * @throws RuntimeException if any children are provided - */ - public static Root parse(byte[] _metadata, List children) { - if (!children.isEmpty()) { - throw new IllegalArgumentException("Root expression must have no children, found: " + children.size()); - } - return INSTANCE; - } - - @Override - public String id() { - return "vortex.root"; - } - - @Override - public List children() { - return List.of(); - } - - @Override - public Optional metadata() { - return Optional.of(new byte[] {}); - } - - @Override - public String toString() { - return "$"; - } - - // equals and hashCode depend on address equality to INSTANCE. - - @Override - public T accept(Visitor visitor) { - return visitor.visitRoot(this); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Unknown.java b/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Unknown.java deleted file mode 100644 index 8beedd06e5a..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/expressions/Unknown.java +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import dev.vortex.api.Expression; -import java.util.List; -import java.util.Optional; - -/** - * Represents a generic expression deserialized from a Vortex expression without a concrete Java type. - */ -public final class Unknown implements Expression { - private final String id; - private final List children; - private final byte[] metadata; - - /** - * Creates a new Unknown expression with the specified identifier, children, and metadata. - * This constructor is typically used when deserializing expressions that don't have - * a specific Java implementation, allowing them to be preserved as generic expressions. - * - * @param id the unique identifier for this expression type - * @param children the list of child expressions - * @param metadata the serialized metadata associated with this expression - */ - public Unknown(String id, List children, byte[] metadata) { - this.id = id; - this.children = children; - this.metadata = metadata; - } - - @Override - public String id() { - return id; - } - - @Override - public List children() { - return children; - } - - @Override - public Optional metadata() { - return Optional.of(metadata); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/DTypes.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/DTypes.java deleted file mode 100644 index 6ef609f9514..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/DTypes.java +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.proto; - -import static dev.vortex.api.proto.TemporalMetadatas.TIME_UNIT_MICROS; -import static dev.vortex.api.proto.TemporalMetadatas.TIME_UNIT_NANOS; - -import com.google.protobuf.ByteString; -import dev.vortex.proto.DTypeProtos; -import java.util.Optional; - -/** - * Factory class for creating Vortex data type definitions. - *

- * This class provides static factory methods to create {@link DTypeProtos.DType} instances - * for all supported Vortex data types. Data types in Vortex can be primitive types - * (like integers, floats, booleans), complex types (like strings, decimals, binary data), - * or extension types (like temporal types with specific metadata). - *

- *

- * Each data type can optionally be nullable, which affects how null values are handled - * during processing. The factory methods follow a consistent pattern where the boolean - * {@code nullable} parameter determines whether the type can contain null values. - *

- *

- * Extension types like dates, times, and timestamps are implemented as extensions - * over primitive storage types with additional metadata that defines their semantic - * meaning and encoding parameters. - *

- *

- * This class is primarily used internally by the {@link Scalars} factory class - * and other parts of the Vortex API to ensure consistent type definitions. - *

- */ -public final class DTypes { - private DTypes() {} - - static DTypeProtos.DType nullType() { - return DTypeProtos.DType.newBuilder() - .setNull(DTypeProtos.Null.newBuilder().build()) - .build(); - } - - static DTypeProtos.DType bool(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setBool(DTypeProtos.Bool.newBuilder().setNullable(nullable).build()) - .build(); - } - - static DTypeProtos.DType int8(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.I8) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType int16(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.I16) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType int32(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.I32) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType int64(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.I64) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType float32(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.F32) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType float64(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setPrimitive(DTypeProtos.Primitive.newBuilder() - .setType(DTypeProtos.PType.F64) - .setNullable(nullable) - .build()) - .build(); - } - - static DTypeProtos.DType decimal(boolean nullable, int precision, int scale) { - return DTypeProtos.DType.newBuilder() - .setDecimal(DTypeProtos.Decimal.newBuilder() - .setNullable(nullable) - .setPrecision(precision) - .setScale(scale) - .build()) - .build(); - } - - static DTypeProtos.DType string(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setUtf8(DTypeProtos.Utf8.newBuilder().setNullable(nullable).build()) - .build(); - } - - static DTypeProtos.DType binary(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setBinary(DTypeProtos.Binary.newBuilder().setNullable(nullable).build()) - .build(); - } - - static DTypeProtos.DType dateDays(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.date") - .setStorageDtype(DTypes.int32(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.DATE_DAYS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType dateMillis(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.date") - .setStorageDtype(DTypes.int64(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.DATE_MILLIS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType timeSeconds(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.time") - .setStorageDtype(DTypes.int32(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.TIME_SECONDS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType timeMillis(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.time") - .setStorageDtype(DTypes.int32(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.TIME_MILLIS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType timeMicros(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.time") - .setStorageDtype(DTypes.int64(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.TIME_MICROS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType timeNanos(boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.time") - .setStorageDtype(DTypes.int64(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.TIME_NANOS.get())) - .build()) - .build(); - } - - static DTypeProtos.DType timestampMillis(Optional timeZone, boolean nullable) { - return timestamp(TemporalMetadatas.TIME_UNIT_MILLIS, timeZone, nullable); - } - - static DTypeProtos.DType timestampMicros(Optional timeZone, boolean nullable) { - return timestamp(TIME_UNIT_MICROS, timeZone, nullable); - } - - static DTypeProtos.DType timestampNanos(Optional timeZone, boolean nullable) { - return timestamp(TIME_UNIT_NANOS, timeZone, nullable); - } - - private static DTypeProtos.DType timestamp(byte timeUnit, Optional timeZone, boolean nullable) { - return DTypeProtos.DType.newBuilder() - .setExtension(DTypeProtos.Extension.newBuilder() - .setId("vortex.timestamp") - .setStorageDtype(DTypes.int64(nullable)) - .setMetadata(ByteString.copyFrom(TemporalMetadatas.timestamp(timeUnit, timeZone))) - .build()) - .build(); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/EndianUtils.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/EndianUtils.java deleted file mode 100644 index 6855d92e454..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/EndianUtils.java +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.proto; - -import com.google.protobuf.ByteString; -import java.math.BigDecimal; -import java.math.BigInteger; - -/** - * Utility class for handling endianness conversions in Vortex protocol buffers. - * Provides methods for converting between big-endian and little-endian byte representations, - * particularly for decimal values that need to be serialized to the Vortex format. - */ -public final class EndianUtils { - /** - * Reverses the byte order of a ByteString, converting from one endianness to another. - * - * @param src the source ByteString to reverse - * @return a new byte array with bytes in reverse order - */ - public static byte[] reverse(ByteString src) { - byte[] dst = new byte[src.size()]; - for (int i = 0; i < dst.length; i++) { - dst[i] = src.byteAt(dst.length - 1 - i); - } - return dst; - } - - /** - * Converts a BigDecimal to a little-endian byte array representation suitable for Vortex decimals. - * The method extracts the unscaled value of the decimal, converts it from big-endian to little-endian, - * and pads to a standard size (1, 2, 4, 8, 16, or 32 bytes) with proper sign extension for negative values. - * - * @param decimal the BigDecimal value to convert - * @return a little-endian byte array representation of the decimal's unscaled value - * @throws RuntimeException if the BigDecimal is too large for Arrow - */ - public static byte[] littleEndianDecimal(BigDecimal decimal) { - BigInteger unscaled = decimal.unscaledValue(); - byte[] bigEndianBytes = unscaled.toByteArray(); - - // Determine target size (1, 2, 4, 8, 16, or 32 bytes) - int targetSize; - if (bigEndianBytes.length <= 1) { - targetSize = 1; - } else if (bigEndianBytes.length <= 2) { - targetSize = 2; - } else if (bigEndianBytes.length <= 4) { - targetSize = 4; - } else if (bigEndianBytes.length <= 8) { - targetSize = 8; - } else if (bigEndianBytes.length <= 16) { - targetSize = 16; - } else if (bigEndianBytes.length <= 32) { - targetSize = 32; - } else { - throw new IllegalArgumentException( - "BigDecimal with " + bigEndianBytes.length + " bytes overflows maximum Vortex decimal size"); - } - - byte[] result = new byte[targetSize]; - - // Copy bytes in reverse order (big endian to little endian) - for (int i = 0; i < bigEndianBytes.length; i++) { - result[i] = bigEndianBytes[bigEndianBytes.length - 1 - i]; - } - - // Sign extend if negative - if (unscaled.signum() < 0) { - for (int i = bigEndianBytes.length; i < targetSize; i++) { - result[i] = (byte) 0xFF; - } - } - - return result; - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java deleted file mode 100644 index 2e4d80691c6..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.proto; - -import com.google.protobuf.ByteString; -import dev.vortex.api.Expression; -import dev.vortex.api.expressions.*; -import dev.vortex.proto.ExprProtos; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Generate a protocol buffers representation of an {@link Expression}. - */ -public final class Expressions { - /** - * Serialize an {@link Expression} to a protocol buffer. - */ - public static ExprProtos.Expr serialize(Expression expression) { - ByteString metadata = ByteString.copyFrom(expression - .metadata() - .orElseThrow(() -> new IllegalArgumentException("Expression is not serializable: " + expression.id()))); - - return ExprProtos.Expr.newBuilder() - .setId(expression.id()) - .addAllChildren(expression.children().stream() - .map(Expressions::serialize) - .collect(Collectors.toList())) - .setMetadata(metadata) - .build(); - } - - /** - * Deserialize a protocol buffer representation back into an {@link Expression} object. - * The method examines the expression ID and creates the appropriate concrete expression type - * based on the registered expression types (binary, get_item, root, literal, not, is null, - * is not null). - * If the expression ID is not recognized, an {@link Unknown} expression is created. - * - * @param expr the protocol buffer expression to deserialize - * @return the deserialized Expression object - */ - public static Expression deserialize(ExprProtos.Expr expr) { - byte[] metadata = expr.getMetadata().toByteArray(); - List children = - expr.getChildrenList().stream().map(Expressions::deserialize).collect(Collectors.toList()); - - switch (expr.getId()) { - case "vortex.binary": - return Binary.parse(metadata, children); - case "vortex.get_item": - return GetItem.parse(metadata, children); - case "vortex.root": - return Root.parse(metadata, children); - case "vortex.literal": - return Literal.parse(metadata, children); - case "vortex.not": - return Not.parse(metadata, children); - case "vortex.is_null": - return IsNull.parse(metadata, children); - case "vortex.is_not_null": - return IsNotNull.parse(metadata, children); - default: - return new Unknown(expr.getId(), children, expr.getMetadata().toByteArray()); - } - } - - private Expressions() {} -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Scalars.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/Scalars.java deleted file mode 100644 index 40ea7b09880..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/Scalars.java +++ /dev/null @@ -1,443 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.proto; - -import com.google.protobuf.ByteString; -import com.google.protobuf.NullValue; -import dev.vortex.proto.ScalarProtos; -import java.math.BigDecimal; -import java.util.Optional; - -/** - * Factory class for creating Vortex scalar values with their associated data types. - *

- * This class provides static factory methods to create {@link ScalarProtos.Scalar} instances - * for all supported Vortex data types. Each scalar consists of a value and its corresponding - * data type definition. The class supports both nullable and non-nullable variants for most types. - *

- *

- * Factory methods follow a consistent naming pattern: - *

- *
    - *
  • {@code typeName(value)} - Creates a non-nullable scalar with the given value
  • - *
  • {@code nullTypeName()} - Creates a nullable scalar with a null value
  • - *
- *

- * Example usage: - *

- *
{@code
- * // Create a non-nullable integer
- * ScalarProtos.Scalar intScalar = Scalars.int32(42);
- *
- * // Create a nullable string with null value
- * ScalarProtos.Scalar nullString = Scalars.nullString();
- *
- * // Create a string with value
- * ScalarProtos.Scalar stringScalar = Scalars.string("hello");
- * }
- */ -public final class Scalars { - private Scalars() {} - - /** - * Creates a null scalar value. - *

- * This represents a scalar that is explicitly null with null data type. - *

- * - * @return a {@link ScalarProtos.Scalar} representing a null value - */ - public static ScalarProtos.Scalar nullNull() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.nullType()) - .build(); - } - - /** - * Creates a non-nullable boolean scalar with the specified value. - * - * @param value the boolean value - * @return a {@link ScalarProtos.Scalar} containing the boolean value - */ - public static ScalarProtos.Scalar bool(boolean value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setBoolValue(value) - .build()) - .setDtype(DTypes.bool(false)) - .build(); - } - - /** - * Creates a nullable boolean scalar with a null value. - * - * @return a {@link ScalarProtos.Scalar} representing a null boolean - */ - public static ScalarProtos.Scalar nullBool() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.bool(true)) - .build(); - } - - public static ScalarProtos.Scalar int8(byte value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.int8(false)) - .build(); - } - - public static ScalarProtos.Scalar nullInt8() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.int8(true)) - .build(); - } - - public static ScalarProtos.Scalar int16(short value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.int16(false)) - .build(); - } - - public static ScalarProtos.Scalar nullInt16() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.int16(true)) - .build(); - } - - /** - * Creates a non-nullable 32-bit integer scalar with the specified value. - * - * @param value the integer value - * @return a {@link ScalarProtos.Scalar} containing the 32-bit integer value - */ - public static ScalarProtos.Scalar int32(int value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.int32(false)) - .build(); - } - - public static ScalarProtos.Scalar nullInt32() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.int32(true)) - .build(); - } - - public static ScalarProtos.Scalar int64(long value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.int64(false)) - .build(); - } - - public static ScalarProtos.Scalar nullInt64() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.int64(true)) - .build(); - } - - public static ScalarProtos.Scalar float32(float value) { - return ScalarProtos.Scalar.newBuilder() - .setValue( - ScalarProtos.ScalarValue.newBuilder().setF32Value(value).build()) - .setDtype(DTypes.float32(false)) - .build(); - } - - public static ScalarProtos.Scalar nullFloat32() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.float32(true)) - .build(); - } - - public static ScalarProtos.Scalar float64(double value) { - return ScalarProtos.Scalar.newBuilder() - .setValue( - ScalarProtos.ScalarValue.newBuilder().setF64Value(value).build()) - .setDtype(DTypes.float64(false)) - .build(); - } - - public static ScalarProtos.Scalar nullFloat64() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.float64(true)) - .build(); - } - - /** - * Creates a non-nullable string scalar with the specified value. - * - * @param value the string value - * @return a {@link ScalarProtos.Scalar} containing the string value - */ - public static ScalarProtos.Scalar string(String value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setStringValue(value) - .build()) - .setDtype(DTypes.string(false)) - .build(); - } - - public static ScalarProtos.Scalar nullString() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.string(true)) - .build(); - } - - /** - * Creates a non-nullable decimal scalar with the specified value, precision, and scale. - *

- * The decimal value is converted to little-endian byte representation for storage. - *

- * - * @param decimal the decimal value as a {@link BigDecimal} - * @param precision the total number of digits in the decimal - * @param scale the number of digits after the decimal point - * @return a {@link ScalarProtos.Scalar} containing the decimal value - */ - public static ScalarProtos.Scalar decimal(BigDecimal decimal, int precision, int scale) { - var littleEndian = EndianUtils.littleEndianDecimal(decimal); - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setBytesValue(ByteString.copyFrom(littleEndian)) - .build()) - .setDtype(DTypes.decimal(false, precision, scale)) - .build(); - } - - public static ScalarProtos.Scalar nullDecimal(int precision, int scale) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder().setNullValue(NullValue.NULL_VALUE)) - .setDtype(DTypes.decimal(true, precision, scale)) - .build(); - } - - public static ScalarProtos.Scalar bytes(byte[] value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setBytesValue(ByteString.copyFrom(value)) - .build()) - .setDtype(DTypes.binary(false)) - .build(); - } - - public static ScalarProtos.Scalar nullBytes() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.binary(true)) - .build(); - } - - public static ScalarProtos.Scalar dateDays(int value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.dateDays(false)) - .build(); - } - - public static ScalarProtos.Scalar nullDateDays() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.dateDays(true)) - .build(); - } - - public static ScalarProtos.Scalar dateMillis(long value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.dateMillis(false)) - .build(); - } - - public static ScalarProtos.Scalar nullDateMillis() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.dateMillis(true)) - .build(); - } - - public static ScalarProtos.Scalar timeSeconds(int value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timeSeconds(false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimeSeconds() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timeSeconds(true)) - .build(); - } - - public static ScalarProtos.Scalar timeMillis(int value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timeMillis(false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimeMillis() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timeMillis(true)) - .build(); - } - - public static ScalarProtos.Scalar timeMicros(long value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timeMicros(false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimeMicros() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timeMicros(true)) - .build(); - } - - public static ScalarProtos.Scalar timeNanos(long value) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timeNanos(false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimeNanos() { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timeNanos(true)) - .build(); - } - - /** - * Creates a non-nullable timestamp scalar with millisecond precision. - *

- * The timestamp represents the number of milliseconds since Unix epoch - * (1970-01-01 00:00:00 UTC), optionally adjusted for the specified time zone. - *

- * - * @param value the timestamp value in milliseconds since Unix epoch - * @param timeZone optional time zone identifier (e.g., "UTC", "America/New_York") - * @return a {@link ScalarProtos.Scalar} containing the timestamp value - */ - public static ScalarProtos.Scalar timestampMillis(long value, Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timestampMillis(timeZone, false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimestampMillis(Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timestampMillis(timeZone, true)) - .build(); - } - - public static ScalarProtos.Scalar timestampMicros(long value, Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timestampMicros(timeZone, false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimestampMicros(Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timestampMicros(timeZone, true)) - .build(); - } - - public static ScalarProtos.Scalar timestampNanos(long value, Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setInt64Value(value) - .build()) - .setDtype(DTypes.timestampNanos(timeZone, false)) - .build(); - } - - public static ScalarProtos.Scalar nullTimestampNanos(Optional timeZone) { - return ScalarProtos.Scalar.newBuilder() - .setValue(ScalarProtos.ScalarValue.newBuilder() - .setNullValue(NullValue.NULL_VALUE) - .build()) - .setDtype(DTypes.timestampNanos(timeZone, true)) - .build(); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/proto/TemporalMetadatas.java b/java/vortex-jni/src/main/java/dev/vortex/api/proto/TemporalMetadatas.java deleted file mode 100644 index d5d5756f98e..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/api/proto/TemporalMetadatas.java +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.proto; - -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -/** - * Utility class for creating and parsing temporal metadata in Vortex protocol buffers. - * Provides constants for time units and methods for serializing/deserializing timestamp - * metadata including time zone information. This class handles the binary format used - * to encode temporal type information in the Vortex data format. - */ -public final class TemporalMetadatas { - private TemporalMetadatas() {} - - /** Time unit constant representing nanoseconds precision. */ - public static byte TIME_UNIT_NANOS = 0; - /** Time unit constant representing microseconds precision. */ - public static byte TIME_UNIT_MICROS = 1; - /** Time unit constant representing milliseconds precision. */ - public static byte TIME_UNIT_MILLIS = 2; - /** Time unit constant representing seconds precision. */ - public static byte TIME_UNIT_SECONDS = 3; - /** Time unit constant representing days precision. */ - public static byte TIME_UNIT_DAYS = 4; - - /** Supplier for date metadata with days precision. */ - public static final Supplier DATE_DAYS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_DAYS}); - /** Supplier for date metadata with milliseconds precision. */ - public static final Supplier DATE_MILLIS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_MILLIS}); - /** Supplier for time metadata with seconds precision. */ - public static final Supplier TIME_SECONDS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_SECONDS}); - /** Supplier for time metadata with milliseconds precision. */ - public static final Supplier TIME_MILLIS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_MILLIS}); - /** Supplier for time metadata with microseconds precision. */ - public static final Supplier TIME_MICROS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_MICROS}); - /** Supplier for time metadata with nanoseconds precision. */ - public static final Supplier TIME_NANOS = Suppliers.memoize(() -> new byte[] {TIME_UNIT_NANOS}); - - /** - * Creates serialized timestamp metadata with the specified time unit and optional time zone. - * The resulting byte array contains the time unit as the first byte, followed by a little-endian - * uint16 length field, and then the UTF-8 encoded time zone string (if present). - * - * @param timeUnit the time unit for the timestamp, must be between TIME_UNIT_NANOS and TIME_UNIT_SECONDS (exclusive of TIME_UNIT_DAYS) - * @param timeZone optional time zone identifier string - * @return serialized timestamp metadata as a byte array - * @throws RuntimeException if timeUnit is invalid for timestamps - */ - public static byte[] timestamp(byte timeUnit, Optional timeZone) { - Preconditions.checkArgument( - timeUnit >= TIME_UNIT_NANOS && timeUnit < TIME_UNIT_DAYS, "invalid timeUnit for Timestamp:" + timeUnit); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(timeUnit); - if (timeZone.isPresent()) { - byte[] timeZoneBytes = timeZone.get().getBytes(StandardCharsets.UTF_8); - // Write length as little-endian uint16. - int lenLow = timeZoneBytes.length & 0xFF; - int lenHigh = (timeZoneBytes.length >> 8) & 0xFF; - baos.write(lenLow); - baos.write(lenHigh); - baos.writeBytes(timeZoneBytes); - } else { - // write uint16 zero value - baos.write(0); - baos.write(0); - } - return baos.toByteArray(); - } - - /** - * Extract the time unit byte from the serialized metadata. - * - * @param serializedMetadata the serialized temporal metadata byte array - * @return the time unit byte from the first position of the metadata - * @throws RuntimeException if the time unit byte is invalid - */ - public static byte getTimeUnit(byte[] serializedMetadata) { - byte timeUnit = serializedMetadata[0]; - Preconditions.checkArgument( - timeUnit >= TIME_UNIT_NANOS && timeUnit <= TIME_UNIT_DAYS, "invalid timeUnit byte: " + timeUnit); - - return timeUnit; - } - - /** - * Read from a serialized metadata representation into a time zone string. - * Parses the time zone information from the serialized metadata, reading the length - * as a little-endian uint16 and then decoding the UTF-8 time zone string. - * - * @param serializedMetadata the serialized temporal metadata byte array - * @return Optional containing the time zone string, or empty if no time zone was specified - */ - public static Optional getTimeZone(byte[] serializedMetadata) { - byte _timeUnit = serializedMetadata[0]; - byte lenLow = serializedMetadata[1]; - byte lenHigh = serializedMetadata[2]; - int len = ((lenHigh & 0xFF) << 8) | (lenLow & 0xFF); - if (len == 0) { - return Optional.empty(); - } else { - byte[] timeZoneBytes = new byte[len]; - System.arraycopy(serializedMetadata, 3, timeZoneBytes, 0, len); - return Optional.of(new String(timeZoneBytes, StandardCharsets.UTF_8)); - } - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArray.java b/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArray.java deleted file mode 100644 index db019339dac..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArray.java +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import com.google.common.base.Preconditions; -import dev.vortex.api.Array; -import dev.vortex.api.DType; -import java.math.BigDecimal; -import java.util.OptionalLong; -import org.apache.arrow.c.ArrowArray; -import org.apache.arrow.c.ArrowSchema; -import org.apache.arrow.c.CDataDictionaryProvider; -import org.apache.arrow.c.Data; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.VectorSchemaRoot; - -public final class JNIArray implements Array { - static { - NativeLoader.loadJni(); - } - - private final ThreadLocal schemaPtr = ThreadLocal.withInitial(() -> new long[1]); - private final ThreadLocal arrayPtr = ThreadLocal.withInitial(() -> new long[1]); - - private OptionalLong pointer; - - public JNIArray(long pointer) { - Preconditions.checkArgument(pointer > 0, "Invalid pointer address: " + pointer); - this.pointer = OptionalLong.of(pointer); - } - - @Override - public long getLen() { - return NativeArrayMethods.getLen(pointer.getAsLong()); - } - - @Override - public long nbytes() { - return NativeArrayMethods.nbytes(pointer.getAsLong()); - } - - @Override - public VectorSchemaRoot exportToArrow(BufferAllocator allocator, VectorSchemaRoot reuse) { - // Export the dataset to Arrow over C Data Interface. - NativeArrayMethods.exportToArrow(pointer.getAsLong(), schemaPtr.get(), arrayPtr.get()); - try (ArrowSchema arrowSchema = ArrowSchema.wrap(schemaPtr.get()[0]); - ArrowArray arrowArray = ArrowArray.wrap(arrayPtr.get()[0]); - CDataDictionaryProvider provider = new CDataDictionaryProvider()) { - if (reuse != null) { - Data.importIntoVectorSchemaRoot(allocator, arrowArray, reuse, provider); - return reuse; - } else { - return Data.importVectorSchemaRoot(allocator, arrowArray, arrowSchema, new CDataDictionaryProvider()); - } - } finally { - NativeArrayMethods.dropArrowSchema(schemaPtr.get()[0]); - NativeArrayMethods.dropArrowArray(arrayPtr.get()[0]); - } - } - - @Override - public DType getDataType() { - return new JNIDType(NativeArrayMethods.getDataType(pointer.getAsLong())); - } - - @Override - public Array getField(int index) { - return new JNIArray(NativeArrayMethods.getField(pointer.getAsLong(), index)); - } - - @Override - public Array slice(int start, int stop) { - return new JNIArray(NativeArrayMethods.slice(pointer.getAsLong(), start, stop)); - } - - @Override - public boolean getNull(int index) { - return NativeArrayMethods.getNull(pointer.getAsLong(), index); - } - - @Override - public int getNullCount() { - return NativeArrayMethods.getNullCount(pointer.getAsLong()); - } - - @Override - public byte getByte(int index) { - return NativeArrayMethods.getByte(pointer.getAsLong(), index); - } - - @Override - public short getShort(int index) { - return NativeArrayMethods.getShort(pointer.getAsLong(), index); - } - - @Override - public int getInt(int index) { - return NativeArrayMethods.getInt(pointer.getAsLong(), index); - } - - @Override - public long getLong(int index) { - return NativeArrayMethods.getLong(pointer.getAsLong(), index); - } - - @Override - public boolean getBool(int index) { - return NativeArrayMethods.getBool(pointer.getAsLong(), index); - } - - @Override - public float getFloat(int index) { - return NativeArrayMethods.getFloat(pointer.getAsLong(), index); - } - - @Override - public double getDouble(int index) { - return NativeArrayMethods.getDouble(pointer.getAsLong(), index); - } - - @Override - public BigDecimal getBigDecimal(int index) { - return NativeArrayMethods.getBigDecimal(pointer.getAsLong(), index); - } - - @Override - public String getUTF8(int index) { - return NativeArrayMethods.getUTF8(pointer.getAsLong(), index); - } - - @Override - public void getUTF8_ptr_len(int index, long[] ptr, int[] len) { - NativeArrayMethods.getUTF8_ptr_len(pointer.getAsLong(), index, ptr, len); - } - - @Override - public byte[] getBinary(int index) { - return NativeArrayMethods.getBinary(pointer.getAsLong(), index); - } - - @Override - public void close() { - if (pointer.isEmpty()) { - return; - } - - NativeArrayMethods.free(pointer.getAsLong()); - pointer = OptionalLong.empty(); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArrayIterator.java b/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArrayIterator.java deleted file mode 100644 index 33f0661e9cc..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIArrayIterator.java +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import com.google.common.base.Preconditions; -import dev.vortex.api.Array; -import dev.vortex.api.ArrayIterator; -import dev.vortex.api.DType; -import java.util.Optional; -import java.util.OptionalLong; - -public final class JNIArrayIterator implements ArrayIterator { - private OptionalLong pointer; - private Optional next; - - public JNIArrayIterator(long pointer) { - Preconditions.checkArgument(pointer > 0, "Invalid pointer address: " + pointer); - this.pointer = OptionalLong.of(pointer); - advance(); - } - - @Override - public boolean hasNext() { - return next.isPresent(); - } - - @Override - public Array next() { - Array array = this.next.get(); - advance(); - return array; - } - - @Override - public DType getDataType() { - return new JNIDType(NativeArrayIteratorMethods.getDType(pointer.getAsLong())); - } - - @Override - public void close() { - if (pointer.isEmpty()) { - return; - } - - NativeArrayIteratorMethods.free(pointer.getAsLong()); - pointer = OptionalLong.empty(); - next = Optional.empty(); - } - - private void advance() { - long next = NativeArrayIteratorMethods.take(pointer.getAsLong()); - if (next <= 0) { - this.next = Optional.empty(); - } else { - this.next = Optional.of(new JNIArray(next)); - } - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIDType.java b/java/vortex-jni/src/main/java/dev/vortex/jni/JNIDType.java deleted file mode 100644 index d19dc0624e7..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIDType.java +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import dev.vortex.api.DType; -import java.util.List; -import java.util.Optional; -import java.util.OptionalLong; - -public final class JNIDType implements DType { - OptionalLong pointer; - final boolean isOwned; // True if this object owns the native memory - - /** - * Creates a JNIDType that borrows a native pointer. - * The caller is responsible for ensuring the pointer remains valid. - */ - public JNIDType(long pointer) { - this(pointer, false); - } - - public long getPointer() { - return pointer.getAsLong(); - } - - /** - * Creates a JNIDType with explicit ownership. - * - * @param pointer the native pointer - * @param isOwned true if this object owns the native memory and should free it - */ - public JNIDType(long pointer, boolean isOwned) { - Preconditions.checkArgument(pointer > 0, "Invalid pointer address: " + pointer); - this.pointer = OptionalLong.of(pointer); - this.isOwned = isOwned; - } - - @Override - public Variant getVariant() { - return Variant.from(NativeDTypeMethods.getVariant(pointer.getAsLong())); - } - - @Override - public boolean isNullable() { - return NativeDTypeMethods.isNullable(pointer.getAsLong()); - } - - @Override - public List getFieldNames() { - return NativeDTypeMethods.getFieldNames(pointer.getAsLong()); - } - - @Override - public List getFieldTypes() { - return Lists.transform(NativeDTypeMethods.getFieldTypes(pointer.getAsLong()), JNIDType::new); - } - - @Override - public DType getElementType() { - // Returns a borrowed reference - the parent DType owns this memory - return new JNIDType(NativeDTypeMethods.getElementType(pointer.getAsLong())); - } - - @Override - public int getFixedSizeListSize() { - return NativeDTypeMethods.getFixedSizeListSize(pointer.getAsLong()); - } - - @Override - public boolean isDate() { - return NativeDTypeMethods.isDate(pointer.getAsLong()); - } - - @Override - public boolean isTime() { - return NativeDTypeMethods.isTime(pointer.getAsLong()); - } - - @Override - public boolean isTimestamp() { - return NativeDTypeMethods.isTimestamp(pointer.getAsLong()); - } - - @Override - public TimeUnit getTimeUnit() { - return TimeUnit.from(NativeDTypeMethods.getTimeUnit(pointer.getAsLong())); - } - - @Override - public Optional getTimeZone() { - return Optional.ofNullable(NativeDTypeMethods.getTimeZone(pointer.getAsLong())); - } - - @Override - public boolean isDecimal() { - return NativeDTypeMethods.isDecimal(pointer.getAsLong()); - } - - @Override - public int getPrecision() { - return NativeDTypeMethods.getDecimalPrecision(pointer.getAsLong()); - } - - @Override - public byte getScale() { - return NativeDTypeMethods.getDecimalScale(pointer.getAsLong()); - } - - @Override - public void close() { - if (isOwned && pointer.isPresent()) { - NativeDTypeMethods.free(pointer.getAsLong()); - pointer = OptionalLong.empty(); - } - } - - /** - * Creates a JNIDType that owns its native memory. - * This should only be used when receiving a newly allocated pointer from Rust. - */ - public static JNIDType ownedDType(long pointer) { - return new JNIDType(pointer, true); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIFile.java b/java/vortex-jni/src/main/java/dev/vortex/jni/JNIFile.java deleted file mode 100644 index d26c20bd18b..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIFile.java +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import com.google.common.base.Preconditions; -import dev.vortex.api.ArrayIterator; -import dev.vortex.api.DType; -import dev.vortex.api.File; -import dev.vortex.api.ScanOptions; -import dev.vortex.api.proto.Expressions; -import java.util.OptionalLong; - -public final class JNIFile implements File { - private OptionalLong pointer; - - public JNIFile(long pointer) { - Preconditions.checkArgument(pointer > 0, "Invalid pointer address: " + pointer); - this.pointer = OptionalLong.of(pointer); - } - - @Override - public DType getDType() { - return new JNIDType(NativeFileMethods.dtype(pointer.getAsLong())); - } - - @Override - public long rowCount() { - return NativeFileMethods.rowCount(pointer.getAsLong()); - } - - @Override - public ArrayIterator newScan(ScanOptions options) { - byte[] predicateProto = null; - - if (options.predicate().isPresent()) { - predicateProto = Expressions.serialize(options.predicate().get()).toByteArray(); - } - - long[] rowIndices = options.rowIndices().orElse(null); - long[] rowRange = options.rowRange().orElse(null); - - return new JNIArrayIterator( - NativeFileMethods.scan(pointer.getAsLong(), options.columns(), predicateProto, rowRange, rowIndices)); - } - - @Override - public void close() { - if (pointer.isEmpty()) { - return; - } - NativeFileMethods.close(pointer.getAsLong()); - pointer = OptionalLong.empty(); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIWriter.java b/java/vortex-jni/src/main/java/dev/vortex/jni/JNIWriter.java deleted file mode 100644 index 6b0c62ca091..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/JNIWriter.java +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import dev.vortex.api.VortexWriter; -import java.io.IOException; -import java.util.OptionalLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * JNI implementation of VortexWriter. - *

- * This class implements AutoCloseable to ensure proper resource cleanup - * when used with try-with-resources. - */ -public final class JNIWriter implements VortexWriter, AutoCloseable { - private static final Logger logger = LoggerFactory.getLogger(JNIWriter.class); - - private OptionalLong ptr; - - /** - * Creates a new JNIWriter with the given native pointer. - * - * @param ptr the native writer pointer - */ - public JNIWriter(long ptr) { - this.ptr = OptionalLong.of(ptr); - logger.debug("Created JNIWriter with ptr={}", ptr); - } - - /** - * Writes a batch of Arrow data to the Vortex file. - * - * @param arrowData the Arrow data in IPC format as byte array - * @throws NullPointerException if this is called after the writer has been closed. - */ - @Override - public void writeBatch(byte[] arrowData) throws IOException { - logger.trace("Writing batch with {} bytes", arrowData.length); - - // Write the Arrow data to Vortex through JNI - boolean success = NativeWriterMethods.writeBatch(ptr.getAsLong(), arrowData); - if (!success) { - logger.error("Failed to write batch to Vortex file"); - throw new IOException("Failed to write batch to Vortex file"); - } - } - - /** - * Writes a batch of Arrow data directly from Arrow C Data Interface pointers. - * - * @param arrowArrayAddr memory address of the ArrowArray struct - * @param arrowSchemaAddr memory address of the ArrowSchema struct - * @throws IOException if writing fails - */ - @Override - public void writeBatchFfi(long arrowArrayAddr, long arrowSchemaAddr) throws IOException { - logger.trace("Writing batch via FFI (arrayAddr={}, schemaAddr={})", arrowArrayAddr, arrowSchemaAddr); - - boolean success = NativeWriterMethods.writeBatchFfi(ptr.getAsLong(), arrowArrayAddr, arrowSchemaAddr); - if (!success) { - logger.error("Failed to write FFI batch to Vortex file"); - throw new IOException("Failed to write FFI batch to Vortex file"); - } - } - - /** - * Closes the writer and finalizes the Vortex file. - * - * @throws RuntimeException if closing fails - */ - @Override - public void close() { - if (this.ptr.isEmpty()) { - logger.debug("Attempted to close already closed JNIWriter, skipping"); - return; - } - - long ptr = this.ptr.getAsLong(); - - logger.debug("Closing JNIWriter with ptr={}", ptr); - NativeWriterMethods.close(ptr); - this.ptr = OptionalLong.empty(); - } -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayIteratorMethods.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayIteratorMethods.java deleted file mode 100644 index 9fb2a366b75..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayIteratorMethods.java +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -public final class NativeArrayIteratorMethods { - static { - NativeLoader.loadJni(); - } - - private NativeArrayIteratorMethods() {} - - /** - * Free all resources associated with the stream behind the pointer. - */ - public static native void free(long pointer); - - /** - * Returns a pointer to the next element in the stream, or -1 if there are no more elements. - *

- * An exception is thrown if the stream is closed, either via free or due to a previous call - * to this method returning -1. - */ - public static native long take(long pointer); - - /** - * Get a pointer to the DType of the elements of the stream. - */ - public static native long getDType(long pointer); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayMethods.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayMethods.java deleted file mode 100644 index d6f19c2ea83..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeArrayMethods.java +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import java.math.BigDecimal; - -public final class NativeArrayMethods { - static { - NativeLoader.loadJni(); - } - - private NativeArrayMethods() {} - - public static native long nbytes(long pointer); - - public static native void exportToArrow(long pointer, long[] schemaPointer, long[] arrayPointer); - - public static native void dropArrowSchema(long arrowSchemaPtr); - - public static native void dropArrowArray(long arrowArrayPtr); - - public static native void free(long pointer); - - public static native long getLen(long pointer); - - public static native long getDataType(long pointer); - - public static native long getField(long pointer, int index); - - public static native long slice(long pointer, int start, int stop); - - public static native boolean getNull(long pointer, int index); - - public static native int getNullCount(long pointer); - - public static native byte getByte(long pointer, int index); - - public static native short getShort(long pointer, int index); - - public static native int getInt(long pointer, int index); - - public static native long getLong(long pointer, int index); - - public static native boolean getBool(long pointer, int index); - - public static native float getFloat(long pointer, int index); - - public static native double getDouble(long pointer, int index); - - public static native BigDecimal getBigDecimal(long pointer, int index); - - public static native String getUTF8(long pointer, int index); - - /** - * Raw-pointer variant of {@link #getUTF8(long, int)} that accepts an array to hold - * a pointer and an output length. - *

- * For Java query engines that use Unsafe to manipulate native memory, this allows working with the string - * inside of the JVM without copying it into Java heap memory. - */ - public static native void getUTF8_ptr_len(long pointer, int index, long[] outPtr, int[] outLen); - - public static native byte[] getBinary(long pointer, int index); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDTypeMethods.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDTypeMethods.java deleted file mode 100644 index a59fc9f95d8..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDTypeMethods.java +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import java.util.List; - -public final class NativeDTypeMethods { - static { - NativeLoader.loadJni(); - } - - private NativeDTypeMethods() {} - - /** - * Create a new native DType for a PType::I8. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newByte(boolean isNullable); - - /** - * Create a new native DType for a PType::I16. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newShort(boolean isNullable); - - /** - * Create a new native DType for a PType::I32. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newInt(boolean isNullable); - - /** - * Create a new native DType for a PType::I64. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newLong(boolean isNullable); - - /** - * Create a new native DType for a PType::F32. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newFloat(boolean isNullable); - - /** - * Create a new native DType for a PType::F64. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newDouble(boolean isNullable); - - /** - * Create a new native DType for a decimal type. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @param precision decimal precision - * @param scale decimal scale - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newDecimal(int precision, int scale, boolean isNullable); - - /** - * Create a new native DType for a UTF-8 string type. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newUtf8(boolean isNullable); - - /** - * Create a new native DType for a Binary type. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newBinary(boolean isNullable); - - /** - * Create a new native DType for a boolean type. The created object lives in native memory. - * - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newBool(boolean isNullable); - - /** - * Create a new native DType for a List type. The created object lives in native memory. - * - * @param elementTypePtr A native pointer to a DType containing the type of the elements - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newList(long elementTypePtr, boolean isNullable); - - /** - * Create a new native DType for a FixedSizeList type. The created object lives in native memory. - * - * @param elementTypePtr A native pointer to a DType containing the type of the elements - * @param size The fixed size of each list - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newFixedSizeList(long elementTypePtr, int size, boolean isNullable); - - /** - * Create a new native DType for a Struct type. The created object lives in native memory. - * - * @param fieldNames An array of field names - * @param fieldTypes An array of native pointers to the DType for each field - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newStruct(String[] fieldNames, long[] fieldTypes, boolean isNullable); - - /** - * Create a new native DType for a List type. The created object lives in native memory. - * - * @param timeUnit A byte that represents a {@link dev.vortex.api.DType.TimeUnit} - * @param zone The time zone or offset string - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newTimestamp(byte timeUnit, String zone, boolean isNullable); - - /** - * Create a new native DType for a List type. The created object lives in native memory. - * - * @param timeUnit A byte that represents a {@link dev.vortex.api.DType.TimeUnit} - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newDate(byte timeUnit, boolean isNullable); - - /** - * Create a new native DType for a List type. The created object lives in native memory. - * - * @param timeUnit A byte that represents a {@link dev.vortex.api.DType.TimeUnit} - * @param isNullable true if the values can be null - * @return Pointer to a new heap-allocated {@code DType}. - */ - public static native long newTime(byte timeUnit, boolean isNullable); - - public static native void free(long pointer); - - public static native byte getVariant(long pointer); - - public static native boolean isNullable(long pointer); - - public static native List getFieldNames(long pointer); - - // Returns a list of DType pointers. - public static native List getFieldTypes(long pointer); - - public static native long getElementType(long pointer); - - public static native int getFixedSizeListSize(long pointer); - - public static native boolean isDate(long pointer); - - public static native boolean isTime(long pointer); - - public static native boolean isTimestamp(long pointer); - - public static native byte getTimeUnit(long pointer); - - public static native String getTimeZone(long pointer); - - public static native boolean isDecimal(long pointer); - - public static native int getDecimalPrecision(long pointer); - - public static native byte getDecimalScale(long pointer); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java new file mode 100644 index 00000000000..d5e418f5a49 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +import java.util.Map; + +/** JNI boundary for {@link dev.vortex.api.DataSource}. */ +public final class NativeDataSource { + static { + NativeLoader.loadJni(); + } + + private NativeDataSource() {} + + /** + * Open a data source from one or more URIs or globs. + * + * @param sessionPointer pointer from {@link NativeSession#newSession()} + * @param uris paths or globs (for example {@code ["file:///a.vortex", "file:///b.vortex"]}) + * @param options object-store properties (may be null) + */ + public static native long open(long sessionPointer, String[] uris, Map options); + + /** Free a data source pointer. */ + public static native void free(long pointer); + + /** Export the data source's schema into the Arrow C Data Interface struct at {@code schemaAddress}. */ + public static native void arrowSchema(long pointer, long schemaAddress); + + /** + * Populate {@code out} with {@code [rows, cardinality]}. Cardinality is one of + * {@code 0=unknown}, {@code 1=estimate}, {@code 2=exact}. + */ + public static native void rowCount(long pointer, long[] out); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java new file mode 100644 index 00000000000..fc5bedbe7b5 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeExpression.java @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +/** JNI boundary for {@link dev.vortex.api.Expression}. */ +public final class NativeExpression { + static { + NativeLoader.loadJni(); + } + + private NativeExpression() {} + + public static native long root(); + + public static native long getItem(String fieldName, long childPointer); + + public static native long select(String[] fieldNames, long childPointer); + + public static native long and(long[] operandPointers); + + public static native long or(long[] operandPointers); + + public static native long binary(byte operator, long lhs, long rhs); + + public static native long not(long childPointer); + + public static native long isNull(long childPointer); + + public static native long literalBool(boolean value, boolean isNull); + + public static native long literalI8(byte value, boolean isNull); + + public static native long literalI16(short value, boolean isNull); + + public static native long literalI32(int value, boolean isNull); + + public static native long literalI64(long value, boolean isNull); + + public static native long literalF32(float value, boolean isNull); + + public static native long literalF64(double value, boolean isNull); + + public static native long literalString(String value); + + public static native void free(long pointer); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFileMethods.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFileMethods.java deleted file mode 100644 index 78f56255fa1..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFileMethods.java +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import java.util.List; -import java.util.Map; - -public final class NativeFileMethods { - static { - NativeLoader.loadJni(); - } - - private NativeFileMethods() {} - - /** - * List all Vortex files underneath the current file path. - * - * @param uri The URI describing both the object store and the path to the file - * @param options Any options required to initialize the object store client - * @return A list of URIs for the Vortex files below the provided path - */ - public static native List listVortexFiles(String uri, Map options); - - /** - * Delete the files at the provided URIs. Use the options to configure an object store client. - */ - public static native void delete(String[] uris, Map options); - - /** - * Open a file using the native library with the provided URI and options. - * - * @param uri The URI of the file to open. e.g. "file://path/to/file". - * @param options A map of options to provide for opening the file. - * @return A native pointer to the opened file. This will be 0 if the open call failed. - */ - public static native long open(String uri, Map options); - - /** - * Get the total row count contained in the file associated with the given pointer. - * - * @param pointer The native pointer to a file. Must be a value returned by {@link #open(String, Map)}. - * @return The number of rows of data encoded in the file. This includes null values. - */ - public static native long rowCount(long pointer); - - /** - * Get the data type of the file associated with the given pointer. - * - * @param pointer The native pointer to a file. Must be a value returned by {@link #open(String, Map)}. - * @return Native pointer to the DType of the file. This pointer is owned by the file and should not be freed. - */ - public static native long dtype(long pointer); - - /** - * Close the file associated with the given pointer. - * - * @param pointer The native pointer to a file. Must be a value returned by {@link #open(String, Map)}. - */ - public static native void close(long pointer); - - /** - * Build a new native scan operator that will materialize Arrays from the file, pushing down the optional - * predicate, row range or row indices to perform data skipping. - */ - public static native long scan( - long pointer, List columns, byte[] predicateProto, long[] rowRange, long[] rowIndices); -} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java new file mode 100644 index 00000000000..27ebc2073bb --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeFiles.java @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +import dev.vortex.api.Session; +import java.util.List; +import java.util.Map; + +/** + * Static utilities for discovering and deleting Vortex files on an object store. The caller + * supplies a {@link Session}; its runtime handle is forwarded to the underlying object store. + */ +public final class NativeFiles { + static { + NativeLoader.loadJni(); + } + + private NativeFiles() {} + + /** List all Vortex files reachable under {@code uri}. */ + public static List listFiles(Session session, String uri, Map options) { + return listFiles(session.nativePointer(), uri, options); + } + + /** Delete files at the given URIs. Silently tolerates an empty list. */ + public static void delete(Session session, String[] uris, Map options) { + delete(session.nativePointer(), uris, options); + } + + private static native List listFiles(long sessionPointer, String uri, Map options); + + private static native void delete(long sessionPointer, String[] uris, Map options); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeLoader.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeLoader.java index 415ae6a9bbb..86ecd25f1c1 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeLoader.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeLoader.java @@ -4,50 +4,28 @@ package dev.vortex.jni; import com.google.common.io.ByteStreams; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.Locale; /** - * Utility class for loading the native Vortex JNI library. - *

- * This class handles the platform-specific loading of the native Vortex library - * by detecting the operating system and architecture, extracting the appropriate - * native library from the classpath, and loading it into the JVM. - *

- *

- * The loader supports Windows, macOS, and Linux platforms with automatic - * detection of the correct library file format (.dll, .dylib, or .so). - *

+ * Loads the native vortex-jni shared library from the classpath. */ public final class NativeLoader { private static boolean loaded = false; private NativeLoader() {} - /** - * Loads the native Vortex JNI library if it hasn't been loaded already. - *

- * This method performs platform detection, extracts the appropriate native - * library from the classpath to a temporary file, and loads it using - * {@link System#load(String)}. The method is thread-safe and will only - * perform the loading operation once per JVM session. - *

- *

- * The native library is expected to be located at: - * {@code /native/{platform}-{arch}/libvortex_jni.{ext}} - * where platform is one of: win, darwin, linux and ext is the appropriate - * library extension for the platform. - *

- * - * @throws UnsupportedOperationException if the current platform is not supported - * @throws RuntimeException if the library cannot be extracted or loaded - */ + /** Load the native library into the current JVM. Thread-safe and idempotent. */ public static synchronized void loadJni() { if (loaded) { return; } - // Load the native library String osName = System.getProperty("os.name").toLowerCase(Locale.ROOT); String osArch = System.getProperty("os.arch").toLowerCase(Locale.ROOT); String libName = "libvortex_jni"; @@ -70,8 +48,6 @@ public static synchronized void loadJni() { throw new UnsupportedOperationException("Unsupported OS: " + osName); } - // Extract the library from classpath - // This assumes the library is in the same package as this class String libPath = "/native/" + osShortName + "-" + osArch + "/" + libName; try (InputStream in = NativeLoader.class.getResourceAsStream(libPath)) { if (in == null) { @@ -88,7 +64,6 @@ public static synchronized void loadJni() { throw new RuntimeException("Failed to load library: " + e.getMessage(), e); } - // Load the library System.load(libName); loaded = true; } diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativePartition.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativePartition.java new file mode 100644 index 00000000000..3a9ed7e7530 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativePartition.java @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +/** JNI boundary for {@link dev.vortex.api.Partition}. */ +public final class NativePartition { + static { + NativeLoader.loadJni(); + } + + private NativePartition() {} + + /** Free a partition pointer that was not consumed by {@link #scanArrow}. */ + public static native void free(long pointer); + + /** Fill {@code out} with {@code [rows, cardinality]}. */ + public static native void rowCount(long pointer, long[] out); + + /** + * Consume the partition into the {@code FFI_ArrowArrayStream} at {@code streamAddress}. + * The partition pointer is invalidated by this call. + * + * @param sessionPointer native session pointer used for execution context + * @param partitionPointer partition pointer to consume + * @param streamAddress address of an allocated {@code FFI_ArrowArrayStream} struct + */ + public static native void scanArrow(long sessionPointer, long partitionPointer, long streamAddress); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeRuntime.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeRuntime.java new file mode 100644 index 00000000000..c9442b0a937 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeRuntime.java @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +/** + * Controls for the JVM-wide current-thread worker pool that backs every Vortex session. + * + *

By default the pool has zero background threads: nothing happens until a Java thread + * calls a blocking Vortex API. Adding workers via {@link #setWorkerThreads(int)} lets the + * pool drive Vortex futures on behalf of the caller — useful when a single consumer + * thread cannot keep the executor busy on its own. Spawning Java threads that each poll + * a thread-safe Vortex iterator is an equivalent alternative. + */ +public final class NativeRuntime { + static { + NativeLoader.loadJni(); + } + + private NativeRuntime() {} + + /** + * Set the number of background worker threads driving Vortex futures. {@code 0} + * disables background execution; any positive value adjusts the pool up or down. + */ + public static native void setWorkerThreads(int n); + + /** Size the pool to {@code Runtime.getRuntime().availableProcessors() - 1}. */ + public static native void setWorkerThreadsToAvailableParallelism(); + + /** Returns the current background worker-thread count. */ + public static native int workerCount(); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java new file mode 100644 index 00000000000..adea8ab2144 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeScan.java @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +/** JNI boundary for {@link dev.vortex.api.Scan}. */ +public final class NativeScan { + static { + NativeLoader.loadJni(); + } + + private NativeScan() {} + + /** + * Create a new scan from a data source. The scan is lazy: no I/O happens until + * {@link #nextPartition(long)} is called. + * + * @param dataSourcePointer pointer from {@link NativeDataSource#open} + * @param projectionPointer native expression pointer, or 0 for "all columns" + * @param filterPointer native filter expression pointer, or 0 for "no filter" + * @param rowRangeBegin inclusive start of the row range, 0 for "unbounded" + * @param rowRangeEnd exclusive end of the row range, 0 for "unbounded" + * @param selectionIndices sorted row indices; may be null + * @param selectionInclude {@code 0} (all), {@code 1} (include {@code selectionIndices}), + * {@code 2} (exclude {@code selectionIndices}) + * @param limit max rows to return, or {@code 0} for "no limit" + * @param ordered true to preserve row order across partitions + */ + public static native long create( + long dataSourcePointer, + long projectionPointer, + long filterPointer, + long rowRangeBegin, + long rowRangeEnd, + long[] selectionIndices, + byte selectionInclude, + long limit, + boolean ordered); + + /** Free a scan pointer. */ + public static native void free(long pointer); + + /** Export the scan's schema into the Arrow C Data Interface struct at {@code schemaAddress}. */ + public static native void arrowSchema(long pointer, long schemaAddress); + + /** Fill {@code out} with {@code [count, cardinality]}. */ + public static native void partitionCount(long pointer, long[] out); + + /** Advance the scan and return the next partition pointer. Returns {@code 0} when exhausted. */ + public static native long nextPartition(long pointer); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeSession.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeSession.java new file mode 100644 index 00000000000..6ebdcacf237 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeSession.java @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +/** JNI boundary for {@link dev.vortex.api.Session}. */ +public final class NativeSession { + static { + NativeLoader.loadJni(); + } + + private NativeSession() {} + + /** Allocate a fresh native session. Free with {@link #free(long)}. */ + public static native long newSession(); + + /** Free a session previously returned by {@link #newSession()}. */ + public static native void free(long pointer); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java new file mode 100644 index 00000000000..b0793d09d84 --- /dev/null +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriter.java @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.jni; + +import java.util.Map; + +/** JNI boundary for {@link dev.vortex.api.VortexWriter}. */ +public final class NativeWriter { + static { + NativeLoader.loadJni(); + } + + private NativeWriter() {} + + /** + * Open a writer at {@code uri} that accepts batches matching the Arrow schema at + * {@code arrowSchemaAddress}. + */ + public static native long create( + long sessionPointer, String uri, long arrowSchemaAddress, Map options); + + /** + * Write a batch directly from Arrow C Data Interface addresses. + * + * @param writerPointer pointer from {@link #create} + * @param arrowArrayAddress address of an {@code ArrowArray} struct + * @param arrowSchemaAddress address of an {@code ArrowSchema} struct + * @return {@code true} on success + */ + public static native boolean writeBatch(long writerPointer, long arrowArrayAddress, long arrowSchemaAddress); + + /** Flush and close the writer. Must be called exactly once. */ + public static native void close(long writerPointer); +} diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriterMethods.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriterMethods.java deleted file mode 100644 index 8ddc7ce0bb1..00000000000 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeWriterMethods.java +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.jni; - -import java.util.Map; - -/** - * Native JNI methods for writing Vortex files. - */ -public final class NativeWriterMethods { - - static { - NativeLoader.loadJni(); - } - - private NativeWriterMethods() {} - - /** - * Creates a new native Vortex writer. - * - * @param uri the URI to the file, e.g. "file://path/to/file". - * @param dtype native pointer to a writer schema (Vortex DType) - * @param options additional writer options. For cloud storage this includes things like credentials. - * @return a native pointer to the writer, or 0 on failure - */ - public static native long create(String uri, long dtype, Map options); - - /** - * Writes a batch of Arrow data to the Vortex file. - * - * @param writerPtr the native writer pointer - * @param arrowData the Arrow IPC format data - * @return true if successful, false otherwise - */ - public static native boolean writeBatch(long writerPtr, byte[] arrowData); - - /** - * Writes a batch of Arrow data to the Vortex file directly from Arrow C Data Interface pointers. - * - * @param writerPtr the native writer pointer - * @param arrowArrayAddr memory address of the ArrowArray struct - * @param arrowSchemaAddr memory address of the ArrowSchema struct - * @return true if successful, false otherwise - */ - public static native boolean writeBatchFfi(long writerPtr, long arrowArrayAddr, long arrowSchemaAddr); - - /** - * Close and flush the writer, finalizing it to the storage system. - * - * @param writerPtr the native writer pointer - * @throws RuntimeException if the writer fails to close - */ - public static native void close(long writerPtr); -} diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/DTypeTest.java b/java/vortex-jni/src/test/java/dev/vortex/api/DTypeTest.java deleted file mode 100644 index 6946812e1b0..00000000000 --- a/java/vortex-jni/src/test/java/dev/vortex/api/DTypeTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api; - -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.Test; - -public final class DTypeTest { - - @Test - public void testNewFixedSizeListNonNullable() { - var elementType = DType.newInt(false); - var fslType = DType.newFixedSizeList(elementType, 3, false); - assertEquals(DType.Variant.FIXED_SIZE_LIST, fslType.getVariant()); - assertFalse(fslType.isNullable()); - assertEquals(3, fslType.getFixedSizeListSize()); - - var innerType = fslType.getElementType(); - assertEquals(DType.Variant.PRIMITIVE_I32, innerType.getVariant()); - } - - @Test - public void testNewFixedSizeListNullable() { - var elementType = DType.newUtf8(true); - var fslType = DType.newFixedSizeList(elementType, 5, true); - assertEquals(DType.Variant.FIXED_SIZE_LIST, fslType.getVariant()); - assertTrue(fslType.isNullable()); - assertEquals(5, fslType.getFixedSizeListSize()); - - var innerType = fslType.getElementType(); - assertEquals(DType.Variant.UTF8, innerType.getVariant()); - } - - @Test - public void testNewListGetElementType() { - var elementType = DType.newDouble(false); - var listType = DType.newList(elementType, false); - assertEquals(DType.Variant.LIST, listType.getVariant()); - - var innerType = listType.getElementType(); - assertEquals(DType.Variant.PRIMITIVE_F64, innerType.getVariant()); - } - - @Test - public void testNestedFixedSizeList() { - var innerElement = DType.newLong(false); - var innerFsl = DType.newFixedSizeList(innerElement, 2, false); - var outerFsl = DType.newFixedSizeList(innerFsl, 4, true); - assertEquals(DType.Variant.FIXED_SIZE_LIST, outerFsl.getVariant()); - assertTrue(outerFsl.isNullable()); - assertEquals(4, outerFsl.getFixedSizeListSize()); - - var inner = outerFsl.getElementType(); - assertEquals(DType.Variant.FIXED_SIZE_LIST, inner.getVariant()); - } - - @Test - public void testFixedSizeListInStruct() { - var elementType = DType.newFloat(false); - var fslType = DType.newFixedSizeList(elementType, 3, false); - var structType = - DType.newStruct(new String[] {"id", "embedding"}, new DType[] {DType.newInt(false), fslType}, false); - assertEquals(DType.Variant.STRUCT, structType.getVariant()); - - var fieldTypes = structType.getFieldTypes(); - assertEquals(2, fieldTypes.size()); - - var embeddingType = fieldTypes.get(1); - assertEquals(DType.Variant.FIXED_SIZE_LIST, embeddingType.getVariant()); - } -} diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java index abcdd8cf129..b6dd12a069e 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java @@ -3,21 +3,24 @@ package dev.vortex.api; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; -import dev.vortex.api.expressions.Binary; -import dev.vortex.api.expressions.GetItem; -import dev.vortex.api.expressions.Literal; -import dev.vortex.api.expressions.Root; +import dev.vortex.arrow.ArrowAllocation; import java.math.BigDecimal; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.Field; import org.junit.jupiter.api.Test; public final class TestMinimal { - // POJO representing the person data type. static final class Person { public String name; public BigDecimal salary; @@ -29,12 +32,6 @@ public Person(String name, BigDecimal salary, String state) { this.state = state; } - public Person(Map map) { - this.name = (String) map.get("Name"); - this.salary = (BigDecimal) map.get("Salary"); - this.state = (String) map.get("State"); - } - @Override public boolean equals(Object o) { if (!(o instanceof Person)) return false; @@ -51,28 +48,16 @@ public int hashCode() { @Override public String toString() { - return "Person{" + "name='" + name + '\'' + ", salary=" + salary + ", state='" + state + '\'' + '}'; + return "Person{name='" + name + "', salary=" + salary + ", state='" + state + "'}"; } } - private static final String MINIMAL_PATH = - TestMinimal.class.getResource("/minimal.vortex").getPath(); - - // data representing the complete `minimal` test table: - /// ======================= - /// Name | Salary | State - /// ======================= - /// Alice 1000 CA - /// Bob 2000 NY - /// Carol 3000 TX - /// Dan 4000 CA - /// Edward 5000 NY - /// Frida 6000 TX - /// George 7000 CA - /// Henry 8000 NY - /// Ida 9000 TX - /// John 10000 VA - /// ======================= + private static final String MINIMAL_URI = Paths.get( + Objects.requireNonNull(TestMinimal.class.getResource("/minimal.vortex")) + .getPath()) + .toUri() + .toString(); + private static final List MINIMAL_DATA = List.of( new Person("Alice", BigDecimal.valueOf(1000L, 2), "CA"), new Person("Bob", BigDecimal.valueOf(2000L, 2), "NY"), @@ -85,128 +70,102 @@ public String toString() { new Person("Ida", BigDecimal.valueOf(9000L, 2), "TX"), new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")); - private static final List PROJECTED_DATA = List.of( - new Person("Alice", null, "CA"), - new Person("Bob", null, "NY"), - new Person("Carol", null, "TX"), - new Person("Dan", null, "CA"), - new Person("Edward", null, "NY"), - new Person("Frida", null, "TX"), - new Person("George", null, "CA"), - new Person("Henry", null, "NY"), - new Person("Ida", null, "TX"), - new Person("John", null, "VA")); - @Test - public void testFullScan() { - try (var file = Files.open(MINIMAL_PATH); - var fullScan = file.newScan(ScanOptions.of())) { - assertEquals(10, file.rowCount()); - - var dtype = fullScan.getDataType(); - assertEquals(DType.Variant.STRUCT, dtype.getVariant()); - assertEquals(dtype.getFieldNames(), List.of("Name", "Salary", "State")); - - // Perform a full scan, check the result. - var people = readToList(fullScan, new TestCase() { - @Override - public Array[] open(Array batch) { - return new Array[] { - batch.getField(0), batch.getField(1), batch.getField(2), - }; - } + public void testFullScan() throws Exception { + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + DataSource ds = DataSource.open(session, MINIMAL_URI); - @Override - public Person readRow(Array[] fields, int idx) { - return new Person(fields[0].getUTF8(idx), fields[1].getBigDecimal(idx), fields[2].getUTF8(idx)); - } - }); + assertEquals(new DataSource.RowCount.Exact(10L), ds.rowCount()); - assertEquals(MINIMAL_DATA, people); - } + var schema = ds.arrowSchema(allocator); + assertEquals( + List.of("Name", "Salary", "State"), + schema.getFields().stream().map(Field::getName).toList()); + + List people = readAll(ds, ScanOptions.of(), allocator, TestMinimal::readFullBatch); + assertEquals(MINIMAL_DATA, people); } @Test - public void testProjectedScan() { - var projectOptions = ScanOptions.builder().addColumns("Name", "State").build(); - - try (var file = Files.open(MINIMAL_PATH); - var projectedScan = file.newScan(projectOptions)) { - // Do stuff. - var dtype = projectedScan.getDataType(); - assertEquals(DType.Variant.STRUCT, dtype.getVariant()); - assertEquals(dtype.getFieldNames(), List.of("Name", "State")); - - var people = readToList(projectedScan, new TestCase() { - @Override - public Array[] open(Array batch) { - return new Array[] { - batch.getField(0), batch.getField(1), - }; - } - - @Override - public Person readRow(Array[] fields, int idx) { - return new Person(fields[0].getUTF8(idx), null, fields[1].getUTF8(idx)); - } - }); - - assertEquals(PROJECTED_DATA, people); + public void testProjectedScan() throws Exception { + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + DataSource ds = DataSource.open(session, MINIMAL_URI); + Expression projection = Expression.select(new String[] {"Name", "State"}, Expression.root()); + + ScanOptions options = ScanOptions.builder().projection(projection).build(); + + List people = readAll(ds, options, allocator, batch -> { + List results = new ArrayList<>(); + VarCharVector names = (VarCharVector) batch.getVector("Name"); + VarCharVector states = (VarCharVector) batch.getVector("State"); + for (int i = 0; i < batch.getRowCount(); i++) { + String name = names.isNull(i) ? null : new String(names.get(i), UTF_8); + String state = states.isNull(i) ? null : new String(states.get(i), UTF_8); + results.add(new Person(name, null, state)); + } + return results; + }); + assertEquals(MINIMAL_DATA.size(), people.size()); + for (int i = 0; i < MINIMAL_DATA.size(); i++) { + assertEquals(MINIMAL_DATA.get(i).name, people.get(i).name); + assertEquals(MINIMAL_DATA.get(i).state, people.get(i).state); } } @Test - public void testProjectedScanWithFilter() { - var filterOptions = ScanOptions.builder() - .addColumns("State", "Salary", "Name") - .predicate(Binary.eq(GetItem.of(Root.INSTANCE, "State"), Literal.string("VA"))) - .build(); - - try (var file = Files.open(MINIMAL_PATH); - var filteredScan = file.newScan(filterOptions)) { - var dtype = filteredScan.getDataType(); - assertEquals(DType.Variant.STRUCT, dtype.getVariant()); - assertEquals(dtype.getFieldNames(), List.of("State", "Salary", "Name")); - - var people = readToList(filteredScan, new TestCase() { - @Override - public Array[] open(Array batch) { - return new Array[] { - batch.getField(0), // state - batch.getField(1), // salary - batch.getField(2), // name - }; - } - - @Override - public Person readRow(Array[] fields, int idx) { - var state = fields[0].getUTF8(idx); - var salary = fields[1].getBigDecimal(idx); - var name = fields[2].getUTF8(idx); - return new Person(name, salary, state); - } - }); - - assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people); - } + public void testProjectedScanWithFilter() throws Exception { + BufferAllocator allocator = ArrowAllocation.rootAllocator(); + Session session = Session.create(); + DataSource ds = DataSource.open(session, MINIMAL_URI); + Expression filter = + Expression.binary(Expression.BinaryOp.EQ, Expression.column("State"), Expression.literal("VA")); + + ScanOptions options = ScanOptions.builder().filter(filter).build(); + List people = readAll(ds, options, allocator, TestMinimal::readFullBatch); + assertEquals(List.of(new Person("John", BigDecimal.valueOf(10_000L, 2), "VA")), people); } - interface TestCase { - Array[] open(Array batch); - - Person readRow(Array[] fields, int idx); + private interface BatchReader { + List read(VectorSchemaRoot root); } - private List readToList(ArrayIterator scan, TestCase testCase) { - List people = new ArrayList<>(); + private static List readAll( + DataSource ds, ScanOptions options, BufferAllocator allocator, BatchReader reader) throws Exception { + List result = new ArrayList<>(); + Scan scan = ds.scan(options); while (scan.hasNext()) { - var batch = scan.next(); - Array[] fields = testCase.open(batch); + Partition partition = scan.next(); + try (ArrowReader arrowReader = partition.scanArrow(allocator)) { + while (arrowReader.loadNextBatch()) { + result.addAll(reader.read(arrowReader.getVectorSchemaRoot())); + } + } + } + return result; + } - for (int batchIdx = 0; batchIdx < batch.getLen(); batchIdx++) { - people.add(testCase.readRow(fields, batchIdx)); + private static List readFullBatch(VectorSchemaRoot root) { + List result = new ArrayList<>(); + VarCharVector names = (VarCharVector) root.getVector("Name"); + FieldVector salaries = root.getVector("Salary"); + VarCharVector states = (VarCharVector) root.getVector("State"); + + for (int i = 0; i < root.getRowCount(); i++) { + String name = names.isNull(i) ? null : new String(names.get(i), UTF_8); + String state = states.isNull(i) ? null : new String(states.get(i), UTF_8); + BigDecimal salary = null; + if (!salaries.isNull(i)) { + Object v = salaries.getObject(i); + if (v instanceof BigDecimal) { + salary = (BigDecimal) v; + } else { + salary = new BigDecimal(v.toString()); + } } + result.add(new Person(name, salary, state)); } - return people; + return result; } } diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/LiteralTest.java b/java/vortex-jni/src/test/java/dev/vortex/api/expressions/LiteralTest.java deleted file mode 100644 index 30d91bd99d9..00000000000 --- a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/LiteralTest.java +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import dev.vortex.api.Expression; -import dev.vortex.api.proto.Expressions; -import dev.vortex.proto.ExprProtos; -import java.math.BigDecimal; -import java.math.BigInteger; -import org.junit.jupiter.api.Test; - -public final class LiteralTest { - @Test - public void testLiteral_decimals() { - Literal lit = Literal.decimal(new BigDecimal(BigInteger.valueOf(-1234L), 3), 38, 3); - ExprProtos.Expr serialized = Expressions.serialize(lit); - Expression out = Expressions.deserialize(serialized); - assertEquals(lit, out); - } -} diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java b/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java deleted file mode 100644 index ed133bf4060..00000000000 --- a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions.proto; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import dev.vortex.api.Expression; -import dev.vortex.api.expressions.*; -import dev.vortex.api.proto.Expressions; -import dev.vortex.proto.ExprProtos; -import org.junit.jupiter.api.Test; - -public final class TestExpressionProtos { - @Test - public void testRoundTrip() { - Expression expression = Binary.and( - GetItem.of(Root.INSTANCE, "a.b.c"), - Binary.or(Root.INSTANCE, Not.of(Literal.bool(null)), Literal.bool(false)), - Binary.eq(Literal.bool(true), Not.of(Literal.bool(false)))); - ExprProtos.Expr proto = Expressions.serialize(expression); - Expression deserialized = Expressions.deserialize(proto); - assertEquals(expression, deserialized); - } - - @Test - public void testIsNullRoundTrip() { - Expression expression = IsNull.of(GetItem.of(Root.INSTANCE, "a.b.c")); - ExprProtos.Expr proto = Expressions.serialize(expression); - Expression deserialized = Expressions.deserialize(proto); - assertEquals(expression, deserialized); - } - - @Test - public void testIsNotNullRoundTrip() { - Expression expression = IsNotNull.of(GetItem.of(Root.INSTANCE, "a.b.c")); - ExprProtos.Expr proto = Expressions.serialize(expression); - Expression deserialized = Expressions.deserialize(proto); - assertEquals(expression, deserialized); - } -} diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestOpenErrors.java b/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestOpenErrors.java deleted file mode 100644 index 02d824467f0..00000000000 --- a/java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestOpenErrors.java +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.api.expressions.proto; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import dev.vortex.jni.NativeFileMethods; -import org.junit.jupiter.api.Test; - -public final class TestOpenErrors { - @Test - public void testOpenThrows() { - RuntimeException runtimeException = assertThrows(RuntimeException.class, () -> { - NativeFileMethods.open("bad_scheme:///fake-location", null); - }); - assertTrue(runtimeException.getMessage().contains("Invalid URL")); - } -} diff --git a/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java b/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java index 3a9e356d2fa..19d6be77fdc 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java +++ b/java/vortex-jni/src/test/java/dev/vortex/jni/JNIWriterTest.java @@ -8,8 +8,11 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import dev.vortex.api.DType; +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; import dev.vortex.api.VortexWriter; import dev.vortex.arrow.ArrowAllocation; import java.io.IOException; @@ -24,6 +27,7 @@ import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; @@ -32,9 +36,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -/** - * Direct test of JNI writer to isolate pointer alignment issue. - */ public final class JNIWriterTest { @TempDir @@ -45,94 +46,81 @@ public static void loadLibrary() { NativeLoader.loadJni(); } + private static Schema personSchema() { + return new Schema(java.util.List.of( + new Field("name", FieldType.notNullable(new ArrowType.Utf8()), null), + new Field("age", FieldType.notNullable(new ArrowType.Int(32, true)), null))); + } + @Test public void testCreateWriter() throws IOException { - // Test just creating and closing a writer Path outputPath = tempDir.resolve("test_create.vortex"); String writePath = outputPath.toAbsolutePath().toUri().toString(); - // Make a new file writer with a very simple schema. - var writeSchema = DType.newStruct( - new String[] { - "name", "age", - }, - new DType[] {DType.newUtf8(false), DType.newInt(false)}, - false); - - // Minimal Arrow schema + BufferAllocator allocator = ArrowAllocation.rootAllocator(); Map options = new HashMap<>(); - System.err.println("Creating writer for path: " + writePath); - - try (VortexWriter writer = VortexWriter.create(writePath, writeSchema, options)) { + Session session = Session.create(); + try (VortexWriter writer = VortexWriter.create(session, writePath, personSchema(), options, allocator)) { assertNotNull(writer); - System.err.println("Writer created successfully"); } - // Verify file was created - assertTrue(Files.exists(outputPath), "Output file should exist"); - System.err.println("File created at: " + outputPath); + assertTrue(Files.exists(outputPath), "output file should exist"); } @Test - public void testWriteBatchFfi() throws IOException { + public void testWriteBatch() throws IOException { Path outputPath = tempDir.resolve("test_ffi.vortex"); String writePath = outputPath.toAbsolutePath().toUri().toString(); - var writeSchema = DType.newStruct( - new String[] {"name", "age"}, new DType[] {DType.newUtf8(false), DType.newInt(false)}, false); - BufferAllocator allocator = ArrowAllocation.rootAllocator(); - - Schema arrowSchema = new Schema(java.util.List.of( - new Field("name", FieldType.notNullable(new ArrowType.Utf8()), null), - new Field("age", FieldType.notNullable(new ArrowType.Int(32, true)), null))); - - try (VortexWriter writer = VortexWriter.create(writePath, writeSchema, new HashMap<>())) { - // Build a batch with Arrow Java - try (VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, allocator)) { - VarCharVector nameVec = (VarCharVector) root.getVector("name"); - IntVector ageVec = (IntVector) root.getVector("age"); - - nameVec.allocateNew(3); - ageVec.allocateNew(3); - - nameVec.setSafe(0, "Alice".getBytes(UTF_8)); - nameVec.setSafe(1, "Bob".getBytes(UTF_8)); - nameVec.setSafe(2, "Carol".getBytes(UTF_8)); - ageVec.setSafe(0, 30); - ageVec.setSafe(1, 25); - ageVec.setSafe(2, 40); - - root.setRowCount(3); - - // Export to C Data Interface - try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator); - ArrowSchema arrowSchemaFfi = ArrowSchema.allocateNew(allocator)) { - Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchemaFfi); - - writer.writeBatchFfi(arrowArray.memoryAddress(), arrowSchemaFfi.memoryAddress()); - } + Schema schema = personSchema(); + + Session session = Session.create(); + try (VortexWriter writer = VortexWriter.create(session, writePath, schema, new HashMap<>(), allocator); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + VarCharVector nameVec = (VarCharVector) root.getVector("name"); + IntVector ageVec = (IntVector) root.getVector("age"); + + nameVec.allocateNew(3); + ageVec.allocateNew(3); + + nameVec.setSafe(0, "Alice".getBytes(UTF_8)); + nameVec.setSafe(1, "Bob".getBytes(UTF_8)); + nameVec.setSafe(2, "Carol".getBytes(UTF_8)); + ageVec.setSafe(0, 30); + ageVec.setSafe(1, 25); + ageVec.setSafe(2, 40); + + root.setRowCount(3); + + try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator); + ArrowSchema arrowSchemaFfi = ArrowSchema.allocateNew(allocator)) { + Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchemaFfi); + writer.writeBatch(arrowArray.memoryAddress(), arrowSchemaFfi.memoryAddress()); } } - assertTrue(Files.exists(outputPath), "Output file should exist"); - - // Read back and verify - try (var file = dev.vortex.api.Files.open(outputPath.toAbsolutePath().toString()); - var scan = file.newScan(ScanOptions.of())) { - assertEquals(3, file.rowCount()); - - var batch = scan.next(); - var nameField = batch.getField(0); - var ageField = batch.getField(1); - - assertEquals("Alice", nameField.getUTF8(0)); - assertEquals("Bob", nameField.getUTF8(1)); - assertEquals("Carol", nameField.getUTF8(2)); - assertEquals(30, ageField.getInt(0)); - assertEquals(25, ageField.getInt(1)); - assertEquals(40, ageField.getInt(2)); + assertTrue(Files.exists(outputPath), "output file should exist"); + + DataSource ds = DataSource.open(session, writePath); + assertEquals(new DataSource.RowCount.Exact(3L), ds.rowCount()); + + Scan scan = ds.scan(ScanOptions.of()); + while (scan.hasNext()) { + Partition p = scan.next(); + try (ArrowReader reader = p.scanArrow(allocator)) { + reader.loadNextBatch(); + VectorSchemaRoot resultRoot = reader.getVectorSchemaRoot(); + VarCharVector nameOut = (VarCharVector) resultRoot.getVector("name"); + IntVector ageOut = (IntVector) resultRoot.getVector("age"); + assertEquals("Alice", new String(nameOut.get(0), UTF_8)); + assertEquals("Bob", new String(nameOut.get(1), UTF_8)); + assertEquals("Carol", new String(nameOut.get(2), UTF_8)); + assertEquals(30, ageOut.get(0)); + assertEquals(25, ageOut.get(1)); + assertEquals(40, ageOut.get(2)); + } } } } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/ArrowUtils.java b/java/vortex-spark/src/main/java/dev/vortex/spark/ArrowUtils.java index 63f5c86185a..5eba9a3593f 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/ArrowUtils.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/ArrowUtils.java @@ -114,11 +114,7 @@ public static DataType fromArrowType(ArrowType dt) { case Timestamp: { ArrowType.Timestamp ts = (ArrowType.Timestamp) dt; if (ts.getUnit() == TimeUnit.MICROSECOND) { - if (ts.getTimezone() != null) { - return DataTypes.TimestampNTZType; - } else { - return DataTypes.TimestampType; - } + return ts.getTimezone() != null ? DataTypes.TimestampType : DataTypes.TimestampNTZType; } else { throw new UnsupportedOperationException("Unsupported Arrow type: " + dt); } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/SparkTypes.java b/java/vortex-spark/src/main/java/dev/vortex/spark/SparkTypes.java deleted file mode 100644 index bcd4aed6219..00000000000 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/SparkTypes.java +++ /dev/null @@ -1,174 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.spark; - -import dev.vortex.api.DType; -import java.util.Optional; -import org.apache.spark.sql.connector.catalog.Column; -import org.apache.spark.sql.types.*; - -/** - * Helpers for converting between Spark and Vortex type systems. - */ -public final class SparkTypes { - private SparkTypes() {} - - public static DType toDType(StructType schema) { - - String[] fieldNames = new String[schema.length()]; - DType[] fieldTypes = new DType[schema.length()]; - - for (int i = 0; i < schema.length(); i++) { - StructField field = schema.fields()[i]; - fieldNames[i] = field.name(); - fieldTypes[i] = convertField(field.dataType(), field.nullable()); - } - - return DType.newStruct(fieldNames, fieldTypes, false); - } - - // Convert field type to Vortex type. - static DType convertField(DataType dataType, boolean isNullable) { - if (dataType instanceof ByteType) { - return DType.newByte(isNullable); - } else if (dataType instanceof ShortType) { - return DType.newShort(isNullable); - } else if (dataType instanceof IntegerType) { - return DType.newInt(isNullable); - } else if (dataType instanceof LongType) { - return DType.newLong(isNullable); - } else if (dataType instanceof FloatType) { - return DType.newFloat(isNullable); - } else if (dataType instanceof DoubleType) { - return DType.newDouble(isNullable); - } else if (dataType instanceof DecimalType) { - DecimalType decimalType = (DecimalType) dataType; - return DType.newDecimal(decimalType.precision(), decimalType.scale(), isNullable); - } else if (dataType instanceof BooleanType) { - return DType.newBinary(isNullable); - } else if (dataType instanceof StringType) { - return DType.newUtf8(isNullable); - } else if (dataType instanceof BinaryType) { - return DType.newBinary(isNullable); - } else if (dataType instanceof ArrayType) { - ArrayType arrayType = ((ArrayType) dataType); - DType elementType = convertField(arrayType.elementType(), arrayType.containsNull()); - return DType.newList(elementType, isNullable); - } else if (dataType instanceof StructType) { - StructType structType = (StructType) dataType; - return toDType(structType); - } else if (dataType instanceof TimestampType) { - // Spark emits timestamps with UTC timezone and microsecond precision. - return DType.newTimestamp(DType.TimeUnit.MICROSECONDS, Optional.of("UTC"), isNullable); - } else if (dataType instanceof TimestampNTZType) { - // TimestampNTZ is microsecond timestamp without zone - return DType.newTimestamp(DType.TimeUnit.MICROSECONDS, Optional.empty(), isNullable); - } else if (dataType instanceof DateType) { - // TODO(aduffy): any problems with the date values since they're refed to - // gregorian proleptic? - return DType.newDate(DType.TimeUnit.DAYS, isNullable); - } else { - throw new IllegalArgumentException("Unsupported data type for Vortex: " + dataType); - } - } - - /** - * Convert a STRUCT Vortex type to a Spark {@link DataType}. - */ - public static DataType toDataType(DType dType) { - switch (dType.getVariant()) { - case NULL: - return DataTypes.NullType; - case BOOL: - return DataTypes.BooleanType; - case PRIMITIVE_U8: - case PRIMITIVE_I8: - return DataTypes.ByteType; - case PRIMITIVE_U16: - case PRIMITIVE_I16: - return DataTypes.ShortType; - case PRIMITIVE_U32: - case PRIMITIVE_I32: - return DataTypes.IntegerType; - case PRIMITIVE_U64: - case PRIMITIVE_I64: - return DataTypes.LongType; - case PRIMITIVE_F16: - throw new IllegalArgumentException("Spark does not support f16"); - case PRIMITIVE_F32: - return DataTypes.FloatType; - case PRIMITIVE_F64: - return DataTypes.DoubleType; - case UTF8: - return DataTypes.StringType; - case BINARY: - return DataTypes.BinaryType; - case STRUCT: - // For each of the inner struct fields, we capture them together here. - var fieldNames = dType.getFieldNames(); - var fieldTypes = dType.getFieldTypes(); - - // NOTE: it's very important we do this with a for loop. Using the streams API can easily - // lead to StackOverflowError being thrown. - var fields = new StructField[fieldNames.size()]; - for (int i = 0; i < fieldNames.size(); i++) { - var name = fieldNames.get(i); - try (var type = fieldTypes.get(i)) { - fields[i] = new StructField(name, toDataType(type), dType.isNullable(), Metadata.empty()); - } - } - - return DataTypes.createStructType(fields); - case LIST: - case FIXED_SIZE_LIST: - return DataTypes.createArrayType(toDataType(dType.getElementType()), dType.isNullable()); - case EXTENSION: - /* - * Spark does not have a direct equivalent for many of the temporal types we support in Vortex or Arrow. - * Notably, there is no DATE type, and timestamps can have at most µs-level precision. - * This means that we need to "cheat" a little in how we convert into Spark's type system. We support - * the following conversions: - * 1. Vortex DATE -> Spark TIMESTAMP (with 00:00:00 time and local timezone) - * 2. Vortex TIMESTAMP -> Spark TIMESTAMP, with precision truncated to µs - * 3. Vortex TIME -> not supported - */ - if (dType.isTime()) { - throw new IllegalArgumentException("Spark does not support Vortex TIME data type"); - } - - if (dType.isDate()) { - return DateType$.MODULE$; - } - - if (dType.isTimestamp()) { - return TimestampType$.MODULE$; - } - - // TODO(aduffy): other extension types - throw new IllegalArgumentException("Unsupported non-temporal extension type"); - case DECIMAL: - return DataTypes.createDecimalType(dType.getPrecision(), dType.getScale()); - default: - throw new IllegalArgumentException("unreachable"); - } - } - - /** - * Convert a STRUCT Vortex type to a Spark {@link Column}. - */ - public static Column[] toColumns(DType dType) { - var fieldNames = dType.getFieldNames(); - var fieldTypes = dType.getFieldTypes(); - var columns = new Column[fieldNames.size()]; - - for (int i = 0; i < columns.length; i++) { - var name = fieldNames.get(i); - try (var type = fieldTypes.get(i)) { - columns[i] = Column.create(name, toDataType(type), type.isNullable()); - } - } - - return columns; - } -} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java index 66f3e5f001c..02938bfd97e 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java @@ -7,9 +7,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import dev.vortex.api.File; -import dev.vortex.api.Files; -import dev.vortex.jni.NativeFileMethods; +import dev.vortex.api.DataSource; +import dev.vortex.jni.NativeFiles; import dev.vortex.spark.config.HadoopUtils; import dev.vortex.spark.read.PartitionPathUtils; import java.util.Map; @@ -19,12 +18,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.connector.catalog.CatalogV2Util; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableProvider; import org.apache.spark.sql.connector.expressions.Transform; import org.apache.spark.sql.sources.DataSourceRegister; import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import scala.Option; @@ -82,20 +82,26 @@ public StructType inferSchema(CaseInsensitiveStringMap options) { var pathToInfer = Objects.requireNonNull(Iterables.getLast(paths)); // If the path is a directory, scan the directory for a file and use that file if (!pathToInfer.endsWith(".vortex")) { - Optional firstFile = NativeFileMethods.listVortexFiles(pathToInfer, formatOptions).stream() - .findFirst(); + Optional firstFile = + NativeFiles.listFiles(VortexSparkSession.get(formatOptions), pathToInfer, formatOptions).stream() + .findFirst(); if (firstFile.isEmpty()) { - return new StructType(); + throw new RuntimeException(String.format("UNABLE_TO_INFER_SCHEMA format: %s", shortName())); } else { pathToInfer = firstFile.get(); } } StructType dataSchema; - try (File file = Files.open(pathToInfer, formatOptions)) { - var columns = SparkTypes.toColumns(file.getDType()); - dataSchema = CatalogV2Util.v2ColumnsToStructType(columns); + { + DataSource ds = DataSource.open(VortexSparkSession.get(formatOptions), pathToInfer, formatOptions); + var arrowSchema = ds.arrowSchema(dev.vortex.arrow.ArrowAllocation.rootAllocator()); + StructField[] fields = arrowSchema.getFields().stream() + .map(f -> new StructField( + f.getName(), ArrowUtils.fromArrowField(f), f.isNullable(), Metadata.empty())) + .toArray(StructField[]::new); + dataSchema = new StructType(fields); } // Discover partition columns from Hive-style directory paths and append them. @@ -119,9 +125,9 @@ public StructType inferSchema(CaseInsensitiveStringMap options) { * This method creates a VortexWritableTable that can be used to both read from and write to * Vortex files. The partitioning parameter is currently ignored. * - * @param schema the table schema + * @param schema the table schema * @param partitioning table partitioning transforms - * @param properties the table properties containing file paths and other options + * @param properties the table properties containing file paths and other options * @return a VortexTable instance for reading and writing data * @throws RuntimeException if required path properties are missing */ diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexFilePartition.java b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexFilePartition.java index 7d88f8469c4..78a26aac9cb 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexFilePartition.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexFilePartition.java @@ -3,74 +3,32 @@ package dev.vortex.spark; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.io.Serializable; +import java.util.List; import java.util.Map; -import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.types.StructType; /** - * An {@link InputPartition} for reading a whole Vortex file. - *

- * This class represents a partition that corresponds to a single Vortex file. - * It contains the file path and the columns to be read from that file. - * Each partition can be processed independently by Spark executors. + * An {@link InputPartition} describing a group of Vortex files that a single reader + * should handle together. + * + *

Each executor opens a single Vortex {@code Session}, {@code DataSource} and + * {@code Scan} over the partition's {@link #paths()} and consumes every Vortex partition + * produced by that scan before moving on to the next Spark {@code InputPartition}. + * + *

The requested output schema is carried as a {@link StructType} rather than a list of + * {@code Column} objects: {@code StructType} is the stable serialization surface in Spark + * and survives shipping to executors reliably. + * + * @param paths the Vortex file paths (or globs) belonging to this input partition + * @param readSchema the requested output schema (data columns + partition columns) + * @param formatOptions object-store properties used to open the files + * @param partitionValues Hive-style partition column values shared by all {@link #paths()} */ -public final class VortexFilePartition implements InputPartition, Serializable { - private final String path; - private final ImmutableList columns; - private final ImmutableMap formatOptions; - private final ImmutableMap partitionValues; - - /** - * Creates a new Vortex file partition. - * - * @param path the file system path to the Vortex file - * @param columns the list of columns to read from the file - * @param formatOptions options for accessing the file (S3/Azure credentials, etc.) - * @param partitionValues Hive-style partition column values extracted from the file path - */ - public VortexFilePartition( - String path, - ImmutableList columns, - ImmutableMap formatOptions, - ImmutableMap partitionValues) { - this.path = path; - this.columns = columns; - this.formatOptions = formatOptions; - this.partitionValues = partitionValues; - } - - /** - * Returns the file system path to the Vortex file for this partition. - * - * @return the file path - */ - public String getPath() { - return path; - } - - /** - * Returns the list of columns to be read from this partition. - * - * @return the immutable list of columns - */ - public ImmutableList getColumns() { - return columns; - } - - public Map getFormatOptions() { - return formatOptions; - } - - /** - * Returns the partition column values parsed from this file's Hive-style directory path. - * Keys are column names, values are the string-encoded partition values. - * - * @return the partition values, empty if the file is not in a partitioned directory - */ - public ImmutableMap getPartitionValues() { - return partitionValues; - } -} +public record VortexFilePartition( + List paths, + StructType readSchema, + Map formatOptions, + Map partitionValues) + implements InputPartition, Serializable {} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSessionProvider.java b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSessionProvider.java new file mode 100644 index 00000000000..6e2a19e25fa --- /dev/null +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSessionProvider.java @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark; + +import dev.vortex.api.Session; + +/** + * User hook for supplying a custom {@link Session} to Vortex Spark readers and writers. + * + *

Implementations must have a public no-argument constructor. They are instantiated + * exactly once per JVM (driver or executor) the first time a scan or write references + * the provider's class name through the {@code vortex.session.provider} option. The + * returned {@link Session} is then shared across every Vortex task on that JVM. + * + *

Typical use: install custom encodings, scalar functions, or layouts on a + * {@link Session} before returning it. + */ +public interface VortexSessionProvider { + /** + * Construct (or return a cached) {@link Session}. Called at most once per JVM per + * provider class. The returned session is retained for the JVM's lifetime. + */ + Session get(); +} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSparkSession.java b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSparkSession.java new file mode 100644 index 00000000000..184c18a0d62 --- /dev/null +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexSparkSession.java @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark; + +import dev.vortex.api.Session; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * JVM-wide holder for one or more Vortex {@link Session}s used by Spark readers and + * writers. The Rust side multiplexes every session onto one shared current-thread runtime, + * so sharing a single Java handle per JVM amortises session construction across every + * Spark task. + * + *

Three levels of customisation: + * + *

    + *
  1. Default: call {@link #get()} — returns a lazily-initialised singleton that + * lives for the life of the JVM. No configuration required. + *
  2. Driver-side override: call {@link #setDefault(Session)} with a custom + * session before any Spark action. Effective on the driver JVM only. + *
  3. Executor-friendly override: pass option {@code vortex.session.provider} + * with the fully-qualified name of a {@link VortexSessionProvider} implementation + * (no-arg constructor). Spark ships the class name to every executor, which + * instantiates the provider once per JVM and caches the returned session. Use this + * when you need the same custom session on the driver and on executors. + *
+ * + *

Native resources are released by {@code VortexCleaner} when the JVM shuts down; + * sessions held here are strongly referenced for the JVM's lifetime. + */ +public final class VortexSparkSession { + /** Options key used to select a {@link VortexSessionProvider} by class name. */ + public static final String PROVIDER_OPTION = "vortex.session.provider"; + + private static final ConcurrentHashMap providerCache = new ConcurrentHashMap<>(); + private static volatile Session defaultSession; + + private VortexSparkSession() {} + + /** Returns the default JVM-wide session, creating it on first use. */ + public static Session get() { + Session s = defaultSession; + if (s != null) { + return s; + } + synchronized (VortexSparkSession.class) { + if (defaultSession == null) { + defaultSession = Session.create(); + } + return defaultSession; + } + } + + /** + * Resolve the session to use for a given set of Spark format options. Honours the + * {@value #PROVIDER_OPTION} key; falls back to {@link #get()} otherwise. + */ + public static Session get(Map options) { + String providerClass = options == null ? null : options.get(PROVIDER_OPTION); + if (providerClass == null || providerClass.isEmpty()) { + return get(); + } + return providerCache.computeIfAbsent(providerClass, VortexSparkSession::loadProvider); + } + + /** + * Replace the default session. Intended for driver-side customisation before any + * Spark action runs. Does not propagate to executors — use + * {@link VortexSessionProvider} for that. + */ + public static void setDefault(Session session) { + Objects.requireNonNull(session, "session"); + synchronized (VortexSparkSession.class) { + defaultSession = session; + } + } + + private static Session loadProvider(String className) { + try { + Class cls = Class.forName(className, true, Thread.currentThread().getContextClassLoader()); + Object instance = cls.getDeclaredConstructor().newInstance(); + if (!(instance instanceof VortexSessionProvider provider)) { + throw new IllegalArgumentException( + className + " does not implement " + VortexSessionProvider.class.getName()); + } + return Objects.requireNonNull(provider.get(), className + ".get() returned null"); + } catch (ReflectiveOperationException e) { + throw new IllegalArgumentException("Failed to load Vortex session provider " + className, e); + } + } +} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexTable.java b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexTable.java index d650923ee7f..55b0835a3f2 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/VortexTable.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/VortexTable.java @@ -6,8 +6,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; import dev.vortex.spark.read.VortexScanBuilder; import dev.vortex.spark.write.VortexWriteBuilder; +import java.util.Arrays; import java.util.Map; import java.util.Set; import org.apache.spark.sql.connector.catalog.*; @@ -47,18 +49,19 @@ public VortexTable( * Creates a new ScanBuilder for this table. *

* The scan builder is pre-configured with all the file paths and columns - * from this table. The options parameter is currently unused but reserved - * for future use (e.g., S3 credentials). + * from this table. * - * @param _options scan options (currently unused) + * @param options scan options * @return a new VortexScanBuilder configured for this table */ @Override - public ScanBuilder newScanBuilder(CaseInsensitiveStringMap _options) { - ImmutableList readColumns = ImmutableList.builder() - .add(CatalogV2Util.structTypeToV2Columns(schema)) - .build(); - return new VortexScanBuilder(formatOptions).addAllPaths(paths).addAllColumns(readColumns); + public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { + Map opts = Maps.newHashMap(); + opts.putAll(formatOptions); + opts.putAll(options); + return new VortexScanBuilder(opts) + .addAllPaths(paths) + .addAllColumns(Arrays.asList(CatalogV2Util.structTypeToV2Columns(schema))); } /** diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/PartitionPathUtils.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/PartitionPathUtils.java index bbd73f6115c..e73f9b0c6b5 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/PartitionPathUtils.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/PartitionPathUtils.java @@ -3,6 +3,10 @@ package dev.vortex.spark.read; +import com.google.common.base.Splitter; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.util.LinkedHashMap; @@ -16,6 +20,7 @@ */ public final class PartitionPathUtils { private static final String HIVE_DEFAULT_PARTITION = "__HIVE_DEFAULT_PARTITION__"; + private static final Splitter PATH_SPLITTER = Splitter.on('/'); private PartitionPathUtils() {} @@ -26,8 +31,7 @@ private PartitionPathUtils() {} */ public static Map parsePartitionValues(String filePath) { LinkedHashMap values = new LinkedHashMap<>(); - String[] segments = filePath.split("/"); - for (String segment : segments) { + for (String segment : PATH_SPLITTER.split(filePath)) { int eqIdx = segment.indexOf('='); if (eqIdx > 0 && eqIdx < segment.length() - 1) { String key = URLDecoder.decode(segment.substring(0, eqIdx), StandardCharsets.UTF_8); @@ -46,20 +50,14 @@ public static DataType inferPartitionColumnType(String value) { if (value == null || HIVE_DEFAULT_PARTITION.equals(value)) { return DataTypes.StringType; } - try { - Integer.parseInt(value); + if (Ints.tryParse(value) != null) { return DataTypes.IntegerType; - } catch (NumberFormatException ignored) { } - try { - Long.parseLong(value); + if (Longs.tryParse(value) != null) { return DataTypes.LongType; - } catch (NumberFormatException ignored) { } - try { - Double.parseDouble(value); + if (Doubles.tryParse(value) != null) { return DataTypes.DoubleType; - } catch (NumberFormatException ignored) { } if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { return DataTypes.BooleanType; diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/ReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/ReaderFactory.java deleted file mode 100644 index 044c21dfad3..00000000000 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/ReaderFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.spark.read; - -import dev.vortex.spark.VortexFilePartition; -import java.io.Serializable; -import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.connector.read.InputPartition; -import org.apache.spark.sql.connector.read.PartitionReader; -import org.apache.spark.sql.connector.read.PartitionReaderFactory; -import org.apache.spark.sql.vectorized.ColumnarBatch; - -/** - * A {@link PartitionReaderFactory} for Vortex file partitions. - *

- * This factory creates partition readers for reading Vortex files. It implements the singleton - * pattern using an enum and only supports columnar reads for optimal performance. - * Row-based reads are not supported as Vortex is designed for columnar data processing. - */ -enum ReaderFactory implements PartitionReaderFactory, Serializable { - INSTANCE; - - private static final boolean SUPPORTS_COLUMNAR_READS = true; - - /** - * Creates a row-based partition reader. - *

- * This method is not supported as Vortex only supports columnar reads for performance reasons. - * - * @param partition the input partition to read from - * @return never returns, always throws an exception - * @throws UnsupportedOperationException always, as row-based reading is not supported - */ - @Override - public PartitionReader createReader(InputPartition partition) { - throw new UnsupportedOperationException("ReaderFactory only supports columnar reads"); - } - - /** - * Creates a columnar partition reader for the given partition. - *

- * This method creates a VortexPartitionReader that can efficiently read columnar data - * from a Vortex file partition. - * - * @param partition the input partition to read from, must be a VortexFilePartition - * @return a partition reader that produces ColumnarBatch objects - * @throws ClassCastException if the partition is not a VortexFilePartition - */ - @Override - public PartitionReader createColumnarReader(InputPartition partition) { - return new VortexPartitionReader((VortexFilePartition) partition); - } - - /** - * Indicates whether this factory supports columnar reads for the given partition. - *

- * Vortex always supports and prefers columnar reads for optimal performance. - * - * @param partition the input partition to check (parameter is ignored) - * @return always true, indicating columnar reads are supported - */ - @Override - public boolean supportColumnarReads(InputPartition partition) { - return SUPPORTS_COLUMNAR_READS; - } -} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexArrowColumnVector.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexArrowColumnVector.java index aabbe36031d..99e45feab72 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexArrowColumnVector.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexArrowColumnVector.java @@ -67,22 +67,12 @@ public int numNulls() { } /** - * Closes this column vector and releases any associated resources. - *

- * This method recursively closes any child columns (for complex types like structs) - * and then closes the underlying Arrow vector accessor. + * No-op: the underlying Arrow {@link ValueVector}s are owned by the + * {@link dev.vortex.relocated.org.apache.arrow.vector.ipc.ArrowReader} that produced + * this batch and are released when that reader is closed. */ @Override - public void close() { - if (childColumns != null) { - for (int i = 0; i < childColumns.length; i++) { - childColumns[i].close(); - childColumns[i] = null; - } - childColumns = null; - } - accessor.close(); - } + public void close() {} /** * Returns whether the value at the specified row is null. @@ -359,10 +349,6 @@ final int getNullCount() { return vector.getNullCount(); } - final void close() { - vector.close(); - } - boolean getBoolean(int rowId) { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java index 199513f1d8a..11dc13951ea 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java @@ -3,75 +3,92 @@ package dev.vortex.spark.read; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import dev.vortex.jni.NativeFileMethods; +import dev.vortex.jni.NativeFiles; import dev.vortex.spark.VortexFilePartition; +import dev.vortex.spark.VortexSparkSession; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.spark.sql.connector.catalog.CatalogV2Util; import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.types.StructType; /** * Execution source for batch scans of Vortex file tables. */ public final class VortexBatchExec implements Batch { - private final ImmutableList paths; - private final ImmutableList columns; - private final ImmutableMap formatOptions; + private final List paths; + private final StructType readSchema; + private final Map formatOptions; + private List resolvedPaths; /** * Creates a new VortexBatchExec for scanning the specified Vortex files. * - * @param paths the list of file paths to scan + * @param paths the list of file paths to scan * @param columns the list of columns to read from the files */ - public VortexBatchExec( - ImmutableList paths, ImmutableList columns, ImmutableMap formatOptions) { - this.paths = paths; - this.columns = columns; - this.formatOptions = formatOptions; + public VortexBatchExec(List paths, List columns, Map formatOptions) { + this.paths = List.copyOf(paths); + this.readSchema = CatalogV2Util.v2ColumnsToStructType(columns.toArray(new Column[0])); + this.formatOptions = Map.copyOf(formatOptions); } /** * Plans the input partitions for this batch scan. - *

- * Creates one partition per file path, where each partition is responsible - * for reading data from a single Vortex file. * - * @return an array of InputPartition objects, one per file path + *

Directory-like entries are expanded to concrete {@code .vortex} files. Each resolved + * file becomes its own {@link VortexFilePartition}; the partition carries the paths the + * reader should open, the requested schema, and any Hive-style partition values parsed + * out of the path. */ @Override public InputPartition[] planInputPartitions() { - // Scan all paths and assign each file its own partition. - // For each discovered file, parse Hive-style partition values from the path. - return paths.stream() - .flatMap(path -> { - if (path.endsWith(".vortex")) { - return Stream.of(path); - } else { - return NativeFileMethods.listVortexFiles(path, formatOptions).stream(); - } - }) + resolvedPaths = resolvePaths(); + return resolvedPaths.stream() .map(path -> { Map partVals = PartitionPathUtils.parsePartitionValues(path); - return new VortexFilePartition(path, columns, formatOptions, ImmutableMap.copyOf(partVals)); + return new VortexFilePartition( + List.of(path), readSchema, formatOptions, ImmutableMap.copyOf(partVals)); }) .toArray(InputPartition[]::new); } - /** - * Creates a factory for creating partition readers. - *

- * Returns a singleton ReaderFactory instance that can create readers - * capable of reading Vortex file partitions. - * - * @return the PartitionReaderFactory for Vortex files - */ @Override public PartitionReaderFactory createReaderFactory() { - return ReaderFactory.INSTANCE; + List files = resolvedPaths != null ? resolvedPaths : resolvePaths(); + Set partitionColumns = collectPartitionColumnNames(files); + List dataColumnNames = Arrays.stream(readSchema.fieldNames()) + .filter(name -> !partitionColumns.contains(name)) + .collect(Collectors.toList()); + return new VortexPartitionReaderFactory(dataColumnNames, formatOptions); + } + + private List resolvePaths() { + var session = VortexSparkSession.get(formatOptions); + return paths.stream() + .flatMap(path -> { + if (path.endsWith(".vortex")) { + return Stream.of(path); + } + return NativeFiles.listFiles(session, path, formatOptions).stream(); + }) + .collect(Collectors.toList()); + } + + private static Set collectPartitionColumnNames(List files) { + Set all = new HashSet<>(); + for (String path : files) { + all.addAll(PartitionPathUtils.parsePartitionValues(path).keySet()); + } + return all; } } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatch.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatch.java deleted file mode 100644 index 8f89423fa03..00000000000 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatch.java +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.spark.read; - -import dev.vortex.api.Array; -import org.apache.spark.sql.vectorized.ColumnVector; -import org.apache.spark.sql.vectorized.ColumnarBatch; - -/** - * A {@link ColumnarBatch} that returns Vortex-managed memory with Arrow format, shared over the C Data Interface. - */ -public final class VortexColumnarBatch extends ColumnarBatch { - private Array backingArray; - - /** - * Creates a new VortexColumnarBatch with the specified backing array and column vectors. - *

- * The backing array holds the native memory that contains the actual data, - * while the column vectors provide the Spark API for accessing that data. - * - * @param backingArray the Vortex Array that holds the native memory - * @param columns the array of ColumnVector objects for data access - * @param numRows the number of rows in this batch - */ - public VortexColumnarBatch(Array backingArray, ColumnVector[] columns, int numRows) { - super(columns, numRows); - this.backingArray = backingArray; - } - - /** - * Closes this columnar batch and releases all associated resources. - *

- * This method frees the native memory held by the backing Vortex array - * and then delegates to the parent class to close the column vectors. - */ - @Override - public void close() { - freeNativeMemory(); - super.close(); - } - - /** - * Closes this columnar batch if it is freeable and releases all associated resources. - *

- * This method frees the native memory held by the backing Vortex array - * and then delegates to the parent class to close the column vectors if freeable. - */ - @Override - public void closeIfFreeable() { - freeNativeMemory(); - super.closeIfFreeable(); - } - - private void freeNativeMemory() { - backingArray.close(); - backingArray = null; - } -} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatchIterator.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatchIterator.java deleted file mode 100644 index 06ba3237e14..00000000000 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexColumnarBatchIterator.java +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.spark.read; - -import dev.vortex.api.Array; -import dev.vortex.api.ArrayIterator; -import dev.vortex.arrow.ArrowAllocation; -import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; -import java.util.Iterator; -import org.apache.spark.sql.vectorized.ColumnVector; -import org.apache.spark.sql.vectorized.ColumnarBatch; - -/** - * Iterator that converts Vortex Arrays into Spark ColumnarBatch objects. - *

- * This iterator wraps a Vortex ArrayIterator and converts each Array into a Spark ColumnarBatch - * by exporting the data to Arrow format and wrapping it with VortexArrowColumnVector instances. - * The iterator uses prefetching to optimize memory usage and performance by batching arrays - * up to a maximum buffer size. - *

- * The iterator maintains a reusable VectorSchemaRoot to minimize allocation overhead when - * converting between Vortex and Arrow formats. - * - * @see ArrayIterator - * @see ColumnarBatch - * @see VortexArrowColumnVector - */ -public final class VortexColumnarBatchIterator implements Iterator, AutoCloseable { - /** - * Maximum buffer size in bytes for prefetching arrays. - *

- * The iterator will prefetch and batch arrays until this size limit is reached, - * which helps optimize memory usage and reduces the overhead of converting - * small arrays individually. - */ - public static final long MAX_BUFFER_BYTES = 16 * 1024 * 1024; // 16MB - - private final ArrayIterator backing; - private final PrefetchingIterator prefetching; - - // Reusable root - private VectorSchemaRoot root = null; - - /** - * Creates a new VortexColumnarBatchIterator that wraps the given ArrayIterator. - *

- * The iterator will use prefetching to batch arrays up to MAX_BUFFER_BYTES - * to optimize memory usage and conversion performance. - * - * @param backing the underlying ArrayIterator to wrap - */ - public VortexColumnarBatchIterator(ArrayIterator backing) { - this.backing = backing; - this.prefetching = new PrefetchingIterator<>(backing, MAX_BUFFER_BYTES, Array::nbytes); - } - - /** - * Returns whether there are more columnar batches available. - * - * @return true if there are more batches to iterate over, false otherwise - */ - @Override - public boolean hasNext() { - return prefetching.hasNext(); - } - - /** - * Returns the next columnar batch from the iterator. - *

- * This method retrieves the next Array from the prefetching iterator, - * exports it to Arrow format using a reusable VectorSchemaRoot, - * and wraps each field vector in a VortexArrowColumnVector to create - * a VortexColumnarBatch. - * - * @return the next ColumnarBatch containing the data from the next Array - * @throws java.util.NoSuchElementException if there are no more elements - */ - @Override - public ColumnarBatch next() { - Array next = prefetching.next(); - - root = next.exportToArrow(ArrowAllocation.rootAllocator(), root); - - int rowCount = root.getRowCount(); - ColumnVector[] vectors = new ColumnVector[root.getFieldVectors().size()]; - for (int i = 0; i < root.getFieldVectors().size(); i++) { - vectors[i] = new VortexArrowColumnVector(root.getFieldVectors().get(i)); - } - return new VortexColumnarBatch(next, vectors, rowCount); - } - - /** - * Closes this iterator and releases all associated resources. - *

- * This method closes the prefetching iterator, the backing ArrayIterator, - * and the reusable VectorSchemaRoot if it exists. - */ - @Override - public void close() { - this.prefetching.close(); - this.backing.close(); - if (root != null) { - root.close(); - root = null; - } - } -} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java index 3904e0272f6..c1a5f508cc8 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReader.java @@ -3,157 +3,148 @@ package dev.vortex.spark.read; -import static com.google.common.base.Preconditions.checkNotNull; - -import dev.vortex.api.File; -import dev.vortex.api.Files; +import dev.vortex.api.DataSource; +import dev.vortex.api.Expression; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.relocated.org.apache.arrow.memory.BufferAllocator; +import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; +import dev.vortex.relocated.org.apache.arrow.vector.ipc.ArrowReader; import dev.vortex.spark.VortexFilePartition; -import java.util.*; -import org.apache.spark.sql.connector.catalog.Column; +import dev.vortex.spark.VortexSparkSession; +import java.io.IOException; +import java.util.List; +import java.util.Map; import org.apache.spark.sql.connector.read.PartitionReader; +import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.vectorized.ColumnVector; import org.apache.spark.sql.vectorized.ColumnarBatch; /** - * A {@link PartitionReader} that reads columnar batches out of a Vortex file into - * Vortex memory format. - *

- * When reading from partitioned directories, partition column values are extracted from the - * Hive-style file path and materialized as Spark - * {@link org.apache.spark.sql.execution.vectorized.ConstantColumnVector} instances that are - * spliced into each output batch. + * Per-{@link VortexFilePartition} columnar reader. + * + *

Opens a single Vortex {@link Session}, {@link DataSource} and {@link Scan} spanning + * all of {@link VortexFilePartition#paths()} and streams every Vortex partition's record + * batches through the {@link PartitionReader} interface. */ final class VortexPartitionReader implements PartitionReader { - private final VortexFilePartition partition; - - private File file; - private VortexColumnarBatchIterator batches; - - /** Names of columns whose values come from the partition path rather than the data file. */ - private Set partitionColumnNames; - - /** Tracks the last data batch so its native memory can be freed properly. */ - private ColumnarBatch lastDataBatch; - - VortexPartitionReader(VortexFilePartition partition) { - this.partition = partition; - initNativeResources(); + private final VortexFilePartition spark; + private final BufferAllocator allocator; + + // Held so the DataSource/Scan stay reachable even if the JVM-wide singleton is + // ever reset during a task; the actual native session is owned by + // {@link VortexSparkSession} and is not released when this reader closes. + private Session session; + private DataSource dataSource; + private Scan scan; + + private Partition currentPartition; + private ArrowReader currentReader; + private boolean currentBatchLoaded; + private boolean exhausted; + + VortexPartitionReader(VortexFilePartition spark, List dataColumnNames, Map formatOptions) { + this.spark = spark; + this.allocator = ArrowAllocation.rootAllocator(); + + session = VortexSparkSession.get(formatOptions); + dataSource = DataSource.open(session, spark.paths(), formatOptions); + + var options = ScanOptions.builder(); + if (!dataColumnNames.isEmpty()) { + Expression projection = Expression.select(dataColumnNames.toArray(new String[0]), Expression.root()); + options.projection(projection); + } + scan = dataSource.scan(options.build()); } @Override public boolean next() { - checkNotNull(batches, "batches"); - return batches.hasNext(); + if (exhausted) { + return false; + } + while (true) { + if (currentReader != null) { + try { + if (currentReader.loadNextBatch()) { + currentBatchLoaded = true; + return true; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + closeCurrentReader(); + } + if (!scan.hasNext()) { + exhausted = true; + return false; + } + currentPartition = scan.next(); + currentReader = currentPartition.scanArrow(allocator); + } } @Override public ColumnarBatch get() { - checkNotNull(batches, "closed ArrayStream"); - - // Free previous data batch native memory - if (lastDataBatch != null) { - lastDataBatch.close(); - lastDataBatch = null; + if (!currentBatchLoaded) { + throw new IllegalStateException("no batch loaded; call next() first"); } + currentBatchLoaded = false; - ColumnarBatch dataBatch = batches.next(); - - if (partitionColumnNames.isEmpty()) { - return dataBatch; + VectorSchemaRoot root; + try { + root = currentReader.getVectorSchemaRoot(); + } catch (IOException e) { + throw new RuntimeException(e); } - // Track the data batch for lifecycle management - lastDataBatch = dataBatch; - return buildCombinedBatch(dataBatch); - } - - /** - * Builds a combined batch with data columns from the file and constant partition columns - * in the order expected by the full table schema. - */ - private ColumnarBatch buildCombinedBatch(ColumnarBatch dataBatch) { - int rowCount = dataBatch.numRows(); - Map partVals = partition.getPartitionValues(); - List allColumns = partition.getColumns(); - ColumnVector[] combined = new ColumnVector[allColumns.size()]; - - int dataIdx = 0; - for (int i = 0; i < allColumns.size(); i++) { - Column col = allColumns.get(i); - String partValue = partVals.get(col.name()); - if (partValue != null) { - combined[i] = PartitionPathUtils.createConstantVector(rowCount, col.dataType(), partValue); - } else { - combined[i] = dataBatch.column(dataIdx++); + int rowCount = root.getRowCount(); + Map partVals = spark.partitionValues(); + if (partVals.isEmpty()) { + ColumnVector[] vectors = new ColumnVector[root.getFieldVectors().size()]; + for (int i = 0; i < vectors.length; i++) { + vectors[i] = new VortexArrowColumnVector(root.getFieldVectors().get(i)); } + return new ColumnarBatch(vectors, rowCount); } - return new CombinedColumnarBatch(combined, rowCount); - } - - /** - * Initialize the Vortex File and ArrayStream resources. - *

- * Partition columns are identified by matching requested column names against the - * partition values from the file path. Only non-partition columns are pushed down - * to the Vortex scan. - */ - void initNativeResources() { - Map partVals = partition.getPartitionValues(); - this.partitionColumnNames = new HashSet<>(); - - List dataColumnNames = new ArrayList<>(); - for (Column col : partition.getColumns()) { - if (partVals.containsKey(col.name())) { - partitionColumnNames.add(col.name()); + StructField[] fields = spark.readSchema().fields(); + ColumnVector[] combined = new ColumnVector[fields.length]; + int dataIdx = 0; + for (int i = 0; i < fields.length; i++) { + StructField field = fields[i]; + String partValue = partVals.get(field.name()); + if (partValue != null) { + combined[i] = PartitionPathUtils.createConstantVector(rowCount, field.dataType(), partValue); } else { - dataColumnNames.add(col.name()); + combined[i] = new VortexArrowColumnVector(root.getFieldVectors().get(dataIdx++)); } } - - file = Files.open(partition.getPath(), partition.getFormatOptions()); - batches = new VortexColumnarBatchIterator( - file.newScan(ScanOptions.builder().columns(dataColumnNames).build())); + return new ColumnarBatch(combined, rowCount); } @Override public void close() { - if (lastDataBatch != null) { - lastDataBatch.close(); - lastDataBatch = null; - } - - checkNotNull(file, "File was closed"); - checkNotNull(batches, "ArrayStream was closed"); - - batches.close(); - batches = null; - - file.close(); - file = null; + closeCurrentReader(); + // Scan and DataSource native resources are released by VortexCleaner once + // references are dropped. Session is the JVM-wide singleton and outlives this reader. + scan = null; + dataSource = null; + session = null; } - /** - * A ColumnarBatch that does not close its column vectors on {@link #close()}. - *

- * The data column vectors are owned by the underlying {@link VortexColumnarBatch} - * (tracked via {@link #lastDataBatch}), and the constant partition vectors have trivial - * lifecycle. Neither should be closed by this wrapper. - */ - private static final class CombinedColumnarBatch extends ColumnarBatch { - CombinedColumnarBatch(ColumnVector[] columns, int numRows) { - super(columns, numRows); - } - - @Override - public void close() { - // Intentionally empty: lifecycle is managed by VortexPartitionReader - } - - @Override - public void closeIfFreeable() { - // Intentionally empty + private void closeCurrentReader() { + if (currentReader != null) { + try { + currentReader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + currentReader = null; } + currentPartition = null; } } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java new file mode 100644 index 00000000000..3a0b3fb5dfb --- /dev/null +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark.read; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import dev.vortex.spark.VortexFilePartition; +import java.io.Serializable; +import java.util.List; +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.read.InputPartition; +import org.apache.spark.sql.connector.read.PartitionReader; +import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.vectorized.ColumnarBatch; + +/** + * Factory that produces columnar readers for Vortex files. + * + *

microsoft/TypeScript (typescript) ### [`v6.0.3`](https://redirect.github.com/microsoft/TypeScript/compare/v6.0.2...050880ce59e30b356b686bd3144efe24f875ebc8) [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v6.0.2...v6.0.3) ### [`v6.0.2`](https://redirect.github.com/microsoft/TypeScript/compare/v5.9.3...607a22a90d1a5a1b507ce01bb8cd7ec020f954e7) [Compare Source](https://redirect.github.com/microsoft/TypeScript/compare/v5.9.3...v6.0.2)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- vortex-web/package-lock.json | 8 ++++---- vortex-web/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index aa41e786f74..2dccff8bdd4 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -33,7 +33,7 @@ "prettier": "^3.8.1", "storybook": "^10.3.3", "tailwindcss": "^4.1.4", - "typescript": "~5.9.0", + "typescript": "~6.0.0", "typescript-eslint": "^8.30.1", "vite": "^6.3.2" } @@ -5265,9 +5265,9 @@ } }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/vortex-web/package.json b/vortex-web/package.json index 55740faee04..b0b1d825faa 100644 --- a/vortex-web/package.json +++ b/vortex-web/package.json @@ -44,7 +44,7 @@ "prettier": "^3.8.1", "storybook": "^10.3.3", "tailwindcss": "^4.1.4", - "typescript": "~5.9.0", + "typescript": "~6.0.0", "typescript-eslint": "^8.30.1", "vite": "^6.3.2" } From a77c7145151a321144477002fe216588f2350bdc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:01:33 +0000 Subject: [PATCH 197/250] Update Rust crate zip to v8.6.0 (#7650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [zip](https://redirect.github.com/zip-rs/zip2) | workspace.dependencies | minor | `8.5.1` → `8.6.0` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
zip-rs/zip2 (zip) ### [`v8.6.0`](https://redirect.github.com/zip-rs/zip2/blob/HEAD/CHANGELOG.md#860---2026-04-25) [Compare Source](https://redirect.github.com/zip-rs/zip2/compare/v8.5.1...v8.6.0) ##### 🚀 Features - add `compression not supported` as enum error ([#​774](https://redirect.github.com/zip-rs/zip2/pull/774)) ##### 🐛 Bug Fixes - allow for `[u8]` as filename ([#​775](https://redirect.github.com/zip-rs/zip2/pull/775)) ##### 🚜 Refactor - mark `ZipFlags` as non-exhaustive and add test for `HasZipMetadata` ([#​777](https://redirect.github.com/zip-rs/zip2/pull/777)) - use and simplify is\_dir ([#​776](https://redirect.github.com/zip-rs/zip2/pull/776))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- Cargo.lock | 75 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22b8445fcfd..92b8f535503 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,13 +10,13 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" -version = "0.8.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8" dependencies = [ - "cfg-if", "cipher", - "cpufeatures 0.2.17", + "cpubits", + "cpufeatures 0.3.0", ] [[package]] @@ -1053,6 +1053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -1404,11 +1405,11 @@ dependencies = [ [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.2.1", "inout", ] @@ -1473,6 +1474,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -1807,6 +1814,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef0c543070d296ea414df2dd7625d1b24866ce206709d8a4a424f28377f5861" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1971,6 +1984,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "cudarc" version = "0.18.2" @@ -3405,7 +3427,7 @@ dependencies = [ "percent-encoding", "rand 0.9.4", "serde_json", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "url", ] @@ -3570,6 +3592,8 @@ dependencies = [ "block-buffer 0.12.0", "const-oid", "crypto-common 0.2.1", + "ctutils", + "zeroize", ] [[package]] @@ -4352,11 +4376,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hmac" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest 0.10.7", + "digest 0.11.2", ] [[package]] @@ -4695,11 +4719,11 @@ dependencies = [ [[package]] name = "inout" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -6754,11 +6778,11 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest 0.10.7", + "digest 0.11.2", "hmac", ] @@ -8408,6 +8432,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha2" version = "0.10.9" @@ -11849,9 +11884,9 @@ dependencies = [ [[package]] name = "zip" -version = "8.5.1" +version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ "aes", "bzip2", @@ -11866,7 +11901,7 @@ dependencies = [ "memchr", "pbkdf2", "ppmd-rust", - "sha1", + "sha1 0.11.0", "time", "typed-path", "zeroize", From bbfecc839636d8cd3b6fc198025abae376e90b4c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:02:52 +0000 Subject: [PATCH 198/250] Update dependency com.adobe.testing:s3mock-testcontainers to v5 (#7655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [com.adobe.testing:s3mock-testcontainers](https://redirect.github.com/adobe/S3Mock) | `4.12.4` → `5.0.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.adobe.testing:s3mock-testcontainers/5.0.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.adobe.testing:s3mock-testcontainers/4.12.4/5.0.0?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
adobe/S3Mock (com.adobe.testing:s3mock-testcontainers) ### [`v5.0.0`](https://redirect.github.com/adobe/S3Mock/blob/HEAD/CHANGELOG.md#500) [Compare Source](https://redirect.github.com/adobe/S3Mock/compare/4.12.4...5.0.0) - Breaking changes - File system: Root directories created by S3Mock 4.x are not compatible with 5.x. Existing persisted data must be discarded. - The migration to Jackson 3 changes the serialized metadata format (`bucketMetadata.json`, `objectMetadata.json`). - "DisplayName" was removed from Owner (fixes [#​2738](https://redirect.github.com/adobe/S3Mock/issues/2738)). AWS APIs stopped returning "DisplayName" in November 2025. - Spring Boot 3.x → 4.x: Customers using S3Mock Java artifacts (JUnit 5 extension, TestNG listener, or embedding S3Mock directly) must ensure their project is compatible with Spring Boot 4.x and Spring Framework 7.x transitive dependencies. - Discontinued configuration properties and environment variables: - The following legacy environment variables / system properties that were deprecated in 4.5.0 are no longer supported. Use the current environment variables listed in [Configuration](README.md#configuration) instead. - `root` → use `COM_ADOBE_TESTING_S3MOCK_STORE_ROOT` - `initialBuckets` → use `COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS` - `validKmsKeys` → use `COM_ADOBE_TESTING_S3MOCK_STORE_VALID_KMS_KEYS` - `retainFilesOnExit` → use `COM_ADOBE_TESTING_S3MOCK_STORE_RETAIN_FILES_ON_EXIT` - `COM_ADOBE_TESTING_S3MOCK_REGION` → use `COM_ADOBE_TESTING_S3MOCK_STORE_REGION` - `http.port` → use Spring Boot's `SERVER_PORT` or `com.adobe.testing.s3mock.httpPort` - The legacy Spring configuration property prefix `com.adobe.testing.s3mock.domain.*` is no longer supported. Use the `com.adobe.testing.s3mock.store.*` prefix instead. - `com.adobe.testing.s3mock.domain.root` → use `com.adobe.testing.s3mock.store.root` - `com.adobe.testing.s3mock.domain.initialBuckets` → use `com.adobe.testing.s3mock.store.initialBuckets` - `com.adobe.testing.s3mock.domain.validKmsKeys` → use `com.adobe.testing.s3mock.store.validKmsKeys` - `com.adobe.testing.s3mock.domain.retainFilesOnExit` → use `com.adobe.testing.s3mock.store.retainFilesOnExit` - Features and fixes - Add "actuator" Spring profile that enables JMX and all Spring Boot Actuator endpoints. The "debug" and "trace" profiles now automatically activate the "actuator" profile via profile groups. Actuator endpoints are disabled by default. - Get object with range now returns the same headers as non-range calls. - Docker: Copy "s3mock.jar" to "/opt/", run with absolute path reference to avoid issues when working directory is changed. (fixes [#​2827](https://redirect.github.com/adobe/S3Mock/issues/2827)) - S3Mock supports ChecksumType.FULL\_OBJECT for Multipart uploads (fixes [#​2843](https://redirect.github.com/adobe/S3Mock/issues/2843)) - Return 412 on if-none-match=true when making CompleteMultipartRequest (fixes [#​2790](https://redirect.github.com/adobe/S3Mock/issues/2790)) - Refactorings - Use Jackson 3 annotations and mappers. - AWS has deprecated SDK for Java v1 and will remove support EOY 2025. - Remove Java SDK v1. - JUnit 4.x deprecation - Remove JUnit 4.x support. - Remove legacy properties for S3Mock configuration. - Move all controller-related code from "com.adobe.testing.s3mock" to "com.adobe.testing.s3mock.controller" package. - Remove Apache libraries like "commons-compress", "commons-codec" or "commons-lang3" from dependencies. Kotlin and Java standard library provide similar functionality. - Version updates (deliverable dependencies) - Bump spring-boot.version from 3.5.8 to 4.0.5 - Bump Java version from 17 to 25 - Compile with Java 25, target Java 17. [This follows Spring guidance](https://spring.io/blog/2025/11/13/spring-framework-7-0-general-availability) - Docker container runs Java 25 - Bump testcontainers.version from 1.21.3 to 2.0.2 - Bump kotlin.version from 2.2.21 to 2.3.10 - Compile with Kotlin 2.3, target Kotlin 2.2. [This follows Spring guidance](https://spring.io/blog/2025/12/18/next-level-kotlin-support-in-spring-boot-4#kotlin-2-baseline) - Bump alpine from 3.23.0 to 3.23.3 in /docker - Bump org.testng:testng from 7.11.0 to 7.12.0 - Bump aws-v2.version from 2.40.0 to 2.42.29 - Bump org.jetbrains:annotations from 26.0.2-1 to 26.1.0 - Version updates (build dependencies) - Bump aws.sdk.kotlin:s3-jvm from 1.5.95 to 1.6.52 - Bump Maven to 4.0.0-rc5 (TODO: update to 4.0.0) - Bump com.github.gantsign.maven:ktlint-maven-plugin from 3.5.0 to 3.7.0 - Bump com.puppycrawl.tools:checkstyle from 13.2.0 to 13.4.0 - Bump io.fabric8:docker-maven-plugin from 0.48.0 to 0.48.1 - Bump org.apache.maven.plugins:maven-compiler-plugin from 3.14.1 to 3.15.0 - Bump org.apache.maven.plugins:maven-dependency-plugin from 3.9.0 to 3.10.0 - Bump org.apache.maven.plugins:maven-failsafe-plugin from 3.5.4 to 3.5.5 - Bump org.apache.maven.plugins:maven-release-plugin from 3.3.0 to 3.3.1 - Bump org.apache.maven.plugins:maven-resources-plugin from 3.4.0 to 3.5.0 - Bump org.apache.maven.plugins:maven-surefire-plugin from 3.5.4 to 3.5.5 - Bump org.codehaus.mojo:exec-maven-plugin from 3.6.2 to 3.6.3 - Bump org.jetbrains.dokka:dokka-maven-plugin from 2.1.0 to 2.2.0 - Bump org.mockito.kotlin:mockito-kotlin from 6.1.0 to 6.3.0 - Bump digital.pragmatech.testing:spring-test-profiler from 0.0.14 to 0.1.0 - Bump actions/checkout from 6.0.1 to 6.0.2 - Bump actions/dependency-review-action from 4.8.2 to 4.9.0 - Bump actions/setup-java from 5.0.0 to 5.2.0 - Bump actions/stale from 10.1.1 to 10.2.0 - Bump actions/upload-artifact from 5.0.0 to 7.0.0 - Bump docker/setup-qemu-action from 3.7.0 to 4.0.0 - Bump github/codeql-action from 4.31.6 to 4.35.1 - Bump step-security/harden-runner from 2.13.3 to 2.16.1
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gradle/libs.versions.toml b/java/gradle/libs.versions.toml index 6854f2d5403..67b3155c80c 100644 --- a/java/gradle/libs.versions.toml +++ b/java/gradle/libs.versions.toml @@ -14,7 +14,7 @@ protobuf = "4.34.1" slf4j = "2.0.17" spark3 = "3.5.8" spark4 = "4.1.1" -s3mock = "4.12.4" +s3mock = "5.0.0" testcontainers-jupiter = "1.21.4" [libraries] From 2d19a6551e99b199b6d077dad14906aaffbbd092 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:05:03 +0000 Subject: [PATCH 199/250] Update actions/github-script action to v9 (#7652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [actions/github-script](https://redirect.github.com/actions/github-script) | action | major | `v7` → `v9` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
actions/github-script (actions/github-script) ### [`v9`](https://redirect.github.com/actions/github-script/compare/v8...v9) [Compare Source](https://redirect.github.com/actions/github-script/compare/v9.0.0...v9.0.0) ### [`v9.0.0`](https://redirect.github.com/actions/github-script/releases/tag/v9.0.0) [Compare Source](https://redirect.github.com/actions/github-script/compare/v8.0.0...v9.0.0) **New features:** - **`getOctokit` factory function** — Available directly in the script context. Create additional authenticated Octokit clients with different tokens for multi-token workflows, GitHub App tokens, and cross-org access. See [Creating additional clients with `getOctokit`](https://redirect.github.com/actions/github-script#creating-additional-clients-with-getoctokit) for details and examples. - **Orchestration ID in user-agent** — The `ACTIONS_ORCHESTRATION_ID` environment variable is automatically appended to the user-agent string for request tracing. **Breaking changes:** - **`require('@​actions/github')` no longer works in scripts.** The upgrade to `@actions/github` v9 (ESM-only) means `require('@​actions/github')` will fail at runtime. If you previously used patterns like `const { getOctokit } = require('@​actions/github')` to create secondary clients, use the new injected `getOctokit` function instead — it's available directly in the script context with no imports needed. - `getOctokit` is now an injected function parameter. Scripts that declare `const getOctokit = ...` or `let getOctokit = ...` will get a `SyntaxError` because JavaScript does not allow `const`/`let` redeclaration of function parameters. Use the injected `getOctokit` directly, or use `var getOctokit = ...` if you need to redeclare it. - If your script accesses other `@actions/github` internals beyond the standard `github`/`octokit` client, you may need to update those references for v9 compatibility. ##### What's Changed - Add ACTIONS\_ORCHESTRATION\_ID to user-agent string by [@​Copilot](https://redirect.github.com/Copilot) in [#​695](https://redirect.github.com/actions/github-script/pull/695) - ci: use deployment: false for integration test environments by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​712](https://redirect.github.com/actions/github-script/pull/712) - feat!: add getOctokit to script context, upgrade [@​actions/github](https://redirect.github.com/actions/github) v9, [@​octokit/core](https://redirect.github.com/octokit/core) v7, and related packages by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​700](https://redirect.github.com/actions/github-script/pull/700) ##### New Contributors - [@​Copilot](https://redirect.github.com/Copilot) made their first contribution in [#​695](https://redirect.github.com/actions/github-script/pull/695) **Full Changelog**: ### [`v8`](https://redirect.github.com/actions/github-script/releases/tag/v8): .0.0 [Compare Source](https://redirect.github.com/actions/github-script/compare/v8.0.0...v8.0.0) ##### What's Changed - Update Node.js version support to 24.x by [@​salmanmkc](https://redirect.github.com/salmanmkc) in [#​637](https://redirect.github.com/actions/github-script/pull/637) - README for updating actions/github-script from v7 to v8 by [@​sneha-krip](https://redirect.github.com/sneha-krip) in [#​653](https://redirect.github.com/actions/github-script/pull/653) ##### ⚠️ Minimum Compatible Runner Version **v2.327.1**\ [Release Notes](https://redirect.github.com/actions/runner/releases/tag/v2.327.1) Make sure your runner is updated to this version or newer to use this release. ##### New Contributors - [@​salmanmkc](https://redirect.github.com/salmanmkc) made their first contribution in [#​637](https://redirect.github.com/actions/github-script/pull/637) - [@​sneha-krip](https://redirect.github.com/sneha-krip) made their first contribution in [#​653](https://redirect.github.com/actions/github-script/pull/653) **Full Changelog**: ### [`v8.0.0`](https://redirect.github.com/actions/github-script/compare/v7.1.0...v8.0.0) [Compare Source](https://redirect.github.com/actions/github-script/compare/v7.1.0...v8.0.0) ### [`v7.1.0`](https://redirect.github.com/actions/github-script/releases/tag/v7.1.0) [Compare Source](https://redirect.github.com/actions/github-script/compare/v7.0.1...v7.1.0) #### What's Changed - Upgrade husky to v9 by [@​benelan](https://redirect.github.com/benelan) in [#​482](https://redirect.github.com/actions/github-script/pull/482) - Add workflow file for publishing releases to immutable action package by [@​Jcambass](https://redirect.github.com/Jcambass) in [#​485](https://redirect.github.com/actions/github-script/pull/485) - Upgrade IA Publish by [@​Jcambass](https://redirect.github.com/Jcambass) in [#​486](https://redirect.github.com/actions/github-script/pull/486) - Fix workflow status badges by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​497](https://redirect.github.com/actions/github-script/pull/497) - Update usage of `actions/upload-artifact` by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​512](https://redirect.github.com/actions/github-script/pull/512) - Clear up package name confusion by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​514](https://redirect.github.com/actions/github-script/pull/514) - Update dependencies with `npm audit fix` by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​515](https://redirect.github.com/actions/github-script/pull/515) - Specify that the used script is JavaScript by [@​timotk](https://redirect.github.com/timotk) in [#​478](https://redirect.github.com/actions/github-script/pull/478) - chore: Add Dependabot for NPM and Actions by [@​nschonni](https://redirect.github.com/nschonni) in [#​472](https://redirect.github.com/actions/github-script/pull/472) - Define `permissions` in workflows and update actions by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​531](https://redirect.github.com/actions/github-script/pull/531) - chore: Add Dependabot for .github/actions/install-dependencies by [@​nschonni](https://redirect.github.com/nschonni) in [#​532](https://redirect.github.com/actions/github-script/pull/532) - chore: Remove .vscode settings by [@​nschonni](https://redirect.github.com/nschonni) in [#​533](https://redirect.github.com/actions/github-script/pull/533) - ci: Use github/setup-licensed by [@​nschonni](https://redirect.github.com/nschonni) in [#​473](https://redirect.github.com/actions/github-script/pull/473) - make octokit instance available as octokit on top of github, to make it easier to seamlessly copy examples from GitHub rest api or octokit documentations by [@​iamstarkov](https://redirect.github.com/iamstarkov) in [#​508](https://redirect.github.com/actions/github-script/pull/508) - Remove `octokit` README updates for v7 by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​557](https://redirect.github.com/actions/github-script/pull/557) - docs: add "exec" usage examples by [@​neilime](https://redirect.github.com/neilime) in [#​546](https://redirect.github.com/actions/github-script/pull/546) - Bump ruby/setup-ruby from 1.213.0 to 1.222.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​563](https://redirect.github.com/actions/github-script/pull/563) - Bump ruby/setup-ruby from 1.222.0 to 1.229.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [#​575](https://redirect.github.com/actions/github-script/pull/575) - Clearly document passing inputs to the `script` by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​603](https://redirect.github.com/actions/github-script/pull/603) - Update README.md by [@​nebuk89](https://redirect.github.com/nebuk89) in [#​610](https://redirect.github.com/actions/github-script/pull/610) #### New Contributors - [@​benelan](https://redirect.github.com/benelan) made their first contribution in [#​482](https://redirect.github.com/actions/github-script/pull/482) - [@​Jcambass](https://redirect.github.com/Jcambass) made their first contribution in [#​485](https://redirect.github.com/actions/github-script/pull/485) - [@​timotk](https://redirect.github.com/timotk) made their first contribution in [#​478](https://redirect.github.com/actions/github-script/pull/478) - [@​iamstarkov](https://redirect.github.com/iamstarkov) made their first contribution in [#​508](https://redirect.github.com/actions/github-script/pull/508) - [@​neilime](https://redirect.github.com/neilime) made their first contribution in [#​546](https://redirect.github.com/actions/github-script/pull/546) - [@​nebuk89](https://redirect.github.com/nebuk89) made their first contribution in [#​610](https://redirect.github.com/actions/github-script/pull/610) **Full Changelog**: ### [`v7.0.1`](https://redirect.github.com/actions/github-script/releases/tag/v7.0.1) [Compare Source](https://redirect.github.com/actions/github-script/compare/v7...v7.0.1) #### What's Changed - Avoid setting `baseUrl` to undefined when input is not provided by [@​joshmgross](https://redirect.github.com/joshmgross) in [#​439](https://redirect.github.com/actions/github-script/pull/439) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- .github/workflows/claude-review.yml | 4 ++-- .github/workflows/claude-write.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index a8323f82b51..e274a387756 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -50,7 +50,7 @@ jobs: steps: - name: Check whether this PR event is allowed to reach Claude id: gate - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ github.token }} script: | @@ -178,7 +178,7 @@ jobs: pull-requests: write steps: - name: Comment with the refusal reason - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: script: | const reason = ${{ toJSON(needs.gate.outputs.reason) }}; diff --git a/.github/workflows/claude-write.yml b/.github/workflows/claude-write.yml index d4d4c6ad03f..753e3167d2d 100644 --- a/.github/workflows/claude-write.yml +++ b/.github/workflows/claude-write.yml @@ -52,7 +52,7 @@ jobs: steps: - name: Check whether this event is allowed to reach Claude id: gate - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: github-token: ${{ github.token }} script: | From 83098427e3206d03a111b91834710c0b835a1466 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:06:11 +0000 Subject: [PATCH 200/250] Update dependency node to v24 (#7661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [node](https://redirect.github.com/actions/node-versions) | uses-with | major | `22` → `24` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
actions/node-versions (node) ### [`v24.15.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.15.0-24511264946): 24.15.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.14.1-23521883727...24.15.0-24511264946) Node.js 24.15.0 ### [`v24.14.1`](https://redirect.github.com/actions/node-versions/releases/tag/24.14.1-23521883727): 24.14.1 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.14.0-22380502845...24.14.1-23521883727) Node.js 24.14.1 ### [`v24.14.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.14.0-22380502845): 24.14.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.13.1-21889660756...24.14.0-22380502845) Node.js 24.14.0 ### [`v24.13.1`](https://redirect.github.com/actions/node-versions/releases/tag/24.13.1-21889660756): 24.13.1 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.13.0-20981653924...24.13.1-21889660756) Node.js 24.13.1 ### [`v24.13.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.13.0-20981653924): 24.13.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.12.0-20140960970...24.13.0-20981653924) Node.js 24.13.0 ### [`v24.12.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.12.0-20140960970): 24.12.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.11.1-19282993875...24.12.0-20140960970) Node.js 24.12.0 ### [`v24.11.1`](https://redirect.github.com/actions/node-versions/releases/tag/24.11.1-19282993875): 24.11.1 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.11.0-18894910158...24.11.1-19282993875) Node.js 24.11.1 ### [`v24.11.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.11.0-18894910158): 24.11.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.10.0-18453495281...24.11.0-18894910158) Node.js 24.11.0 ### [`v24.10.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.10.0-18453495281): 24.10.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.9.0-18024003193...24.10.0-18453495281) Node.js 24.10.0 ### [`v24.9.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.9.0-18024003193): 24.9.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.8.0-17630522236...24.9.0-18024003193) Node.js 24.9.0 ### [`v24.8.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.8.0-17630522236): 24.8.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.7.0-17283839804...24.8.0-17630522236) Node.js 24.8.0 ### [`v24.7.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.7.0-17283839804): 24.7.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.6.0-16980723897...24.7.0-17283839804) Node.js 24.7.0 ### [`v24.6.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.6.0-16980723897): 24.6.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.5.0-16666195981...24.6.0-16980723897) Node.js 24.6.0 ### [`v24.5.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.5.0-16666195981): 24.5.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.4.1-16309768053...24.5.0-16666195981) Node.js 24.5.0 ### [`v24.4.1`](https://redirect.github.com/actions/node-versions/releases/tag/24.4.1-16309768053): 24.4.1 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.4.0-16210503505...24.4.1-16309768053) Node.js 24.4.1 ### [`v24.4.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.4.0-16210503505): 24.4.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.3.0-15866716565...24.4.0-16210503505) Node.js 24.4.0 ### [`v24.3.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.3.0-15866716565): 24.3.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.2.0-15549907769...24.3.0-15866716565) Node.js 24.3.0 ### [`v24.2.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.2.0-15549907769): 24.2.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.1.0-15177436545...24.2.0-15549907769) Node.js 24.2.0 ### [`v24.1.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.1.0-15177436545): 24.1.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.0.2-15035852679...24.1.0-15177436545) Node.js 24.1.0 ### [`v24.0.2`](https://redirect.github.com/actions/node-versions/releases/tag/24.0.2-15035852679): 24.0.2 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.0.1-14928016774...24.0.2-15035852679) Node.js 24.0.2 ### [`v24.0.1`](https://redirect.github.com/actions/node-versions/releases/tag/24.0.1-14928016774): 24.0.1 [Compare Source](https://redirect.github.com/actions/node-versions/compare/24.0.0-14863421234...24.0.1-14928016774) Node.js 24.0.1 ### [`v24.0.0`](https://redirect.github.com/actions/node-versions/releases/tag/24.0.0-14863421234): 24.0.0 [Compare Source](https://redirect.github.com/actions/node-versions/compare/22.22.2-23521889093...24.0.0-14863421234) Node.js 24.0.0
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 3c0381a30cb..d2249f65e0f 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -54,7 +54,7 @@ jobs: working-directory: . - uses: actions/setup-node@v6 with: - node-version: "22" + node-version: "24" cache: "npm" cache-dependency-path: vortex-web/package-lock.json - run: npm ci @@ -83,7 +83,7 @@ jobs: working-directory: . - uses: actions/setup-node@v6 with: - node-version: "22" + node-version: "24" cache: "npm" cache-dependency-path: vortex-web/package-lock.json - run: npm ci From 20f11d0db43ad1794f664b7ff77fd253daab7241 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:15:21 +0000 Subject: [PATCH 201/250] Update dorny/paths-filter action to v4 (#7665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [dorny/paths-filter](https://redirect.github.com/dorny/paths-filter) | action | major | `v3` → `v4` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
dorny/paths-filter (dorny/paths-filter) ### [`v4.0.1`](https://redirect.github.com/dorny/paths-filter/releases/tag/v4.0.1) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v4...v4.0.1) #### What's Changed - Support merge queue by [@​masaru-iritani](https://redirect.github.com/masaru-iritani) in [#​255](https://redirect.github.com/dorny/paths-filter/pull/255) #### New Contributors - [@​masaru-iritani](https://redirect.github.com/masaru-iritani) made their first contribution in [#​255](https://redirect.github.com/dorny/paths-filter/pull/255) **Full Changelog**: ### [`v4.0.0`](https://redirect.github.com/dorny/paths-filter/blob/HEAD/CHANGELOG.md#v400) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v4...v4) - [Update action runtime to node24](https://redirect.github.com/dorny/paths-filter/pull/294) ### [`v4`](https://redirect.github.com/dorny/paths-filter/blob/HEAD/CHANGELOG.md#v400) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v3.0.3...v4) - [Update action runtime to node24](https://redirect.github.com/dorny/paths-filter/pull/294) ### [`v3.0.3`](https://redirect.github.com/dorny/paths-filter/blob/HEAD/CHANGELOG.md#v303) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v3.0.2...v3.0.3) - [Add missing predicate-quantifier](https://redirect.github.com/dorny/paths-filter/pull/279) ### [`v3.0.2`](https://redirect.github.com/dorny/paths-filter/blob/HEAD/CHANGELOG.md#v302) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v3.0.1...v3.0.2) - [Add config parameter for predicate quantifier](https://redirect.github.com/dorny/paths-filter/pull/224) ### [`v3.0.1`](https://redirect.github.com/dorny/paths-filter/blob/HEAD/CHANGELOG.md#v301) [Compare Source](https://redirect.github.com/dorny/paths-filter/compare/v3...v3.0.1) - [Compare base and ref when token is empty](https://redirect.github.com/dorny/paths-filter/pull/133)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- .github/workflows/web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index d2249f65e0f..77abe7b9013 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -29,7 +29,7 @@ jobs: web: ${{ steps.filter.outputs.web }} steps: - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4 id: filter with: filters: | From 74d803be99cec66c248d3a6855e518951fc98a13 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 27 Apr 2026 13:15:42 +0100 Subject: [PATCH 202/250] feat: dict into parts (#7668) Add a `DictParts` used to access all the element of a owned dict array Signed-off-by: Joe Isaacs --- vortex-array/public-api.lock | 22 ++++++++++- vortex-array/src/arrays/dict/array.rs | 36 ++++++++++++++++- vortex-array/src/arrays/dict/execute.rs | 6 ++- vortex-array/src/arrays/dict/mod.rs | 2 - vortex-array/src/arrays/dict/vtable/mod.rs | 45 +++++++++------------- 5 files changed, 77 insertions(+), 34 deletions(-) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 02d767b3cb1..2f36c30a472 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -2098,6 +2098,14 @@ pub fn vortex_array::arrays::dict::DictMetadata::clear(&mut self) pub fn vortex_array::arrays::dict::DictMetadata::encoded_len(&self) -> usize +pub struct vortex_array::arrays::dict::DictParts + +pub vortex_array::arrays::dict::DictParts::codes: vortex_array::ArrayRef + +pub vortex_array::arrays::dict::DictParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::dict::DictParts::values: vortex_array::ArrayRef + pub struct vortex_array::arrays::dict::DictSlots pub vortex_array::arrays::dict::DictSlots::codes: vortex_array::ArrayRef @@ -2204,6 +2212,14 @@ pub fn T::slots_view(&self) -> vortex_array::arrays::dict::DictSlotsView<'_> pub fn T::values(&self) -> &vortex_array::ArrayRef +pub trait vortex_array::arrays::dict::DictOwnedExt + +pub fn vortex_array::arrays::dict::DictOwnedExt::into_parts(self) -> vortex_array::arrays::dict::DictParts + +impl vortex_array::arrays::dict::DictOwnedExt for vortex_array::Array + +pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::dict::DictParts + pub trait vortex_array::arrays::dict::TakeExecute: vortex_array::VTable pub fn vortex_array::arrays::dict::TakeExecute::take(array: vortex_array::ArrayView<'_, Self>, indices: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> @@ -2280,8 +2296,6 @@ impl vortex_array::arrays::dict::TakeReduce for vortex_array::arrays::null::Null pub fn vortex_array::arrays::null::Null::take(array: vortex_array::ArrayView<'_, vortex_array::arrays::null::Null>, indices: &vortex_array::ArrayRef) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::take_canonical(values: vortex_array::Canonical, codes: &vortex_array::arrays::PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult - pub type vortex_array::arrays::dict::DictArray = vortex_array::Array pub mod vortex_array::arrays::extension @@ -21888,6 +21902,10 @@ impl core::iter::traits::collect::FromIterator for vorte pub fn vortex_array::Array::from_iter>(iter: T) -> Self +impl vortex_array::arrays::dict::DictOwnedExt for vortex_array::Array + +pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::dict::DictParts + impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::Array pub fn vortex_array::Array::from_iter>>(iter: T) -> Self diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index e960c3f6764..eafbc9f6642 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -189,6 +189,40 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { } impl> DictArrayExt for T {} +/// Concrete parts of a [`DictArray`](super::DictArray) after iterative execution. +pub struct DictParts { + pub dtype: DType, + pub codes: ArrayRef, + pub values: ArrayRef, +} + +pub trait DictOwnedExt { + fn into_parts(self) -> DictParts; +} + +impl DictOwnedExt for Array { + fn into_parts(self) -> DictParts { + match self.try_into_parts() { + Ok(array_parts) => { + let slots = DictSlots::from_slots(array_parts.slots); + DictParts { + dtype: array_parts.dtype, + codes: slots.codes, + values: slots.values, + } + } + Err(array) => { + let slots = DictSlotsView::from_slots(array.slots()); + DictParts { + dtype: array.dtype().clone(), + codes: slots.codes.clone(), + values: slots.values.clone(), + } + } + } + } +} + impl Array { /// Build a new `DictArray` from its components, `codes` and `values`. pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { @@ -255,8 +289,6 @@ impl Array { #[cfg(test)] mod test { - #[expect(unused_imports)] - use itertools::Itertools; use rand::RngExt; use rand::SeedableRng; use rand::distr::Distribution; diff --git a/vortex-array/src/arrays/dict/execute.rs b/vortex-array/src/arrays/dict/execute.rs index 19eb6d56b3d..5b73ec39d9e 100644 --- a/vortex-array/src/arrays/dict/execute.rs +++ b/vortex-array/src/arrays/dict/execute.rs @@ -7,6 +7,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::Canonical; +use crate::CanonicalView; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::Bool; @@ -35,11 +36,12 @@ use crate::arrays::variant::VariantArrayExt; /// /// This is the core operation for dictionary decoding - it expands the dictionary /// by looking up each code in the values array. -pub fn take_canonical( - values: Canonical, +pub(crate) fn take_canonical( + values: CanonicalView, codes: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { + let values = Canonical::from(values); Ok(match values { Canonical::Null(a) => Canonical::Null(take_null(&a, codes)), Canonical::Bool(a) => Canonical::Bool(take_bool(&a, codes, ctx)?), diff --git a/vortex-array/src/arrays/dict/mod.rs b/vortex-array/src/arrays/dict/mod.rs index 7ba509e09e5..0414eea7def 100644 --- a/vortex-array/src/arrays/dict/mod.rs +++ b/vortex-array/src/arrays/dict/mod.rs @@ -17,8 +17,6 @@ pub use array::*; pub(crate) mod compute; mod execute; -pub use execute::take_canonical; - mod take; pub use take::*; diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index a67deff4c45..9f64e089bc5 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -5,7 +5,6 @@ use std::hash::Hasher; use kernel::PARENT_KERNELS; use prost::Message; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -16,9 +15,10 @@ use vortex_session::registry::CachedId; use super::DictData; use super::DictMetadata; +use super::DictOwnedExt; +use super::DictParts; use super::array::DictSlots; use super::array::DictSlotsView; -use super::take_canonical; use crate::AnyCanonical; use crate::ArrayEq; use crate::ArrayHash; @@ -27,6 +27,7 @@ use crate::Canonical; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; +use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::ConstantArray; @@ -34,6 +35,7 @@ use crate::arrays::Primitive; use crate::arrays::dict::DictArrayExt; use crate::arrays::dict::DictArraySlotsExt; use crate::arrays::dict::compute::rules::PARENT_RULES; +use crate::arrays::dict::execute::take_canonical; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::dtype::Nullability; @@ -43,6 +45,8 @@ use crate::executor::ExecutionResult; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::validity::Validity; + mod kernel; mod operations; mod validity; @@ -132,11 +136,10 @@ impl VTable for Dict { dtype: &DType, len: usize, metadata: &[u8], - _buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult> { + ) -> VortexResult> { let metadata = DictMetadata::decode(metadata)?; if children.len() != 2 { vortex_bail!( @@ -155,12 +158,10 @@ impl VTable for Dict { let values = children.get(1, dtype, metadata.values_len as usize)?; let all_values_referenced = metadata.all_values_referenced.unwrap_or(false); - Ok( - crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, unsafe { - DictData::new_unchecked().set_all_values_referenced(all_values_referenced) - }) - .with_slots(vec![Some(codes), Some(values)]), - ) + Ok(ArrayParts::new(self.clone(), dtype.clone(), len, unsafe { + DictData::new_unchecked().set_all_values_referenced(all_values_referenced) + }) + .with_slots(vec![Some(codes), Some(values)])) } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { @@ -177,10 +178,7 @@ impl VTable for Dict { let array = require_child!(array, array.codes(), DictSlots::CODES => Primitive); - // TODO(joe): use stat get instead computing. - // Also not the check to do here it take value validity using code validity, but this approx - // is correct. - if array.codes().all_invalid(ctx)? { + if matches!(array.codes().validity()?, Validity::AllInvalid) { return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), @@ -189,18 +187,13 @@ impl VTable for Dict { let array = require_child!(array, array.values(), DictSlots::VALUES => AnyCanonical); - let codes = array - .codes() - .clone() - .try_downcast::() - .ok() - .vortex_expect("must be primitive"); - let values = array.values().clone(); - debug_assert!(values.is_canonical()); - // TODO: add canonical owned cast. - let values = values.execute::(ctx)?; - - Ok(ExecutionResult::done(take_canonical(values, &codes, ctx)?)) + let DictParts { values, codes, .. } = array.into_parts(); + + Ok(ExecutionResult::done(take_canonical( + values.as_::(), + &codes.downcast::(), + ctx, + )?)) } fn reduce_parent( From c2ab5315338cb05ae6d8d051998961584b5f436f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:27:13 +0000 Subject: [PATCH 203/250] Update all patch updates (#7648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [cc](https://redirect.github.com/rust-lang/cc-rs) | workspace.dependencies | patch | `1.2.60` → `1.2.61` | | [jiff](https://redirect.github.com/BurntSushi/jiff) | workspace.dependencies | patch | `0.2.23` → `0.2.24` | | [lance](https://redirect.github.com/lance-format/lance) | dependencies | patch | `4.0.0` → `4.0.1` | | [lance-encoding](https://redirect.github.com/lance-format/lance) | dependencies | patch | `4.0.0` → `4.0.1` | | [mozilla-actions/sccache-action](https://redirect.github.com/mozilla-actions/sccache-action) | action | patch | `v0.0.9` → `v0.0.10` | | [roaring](https://redirect.github.com/RoaringBitmap/roaring-rs) | workspace.dependencies | patch | `0.11.3` → `0.11.4` | | [spiraldb/actions](https://redirect.github.com/spiraldb/actions) | action | patch | `0.18.5` → `0.18.6` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
rust-lang/cc-rs (cc) ### [`v1.2.61`](https://redirect.github.com/rust-lang/cc-rs/blob/HEAD/CHANGELOG.md#1261---2026-04-24) [Compare Source](https://redirect.github.com/rust-lang/cc-rs/compare/cc-v1.2.60...cc-v1.2.61) ##### Other - fix `OutputKind::Capture` documentation ([#​1705](https://redirect.github.com/rust-lang/cc-rs/pull/1705))
BurntSushi/jiff (jiff) ### [`v0.2.24`](https://redirect.github.com/BurntSushi/jiff/blob/HEAD/CHANGELOG.md#0224-2026-04-23) [Compare Source](https://redirect.github.com/BurntSushi/jiff/compare/0.2.23...0.2.24) \=================== This release primarily adds a new `memory_usage` routine for reporting heap allocation sizes for the `TimeZone` and `Zoned` types. This release also acknowledges and updates the timeline expectations for a Jiff 1.0 release in `README.md`. Enhancements: - [#​520](https://redirect.github.com/BurntSushi/jiff/issues/520): Add `memory_usage` to the `TimeZone` and `Zoned` types. - [#​535](https://redirect.github.com/BurntSushi/jiff/pull/535): Improve comment in `Span::checked_add` example. Bug fixes: - [#​541](https://redirect.github.com/BurntSushi/jiff/pull/541): Update Jiff 1.0 timeline.
lance-format/lance (lance) ### [`v4.0.1`](https://redirect.github.com/lance-format/lance/releases/tag/v4.0.1) [Compare Source](https://redirect.github.com/lance-format/lance/compare/v4.0.0...v4.0.1) ##### What's Changed **Full Changelog**:
mozilla-actions/sccache-action (mozilla-actions/sccache-action) ### [`v0.0.10`](https://redirect.github.com/Mozilla-Actions/sccache-action/releases/tag/v0.0.10) [Compare Source](https://redirect.github.com/mozilla-actions/sccache-action/compare/v0.0.9...v0.0.10) ##### What's Changed - Use tar on all platforms by [@​brianmichel](https://redirect.github.com/brianmichel) in [Mozilla-Actions#193](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/193) - Bump eslint-plugin-prettier from 5.2.3 to 5.2.5 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#196](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/196) - Bump typescript from 5.7.2 to 5.8.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#195](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/195) - Bump [@​types/node](https://redirect.github.com/types/node) from 22.13.0 to 22.13.17 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#194](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/194) - Bump undici from 5.28.5 to 5.29.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#204](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/204) - Bump eslint-config-prettier from 9.1.0 to 10.1.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#201](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/201) - Bump ts-jest from 29.2.6 to 29.3.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#200](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/200) - Bump eslint-plugin-prettier from 5.2.5 to 5.5.4 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#221](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/221) - Bump eslint-config-prettier from 10.1.2 to 10.1.8 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#214](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/214) - Bump the github-actions group across 1 directory with 2 updates by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#220](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/220) - Add job id to annotation title by [@​wetheredge](https://redirect.github.com/wetheredge) in [Mozilla-Actions#212](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/212) - Bump [@​actions/io](https://redirect.github.com/actions/io) from 1.1.3 to 2.0.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#228](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/228) - Bump eslint-plugin-import from 2.31.0 to 2.32.0 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#227](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/227) - Bump actions/setup-node from 5 to 6 in the github-actions group by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#226](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/226) - Bump [@​actions/github](https://redirect.github.com/actions/github) from 6.0.0 to 6.0.1 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#233](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/233) - Bump minimatch by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#239](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/239) - Bump actions/checkout from 5 to 6 in the github-actions group by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#232](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/232) - Bump ts-jest from 29.3.2 to 29.4.6 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#240](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/240) - Bump to `node24` by [@​cakebaker](https://redirect.github.com/cakebaker) in [Mozilla-Actions#245](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/245) - Bump flatted from 3.3.1 to 3.4.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#248](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/248) - Bump picomatch from 2.3.1 to 2.3.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#249](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/249) - Bump handlebars from 4.7.8 to 4.7.9 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#250](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/250) - Fix code block formatting in README.md by [@​baseplate-admin](https://redirect.github.com/baseplate-admin) in [Mozilla-Actions#246](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/246) - Bump js-yaml from 3.14.1 to 3.14.2 by [@​dependabot](https://redirect.github.com/dependabot)\[bot] in [Mozilla-Actions#231](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/231) - prepare version 0.0.10 by [@​sylvestre](https://redirect.github.com/sylvestre) in [Mozilla-Actions#251](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/251) ##### New Contributors - [@​brianmichel](https://redirect.github.com/brianmichel) made their first contribution in [Mozilla-Actions#193](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/193) - [@​wetheredge](https://redirect.github.com/wetheredge) made their first contribution in [Mozilla-Actions#212](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/212) - [@​baseplate-admin](https://redirect.github.com/baseplate-admin) made their first contribution in [Mozilla-Actions#246](https://redirect.github.com/Mozilla-Actions/sccache-action/pull/246) **Full Changelog**:
RoaringBitmap/roaring-rs (roaring) ### [`v0.11.4`](https://redirect.github.com/RoaringBitmap/roaring-rs/releases/tag/v0.11.4) [Compare Source](https://redirect.github.com/RoaringBitmap/roaring-rs/compare/v0.11.3...v0.11.4) #### What's Changed - fix: overflow on 32-bit builds by [@​lucascool12](https://redirect.github.com/lucascool12) in [#​345](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/345) - feat: impl Clone on tree\_map Iter by [@​lucascool12](https://redirect.github.com/lucascool12) in [#​344](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/344) - Implement Eq trait by [@​cormacrelf](https://redirect.github.com/cormacrelf) in [#​341](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/341) - Remove SIMD LaneCount, SupportedLaneCount with upstream changes by [@​xangelix](https://redirect.github.com/xangelix) in [#​347](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/347) - Add `RoaringTreemap::optimize` by [@​0xdeafbeef](https://redirect.github.com/0xdeafbeef) in [#​350](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/350) - Add RoaringTreemap::contains\_range and range\_cardinality by [@​dchristle](https://redirect.github.com/dchristle) in [#​351](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/351) - Fix interval store iterator off by one error by [@​Kerollmops](https://redirect.github.com/Kerollmops) in [#​353](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/353) #### New Contributors - [@​cormacrelf](https://redirect.github.com/cormacrelf) made their first contribution in [#​341](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/341) - [@​xangelix](https://redirect.github.com/xangelix) made their first contribution in [#​347](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/347) - [@​0xdeafbeef](https://redirect.github.com/0xdeafbeef) made their first contribution in [#​350](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/350) - [@​dchristle](https://redirect.github.com/dchristle) made their first contribution in [#​351](https://redirect.github.com/RoaringBitmap/roaring-rs/pull/351) **Full Changelog**:
spiraldb/actions (spiraldb/actions) ### [`v0.18.6`](https://redirect.github.com/spiraldb/actions/releases/tag/0.18.6): 🥂 [Compare Source](https://redirect.github.com/spiraldb/actions/compare/0.18.5...0.18.6) ##### Changes - `dependency_cache` to be optional [@​blaginin](https://redirect.github.com/blaginin) ([#​48](https://redirect.github.com/spiraldb/actions/issues/48)) - Update taiki-e/install-action digest to [`055f5df`](https://redirect.github.com/spiraldb/actions/commit/055f5df) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​40](https://redirect.github.com/spiraldb/actions/issues/40)) - Update astral-sh/setup-uv digest to [`37802ad`](https://redirect.github.com/spiraldb/actions/commit/37802ad) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​41](https://redirect.github.com/spiraldb/actions/issues/41)) - Update Swatinem/rust-cache digest to [`e18b497`](https://redirect.github.com/spiraldb/actions/commit/e18b497) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​43](https://redirect.github.com/spiraldb/actions/issues/43)) - Update release-drafter/release-drafter action to v7 @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​44](https://redirect.github.com/spiraldb/actions/issues/44)) - Pin dtolnay/rust-toolchain action to [`3c5f7ea`](https://redirect.github.com/spiraldb/actions/commit/3c5f7ea) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​46](https://redirect.github.com/spiraldb/actions/issues/46)) - Update actions/cache digest to [`27d5ce7`](https://redirect.github.com/spiraldb/actions/commit/27d5ce7) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​47](https://redirect.github.com/spiraldb/actions/issues/47)) - Update actions/cache digest to [`6682284`](https://redirect.github.com/spiraldb/actions/commit/6682284) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​45](https://redirect.github.com/spiraldb/actions/issues/45)) - Update release-drafter/release-drafter digest to [`6db134d`](https://redirect.github.com/spiraldb/actions/commit/6db134d) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​39](https://redirect.github.com/spiraldb/actions/issues/39)) - Update actions/cache action to v5 @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​38](https://redirect.github.com/spiraldb/actions/issues/38)) - Update Swatinem/rust-cache digest to [`779680d`](https://redirect.github.com/spiraldb/actions/commit/779680d) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​37](https://redirect.github.com/spiraldb/actions/issues/37)) - Update astral-sh/setup-uv digest to [`803947b`](https://redirect.github.com/spiraldb/actions/commit/803947b) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​36](https://redirect.github.com/spiraldb/actions/issues/36)) - Update taiki-e/install-action digest to [`a362280`](https://redirect.github.com/spiraldb/actions/commit/a362280) @​[renovate\[bot\]](https://redirect.github.com/apps/renovate) ([#​32](https://redirect.github.com/spiraldb/actions/issues/32))
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- .github/actions/setup-rust/action.yml | 2 +- .github/workflows/bench-pr.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/claude-write.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/publish-dry-runs.yml | 2 +- .github/workflows/sql-benchmarks.yml | 2 +- Cargo.lock | 82 +++++++++++++------------- 8 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 2af54b1c20f..82ffa9913c4 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -73,7 +73,7 @@ runs: - name: Rust Compile Cache if: inputs.enable-sccache == 'true' - uses: mozilla-actions/sccache-action@v0.0.9 + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Pre-start sccache server if: inputs.enable-sccache == 'true' diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 1a5de341360..4e3f6dca730 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -89,7 +89,7 @@ jobs: aws-region: us-east-1 - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: sync: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6b146d35b2..d9c7504ca93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v6 - - uses: spiraldb/actions/.github/actions/lint-toml@0.18.5 + - uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 validate-workflow-yaml: runs-on: ubuntu-latest diff --git a/.github/workflows/claude-write.yml b/.github/workflows/claude-write.yml index 753e3167d2d..86e09fd4cff 100644 --- a/.github/workflows/claude-write.yml +++ b/.github/workflows/claude-write.yml @@ -192,7 +192,7 @@ jobs: enable-sccache: "false" - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: sync: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b718d731baf..5a41908878a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -30,7 +30,7 @@ jobs: java-version: "17" distribution: "temurin" - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: sync: false prune-cache: false diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 5a2c9763cad..c7946ca3fc6 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -38,7 +38,7 @@ jobs: enable-sccache: "false" - uses: mlugg/setup-zig@v2.2.1 - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: sync: false prune-cache: false diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 8dcb56bceda..81871981788 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -290,7 +290,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@0.18.5 + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: sync: false diff --git a/Cargo.lock b/Cargo.lock index 92b8f535503..7292f998922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -967,7 +967,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -1298,9 +1298,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -1488,7 +1488,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.2", + "unicode-width 0.1.14", ] [[package]] @@ -4003,9 +4003,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195cc7f87e84bd695586137de99605e7e9579b26ec5e01b82960ddb4d0922f2" +checksum = "2b3a6f3550e61b999febd7168d462db953948eff4fc3448276b3d10d10324dbb" dependencies = [ "arrow-array 57.3.0", "rand 0.9.4", @@ -4852,9 +4852,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -4867,9 +4867,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -5047,9 +5047,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lance" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efe6c3ddd79cdfd2b7e1c23cafae52806906bc40fbd97de9e8cf2f8c7a75fc04" +checksum = "f63e285ceee2b4ca8eb3a8742266cc1ac8161599767a8ecb4d8c2f9fd43d8b29" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5113,9 +5113,9 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d9f5d95bdda2a2b790f1fb8028b5b6dcf661abeb3133a8bca0f3d24b054af87" +checksum = "5c55e62fc04422ef4cd4af6f863ada32641ae23124f9b2e9c567a40d617e8c97" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5153,9 +5153,9 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f827d6ab9f8f337a9509d5ad66a12f3314db8713868260521c344ef6135eb4e4" +checksum = "a48d232a2908645af0040f96c60a6387fea2df75e762d7033e93e17bb420c6a1" dependencies = [ "arrayref", "paste", @@ -5164,9 +5164,9 @@ dependencies = [ [[package]] name = "lance-core" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e25df6a79bf72ee6bcde0851f19b1cd36c5848c1b7db83340882d3c9fdecb" +checksum = "ce071baaff88fcdcf67f1dd0af54e17656f52ae75aaeb75f25f9cf4da29241f2" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5203,9 +5203,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93146de8ae720cb90edef81c2f2d0a1b065fc2f23ecff2419546f389b0fa70a4" +checksum = "11ebc97ee94fa8e1af6fd0520066c7e7e0eab38a100e750ba9aabad644c5aa57" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -5235,9 +5235,9 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccec8ce4d8e0a87a99c431dab2364398029f2ffb649c1a693c60c79e05ed30dd" +checksum = "9b90dbb2829875b3a3d00f88fd3a3e39a9e4c7d34c266f67da6550fcda54c76e" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -5255,9 +5255,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1aec0bbbac6bce829bc10f1ba066258126100596c375fb71908ecf11c2c2a5" +checksum = "65ec429cc2e18ad1b7e43cc7ec57a2f2e49229cfbd934da45e619751a886b8cd" dependencies = [ "arrow-arith 57.3.0", "arrow-array 57.3.0", @@ -5294,9 +5294,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a8c548804f5b17486dc2d3282356ed1957095a852780283bc401fdd69e9075" +checksum = "418afe3f82487615fa09222b95a4b5853103f3f0425996d24a537ca750381f83" dependencies = [ "arrow-arith 57.3.0", "arrow-array 57.3.0", @@ -5328,9 +5328,9 @@ dependencies = [ [[package]] name = "lance-index" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da212f0090ea59f79ac3686660f596520c167fe1cb5f408900cf71d215f0e03" +checksum = "936b3deeb6ee075646d18f27b01cf2d2e846c3f5f6c5fa45b30aa41dd5b4c4e2" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5394,9 +5394,9 @@ dependencies = [ [[package]] name = "lance-io" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d958eb4b56f03bbe0f5f85eb2b4e9657882812297b6f711f201ffc995f259f" +checksum = "4103e4cebe146af15bfb198c8142d6ea37d5b25fa04158bf2d9be4597bf174d3" dependencies = [ "arrow 57.3.0", "arrow-arith 57.3.0", @@ -5433,9 +5433,9 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0285b70da35def7ed95e150fae1d5308089554e1290470403ed3c50cb235bc5e" +checksum = "c00c7ad71eca93635404519e77add6689947c9342134bb2133578f81249bf809" dependencies = [ "arrow-array 57.3.0", "arrow-buffer 57.3.0", @@ -5451,9 +5451,9 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f78e2a828b654e062a495462c6e3eb4fcf0e7e907d761b8f217fc09ccd3ceac" +checksum = "e0c59a574e72a4b72da8096bcaaa1b1e5b44f6a83da164cc714c286fab30c369" dependencies = [ "arrow 57.3.0", "async-trait", @@ -5479,9 +5479,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df9c4adca3eb2074b3850432a9fb34248a3d90c3d6427d158b13ff9355664ee" +checksum = "943b9c503f23ebab9e0dbee356f528bc4cbcafded87a6848451f205b0bb473d7" dependencies = [ "arrow 57.3.0", "arrow-array 57.3.0", @@ -6874,7 +6874,7 @@ checksum = "044b1fa4f259f4df9ad5078e587b208f5d288a25407575fcddb9face30c7c692" dependencies = [ "rand 0.9.4", "socket2", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -7081,7 +7081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.10.5", "log", "multimap", "petgraph", @@ -7113,7 +7113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -7913,9 +7913,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", From d763ece9e2b2837db0247f72b57ff23d8e7435cc Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 27 Apr 2026 14:41:33 +0100 Subject: [PATCH 204/250] perf: zero copy validity export to duckdb (#7371) Zero copy export validity similarly to how we export data for Primitive or Decimal. --------- Signed-off-by: Joe Isaacs --- vortex-duckdb/cpp/include/duckdb_vx/vector.h | 9 +++ vortex-duckdb/cpp/vector.cpp | 38 ++++++++++ vortex-duckdb/src/convert/expr.rs | 4 +- vortex-duckdb/src/duckdb/vector.rs | 36 ++++++++++ vortex-duckdb/src/exporter/constant.rs | 1 - vortex-duckdb/src/exporter/list.rs | 20 +----- vortex-duckdb/src/exporter/list_view.rs | 20 +----- vortex-duckdb/src/exporter/primitive.rs | 72 +++++++++++++++++++ vortex-duckdb/src/exporter/validity.rs | 49 ++++++++++++- vortex-duckdb/src/exporter/vector.rs | 73 +++++++++++++------- vortex-io/src/read_at.rs | 20 ++++-- 11 files changed, 268 insertions(+), 74 deletions(-) diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector.h b/vortex-duckdb/cpp/include/duckdb_vx/vector.h index 906937f1629..036c4aeb313 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/vector.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/vector.h @@ -54,6 +54,15 @@ void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector); // Set the data pointer for the vector. This is the start of the values array in the vector. void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr); +// Set the validity pointer for the vector to external data, and store the buffer in auxiliary +// to keep it alive. The validity pointer is derived from data_ptr at the given u64 offset. +// The buffer is attached purely as a keep-alive. This enables zero-copy export of validity masks. +void duckdb_vx_vector_set_validity_data(duckdb_vector ffi_vector, + idx_t u64_offset, + idx_t capacity, + duckdb_vx_vector_buffer buffer, + void *data_ptr); + // Converts a duckdb flat vector into a Sequence vector. void duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity); diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp index 0328b7aff3e..e2153886a1b 100644 --- a/vortex-duckdb/cpp/vector.cpp +++ b/vortex-duckdb/cpp/vector.cpp @@ -57,6 +57,21 @@ class DataVector : public Vector { inline void SetDataPtr(data_ptr_t ptr) { data = ptr; }; + + inline ValidityMask &GetValidity() { + return validity; + }; +}; + +// Same hack for ValidityMask: access protected fields via inheritance. +class ExternalValidityMask : public ValidityMask { +public: + inline void SetExternal(idx_t u64_offset, idx_t cap, buffer_ptr keeper) { + validity_data = std::move(keeper); + // Derive validity_mask from validity_data so the two stay consistent. + validity_mask = reinterpret_cast(validity_data.get()) + u64_offset; + capacity = cap; + }; }; } // namespace vortex @@ -82,6 +97,29 @@ extern "C" void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *pt dvector->SetDataPtr((data_ptr_t)ptr); } +extern "C" void duckdb_vx_vector_set_validity_data(duckdb_vector ffi_vector, + idx_t u64_offset, + idx_t capacity, + duckdb_vx_vector_buffer buffer, + void *data_ptr) { + auto dvector = reinterpret_cast(ffi_vector); + auto &validity = dvector->GetValidity(); + // ExternalValidityMask adds no members, so this downcast only exposes + // access to ValidityMask's protected fields. + auto ext_validity = static_cast(&validity); + + // Use the shared_ptr aliasing constructor: the control block ref-counts the + // ExternalVectorBuffer (preventing the Rust buffer from being freed), + // while the stored pointer points to the explicit data_ptr. + auto ext_buf = reinterpret_cast *>(buffer); + auto keeper = shared_ptr>( + *ext_buf, + reinterpret_cast *>(data_ptr)); + + // Set validity_data, derive validity_mask from it at u64_offset, and set capacity. + ext_validity->SetExternal(u64_offset, capacity, std::move(keeper)); +} + extern "C" duckdb_value duckdb_vx_vector_get_value(duckdb_vector ffi_vector, idx_t index) { auto vector = reinterpret_cast(ffi_vector); auto value = duckdb::make_uniq(vector->GetValue(index)); diff --git a/vortex-duckdb/src/convert/expr.rs b/vortex-duckdb/src/convert/expr.rs index 3cbfec883b6..a9f11059805 100644 --- a/vortex-duckdb/src/convert/expr.rs +++ b/vortex-duckdb/src/convert/expr.rs @@ -48,7 +48,7 @@ pub fn try_from_bound_expression( value: &duckdb::ExpressionRef, ) -> VortexResult> { let Some(value) = value.as_class() else { - tracing::debug!("no expression class id {:?}", value.as_class_id()); + debug!("no expression class id {:?}", value.as_class_id()); return Ok(None); }; Ok(Some(match value { @@ -164,7 +164,7 @@ pub fn try_from_bound_expression( Like.new_expr(LikeOptions::default(), [value, pattern]) } _ => { - tracing::debug!("bound function {}", func.scalar_function.name()); + debug!("bound function {}", func.scalar_function.name()); return Ok(None); } }, diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs index 73aca417e49..ac652bed43f 100644 --- a/vortex-duckdb/src/duckdb/vector.rs +++ b/vortex-duckdb/src/duckdb/vector.rs @@ -24,9 +24,20 @@ use crate::duckdb::LogicalTypeRef; use crate::duckdb::SelectionVectorRef; use crate::duckdb::Value; use crate::duckdb::ValueRef; +use crate::duckdb::VectorBuffer; use crate::duckdb::VectorBufferRef; use crate::lifetime_wrapper; +/// External validity data for zero-copy export of validity masks to DuckDB. +/// +/// Holds a [`VectorBuffer`] as a keep-alive and a raw pointer to the validity bitmap. +pub(crate) struct ValidityData { + /// VectorBuffer that keeps the underlying memory alive via DuckDB's ref-counting. + pub(crate) shared_buffer: VectorBuffer, + /// Pointer to the raw validity bitmap data within the buffer. + pub(crate) data_ptr: *const u8, +} + /// Returns the internal vector size used by DuckDB at runtime. #[expect( clippy::cast_possible_truncation, @@ -151,6 +162,31 @@ impl VectorRef { unsafe { cpp::duckdb_vx_vector_set_data_ptr(self.as_ptr(), ptr as *mut c_void) } } + /// Sets the validity data for the vector from a [`ValidityData`]. The buffer is + /// attached purely as a keep-alive, and the data pointer is used as the validity data + /// at the given `u64_offset`. + /// + /// # Safety + /// + /// The data pointer must point to a valid `u64` array with at least + /// `u64_offset + capacity.div_ceil(64)` elements. + pub(crate) unsafe fn set_validity_data( + &self, + u64_offset: usize, + capacity: usize, + zero_copy: &ValidityData, + ) { + unsafe { + cpp::duckdb_vx_vector_set_validity_data( + self.as_ptr(), + u64_offset as idx_t, + capacity as idx_t, + zero_copy.shared_buffer.as_ptr(), + zero_copy.data_ptr as *mut c_void, + ) + } + } + /// Assigns the element at the specified index with a string value. /// FIXME(ngates): remove this. pub fn assign_string_element(&self, idx: usize, value: &CStr) { diff --git a/vortex-duckdb/src/exporter/constant.rs b/vortex-duckdb/src/exporter/constant.rs index ade5fece825..587fe9e9472 100644 --- a/vortex-duckdb/src/exporter/constant.rs +++ b/vortex-duckdb/src/exporter/constant.rs @@ -71,7 +71,6 @@ impl ColumnExporter for ConstantExporter { ) -> VortexResult<()> { match self.value.as_ref() { None => { - // TODO(ngates): would be good if DuckDB supported constant null vectors. vector.set_all_false_validity(); } Some(value) => { diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index c532242fae7..7e7d024e11c 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -19,6 +19,7 @@ use vortex::mask::Mask; use super::ConversionCache; use super::all_invalid; use super::new_array_exporter_with_flatten; +use super::validity; use crate::cpp; use crate::duckdb::LogicalType; use crate::duckdb::Vector; @@ -26,7 +27,6 @@ use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; struct ListExporter { - validity: Mask, /// We cache the child elements of our list array so that we don't have to export it every time, /// and we also share it across any other exporters who want to export this array. /// @@ -92,7 +92,6 @@ pub(crate) fn new_exporter( let boxed = match_each_integer_ptype!(offsets.ptype(), |O| { Box::new(ListExporter { - validity, duckdb_elements: shared_elements, offsets, num_elements, @@ -100,7 +99,7 @@ pub(crate) fn new_exporter( }) as Box }); - Ok(boxed) + Ok(validity::new_exporter(validity, boxed)) } impl ColumnExporter for ListExporter { @@ -111,21 +110,6 @@ impl ColumnExporter for ListExporter { vector: &mut VectorRef, _ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - // Verify that offset + len doesn't exceed the validity mask length. - assert!( - offset + len <= self.validity.len(), - "Export range [{}, {}) exceeds validity mask length {}", - offset, - offset + len, - self.validity.len() - ); - - // Set validity if necessary. - if unsafe { vector.set_validity(&self.validity, offset, len) } { - // All values are null, so no point copying the data. - return Ok(()); - } - let offsets = &self.offsets.as_slice::()[offset..offset + len + 1]; debug_assert_eq!(offsets.len(), len + 1); diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index 4e88d8f4199..5b2f19c09d7 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -19,6 +19,7 @@ use vortex::mask::Mask; use super::ConversionCache; use super::all_invalid; use super::new_array_exporter_with_flatten; +use super::validity; use crate::cpp; use crate::duckdb::LogicalType; use crate::duckdb::Vector; @@ -26,7 +27,6 @@ use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; struct ListViewExporter { - validity: Mask, /// We cache the child elements of our list array so that we don't have to export it every time, /// and we also share it across any other exporters who want to export this array. /// @@ -97,7 +97,6 @@ pub(crate) fn new_exporter( let boxed = match_each_integer_ptype!(offsets.ptype(), |O| { match_each_integer_ptype!(sizes.ptype(), |S| { Box::new(ListViewExporter { - validity, duckdb_elements: shared_elements, offsets, sizes, @@ -108,7 +107,7 @@ pub(crate) fn new_exporter( }) }); - Ok(boxed) + Ok(validity::new_exporter(validity, boxed)) } impl ColumnExporter for ListViewExporter { @@ -119,21 +118,6 @@ impl ColumnExporter for ListViewExporter vector: &mut VectorRef, _ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - // Verify that offset + len doesn't exceed the validity mask length. - assert!( - offset + len <= self.validity.len(), - "Export range [{}, {}) exceeds validity mask length {}", - offset, - offset + len, - self.validity.len() - ); - - // Set validity if necessary. - if unsafe { vector.set_validity(&self.validity, offset, len) } { - // All values are null, so no point copying the data. - return Ok(()); - } - let offsets = &self.offsets.as_slice::()[offset..offset + len]; let sizes = &self.sizes.as_slice::()[offset..offset + len]; debug_assert_eq!(offsets.len(), len); diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs index 1bbec2d79c8..a0b08c80e4b 100644 --- a/vortex-duckdb/src/exporter/primitive.rs +++ b/vortex-duckdb/src/exporter/primitive.rs @@ -97,6 +97,78 @@ mod tests { ); } + #[test] + fn test_primitive_exporter_with_nulls() { + let arr = PrimitiveArray::from_option_iter([Some(10i32), None, Some(30), None, Some(50)]); + + let mut chunk = DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + let mut ctx = SESSION.create_execution_ctx(); + + new_exporter(arr, &mut ctx) + .unwrap() + .export(0, 5, chunk.get_vector_mut(0), &mut ctx) + .unwrap(); + chunk.set_len(5); + + assert_eq!( + format!("{}", String::try_from(&*chunk).unwrap()), + r#"Chunk - [1 Columns] +- FLAT INTEGER: 5 = [ 10, NULL, 30, NULL, 50] +"# + ); + } + + /// Export a large nullable primitive array over many chunks to exercise the + /// zero-copy validity path. The non-zero-copy fallback currently panics, + /// so this test proves every chunk goes through the zero-copy branch. + #[test] + fn test_primitive_exporter_with_nulls_zero_copy() { + let vector_size = duckdb_vector_size(); + const NUM_CHUNKS: usize = 8; + let len = vector_size * NUM_CHUNKS; + + // Every 3rd element is null — guarantees mixed validity in every chunk. + #[expect(clippy::cast_possible_truncation, reason = "test data fits in i32")] + let arr = PrimitiveArray::from_option_iter( + (0..len).map(|i| if i % 3 == 1 { None } else { Some(i as i32) }), + ); + + let mut ctx = SESSION.create_execution_ctx(); + let exporter = new_exporter(arr, &mut ctx).unwrap(); + + for chunk_idx in 0..NUM_CHUNKS { + let mut chunk = + DataChunk::new([LogicalType::new(cpp::duckdb_type::DUCKDB_TYPE_INTEGER)]); + + // This will panic if the non-zero-copy path is hit. + exporter + .export( + chunk_idx * vector_size, + vector_size, + chunk.get_vector_mut(0), + &mut ctx, + ) + .unwrap(); + chunk.set_len(vector_size); + + let vec = chunk.get_vector(0); + for i in 0..vector_size { + let global_idx = chunk_idx * vector_size + i; + if global_idx % 3 == 1 { + assert!( + vec.row_is_null(i as u64), + "expected null at global index {global_idx}" + ); + } else { + assert!( + !vec.row_is_null(i as u64), + "expected non-null at global index {global_idx}" + ); + } + } + } + } + #[test] fn test_long_primitive_exporter() { let vector_size = duckdb_vector_size(); diff --git a/vortex-duckdb/src/exporter/validity.rs b/vortex-duckdb/src/exporter/validity.rs index 76107240212..d1fbf8123cb 100644 --- a/vortex-duckdb/src/exporter/validity.rs +++ b/vortex-duckdb/src/exporter/validity.rs @@ -5,14 +5,42 @@ use vortex::array::ExecutionCtx; use vortex::error::VortexResult; use vortex::mask::Mask; +use crate::duckdb::ValidityData; +use crate::duckdb::VectorBuffer; use crate::duckdb::VectorRef; use crate::exporter::ColumnExporter; struct ValidityExporter { mask: Mask, + /// If the mask's bit buffer is u64-aligned with no sub-byte offset, + /// we can zero-copy it into DuckDB. We hold the ValidityData to keep + /// the underlying memory alive via DuckDB's ref-counting. + zero_copy: Option, exporter: Box, } +/// Returns true if the bit buffer can be zero-copied as a DuckDB validity mask. +/// +/// Requirements: +/// - No sub-byte bit offset (offset == 0) +/// - The underlying byte buffer is u64-aligned +/// - The underlying byte buffer length is a multiple of 8 (so u64 reads are in-bounds) +fn can_zero_copy_validity(mask: &Mask) -> bool { + let Mask::Values(values) = mask else { + return false; + }; + let bit_buf = values.bit_buffer(); + if bit_buf.offset() != 0 { + return false; + } + let inner = bit_buf.inner(); + let slice = inner.as_slice(); + // DuckDB reads validity as u64 words, so the buffer must be u64-aligned and + // its length must be a multiple of 8 bytes to avoid out-of-bounds reads. + (slice.as_ptr() as usize).is_multiple_of(size_of::()) + && slice.len().is_multiple_of(size_of::()) +} + pub(crate) fn new_exporter( mask: Mask, exporter: Box, @@ -20,7 +48,22 @@ pub(crate) fn new_exporter( if mask.all_true() { exporter } else { - Box::new(ValidityExporter { mask, exporter }) + let zero_copy = can_zero_copy_validity(&mask).then(|| { + let Mask::Values(values) = &mask else { + unreachable!() + }; + let buffer = values.bit_buffer().inner().clone(); + let data_ptr = buffer.as_slice().as_ptr(); + ValidityData { + shared_buffer: VectorBuffer::new(buffer), + data_ptr, + } + }); + Box::new(ValidityExporter { + mask, + zero_copy, + exporter, + }) } } @@ -36,7 +79,9 @@ impl ColumnExporter for ValidityExporter { offset + len <= self.mask.len(), "cannot access outside of array" ); - if unsafe { vector.set_validity(&self.mask, offset, len) } { + if unsafe { + vector.set_validity_zero_copy(&self.mask, offset, len, self.zero_copy.as_ref()) + } { // All values are null, so no point copying the data. return Ok(()); } diff --git a/vortex-duckdb/src/exporter/vector.rs b/vortex-duckdb/src/exporter/vector.rs index e6ada5c7dd7..3ab0c484318 100644 --- a/vortex-duckdb/src/exporter/vector.rs +++ b/vortex-duckdb/src/exporter/vector.rs @@ -2,15 +2,30 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex::mask::Mask; -use vortex::mask::MaskValues; use crate::cpp::duckdb_vx_vector_set_all_valid; +use crate::duckdb::ValidityData; use crate::duckdb::Value; use crate::duckdb::VectorRef; use crate::exporter::copy_from_slice; impl VectorRef { - pub(super) unsafe fn set_validity(&mut self, mask: &Mask, offset: usize, len: usize) -> bool { + /// Returns true if all values are null (caller can skip data export). + pub unsafe fn set_validity(&mut self, mask: &Mask, offset: usize, len: usize) -> bool { + unsafe { self.set_validity_zero_copy(mask, offset, len, None) } + } + + /// Like [`set_validity`](Self::set_validity), but attempts a zero-copy path when + /// `zero_copy` is provided and the offset is u64-aligned. + /// + /// Returns true if all values are null (caller can skip data export). + pub(super) unsafe fn set_validity_zero_copy( + &mut self, + mask: &Mask, + offset: usize, + len: usize, + zero_copy: Option<&ValidityData>, + ) -> bool { match mask { Mask::AllTrue(_) => { self.set_all_true_validity(); @@ -20,31 +35,37 @@ impl VectorRef { self.set_all_false_validity(); true } - Mask::Values(arr) => self.set_validity_with_array(arr, len, offset), - } - } + Mask::Values(arr) => { + let true_count = arr.bit_buffer().slice(offset..(offset + len)).true_count(); + if true_count == len { + self.set_all_true_validity() + } else if true_count == 0 { + self.set_all_false_validity() + } else if let Some(zc) = zero_copy.filter(|_| offset.is_multiple_of(64)) { + let u64_offset = offset / 64; + // SAFETY: the underlying buffer is u64-aligned (checked in + // can_zero_copy_validity) and the VectorBuffer keeps the data alive. + // data_ptr points into the buffer at the start of the validity bitmap. + unsafe { self.set_validity_data(u64_offset, len, zc) }; + } else { + // If zero_copy is available and offset is aligned, we should + // have taken the branch above. Assert this invariant. + assert!( + zero_copy.is_none() || !offset.is_multiple_of(64), + "zero-copy validity available and offset {offset} is aligned \ + but copy path was taken" + ); + let source = arr.bit_buffer().inner().as_slice(); + copy_from_slice( + unsafe { self.ensure_validity_slice(len) }, + source, + offset, + len, + ); + } - fn set_validity_with_array(&mut self, arr: &MaskValues, len: usize, offset: usize) -> bool { - let true_count = arr.true_count(); - if true_count == arr.len() { - self.set_all_true_validity(); - return false; - } else if true_count == 0 { - self.set_all_false_validity(); - return true; - } - - let dest = unsafe { self.ensure_validity_slice(len) }; - let source = arr.bit_buffer().inner().as_slice(); - let ones = copy_from_slice(dest, source, offset, len); - if ones == 0 { - self.set_all_false_validity(); - true - } else if ones == len { - self.set_all_true_validity(); - false - } else { - false + true_count == 0 + } } } diff --git a/vortex-io/src/read_at.rs b/vortex-io/src/read_at.rs index 28cc5bf244b..aa9a8a03abf 100644 --- a/vortex-io/src/read_at.rs +++ b/vortex-io/src/read_at.rs @@ -229,10 +229,8 @@ impl InstrumentedReadAt { } } -// We implement drop for `InnerMetrics` so this will be logged only when we eventually drop the final instance of `InstrumentedRead` -impl Drop for InnerMetrics { - #[allow(clippy::cognitive_complexity)] - fn drop(&mut self) { +impl InnerMetrics { + fn log_sizes(&self) { tracing::debug!("Reads: {}", self.sizes.count()); if !self.sizes.is_empty() { tracing::debug!( @@ -245,10 +243,10 @@ impl Drop for InnerMetrics { .vortex_expect("must not be empty"), ); } + tracing::debug!("Total read size: {}", self.total_size.value()); + } - let total_size = self.total_size.value(); - tracing::debug!("Total read size: {total_size}"); - + fn log_durations(&self) { if !self.durations.is_empty() { tracing::debug!( "Read duration: p50={}ms p95={}ms p99={}ms p999={}ms", @@ -273,6 +271,14 @@ impl Drop for InnerMetrics { } } +// We implement drop for `InnerMetrics` so this will be logged only when we eventually drop the final instance of `InstrumentedRead` +impl Drop for InnerMetrics { + fn drop(&mut self) { + self.log_sizes(); + self.log_durations(); + } +} + impl VortexReadAt for InstrumentedReadAt { fn uri(&self) -> Option<&Arc> { self.read.uri() From e75c62c87029bb314c31dbfbd9d6c05a8cf3dc4c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:00:20 +0000 Subject: [PATCH 205/250] Pin dependencies (#7647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [EmbarkStudios/cargo-deny-action](https://redirect.github.com/EmbarkStudios/cargo-deny-action) | action | pinDigest | → `91bf2b6` | | [PyO3/maturin-action](https://redirect.github.com/PyO3/maturin-action) | action | pinDigest | → `e83996d` | | [Swatinem/rust-cache](https://redirect.github.com/Swatinem/rust-cache) | action | pinDigest | → `e18b497` | | [actions-ecosystem/action-remove-labels](https://redirect.github.com/actions-ecosystem/action-remove-labels) | action | pinDigest | → `2ce5d41` | | [actions/checkout](https://redirect.github.com/actions/checkout) | action | pinDigest | → `de0fac2` | | [actions/download-artifact](https://redirect.github.com/actions/download-artifact) | action | pinDigest | → `3e5f45b` | | [actions/setup-java](https://redirect.github.com/actions/setup-java) | action | pinDigest | → `be666c2` | | [actions/setup-node](https://redirect.github.com/actions/setup-node) | action | pinDigest | → `48b55a0` | | [actions/setup-python](https://redirect.github.com/actions/setup-python) | action | pinDigest | → `a309ff8` | | [actions/stale](https://redirect.github.com/actions/stale) | action | pinDigest | → `b5d41d4` | | [actions/upload-artifact](https://redirect.github.com/actions/upload-artifact) | action | pinDigest | → `043fb46` | | [actions/upload-pages-artifact](https://redirect.github.com/actions/upload-pages-artifact) | action | pinDigest | → `fc324d3` | | [anthropics/claude-code-action](https://redirect.github.com/anthropics/claude-code-action) | action | pinDigest | → `567fe95` | | [aws-actions/configure-aws-credentials](https://redirect.github.com/aws-actions/configure-aws-credentials) | action | pinDigest | → `ec61189` | | [cloudflare/wrangler-action](https://redirect.github.com/cloudflare/wrangler-action) | action | pinDigest | → `9acf94a` | | [codecov/codecov-action](https://redirect.github.com/codecov/codecov-action) | action | pinDigest | → `57e3a13` | | [crate-ci/typos](https://redirect.github.com/crate-ci/typos) | action | pinDigest | → `cf5f1c2` | | [docker/build-push-action](https://redirect.github.com/docker/build-push-action) | action | pinDigest | → `bcafcac` | | [docker/login-action](https://redirect.github.com/docker/login-action) | action | pinDigest | → `4907a6d` | | [docker/setup-buildx-action](https://redirect.github.com/docker/setup-buildx-action) | action | pinDigest | → `4d04d5d` | | [docker/setup-qemu-action](https://redirect.github.com/docker/setup-qemu-action) | action | pinDigest | → `ce36039` | | [dtolnay/rust-toolchain](https://redirect.github.com/dtolnay/rust-toolchain) | action | pinDigest | → `29eef33` | | [fsfe/reuse-action](https://redirect.github.com/fsfe/reuse-action) | action | pinDigest | → `676e2d5` | | [gradle/actions](https://redirect.github.com/gradle/actions) | action | pinDigest | → `50e97c2` | | [incident-io/github-action](https://redirect.github.com/incident-io/github-action) | action | pinDigest | → `7aa5f85` | | [mlugg/setup-zig](https://redirect.github.com/mlugg/setup-zig) | action | pinDigest | → `d1434d0` | | [octokit/request-action](https://redirect.github.com/octokit/request-action) | action | pinDigest | → `b91aaba` | | [polarsignals/gh-actions-ps-profiling](https://redirect.github.com/polarsignals/gh-actions-ps-profiling) | action | pinDigest | → `68ae857` | | [pypa/gh-action-pypi-publish](https://redirect.github.com/pypa/gh-action-pypi-publish) | action | pinDigest | → `cef2210` | | [release-drafter/release-drafter](https://redirect.github.com/release-drafter/release-drafter) | action | pinDigest | → `5de9358` | | [rui314/setup-mold](https://redirect.github.com/rui314/setup-mold) | action | pinDigest | → `9c9c13b` | | [ssciwr/doxygen-install](https://redirect.github.com/ssciwr/doxygen-install) | action | pinDigest | → `329d88f` | | [taiki-e/install-action](https://redirect.github.com/taiki-e/install-action) | action | pinDigest | → `cf525cb` | | [thollander/actions-comment-pull-request](https://redirect.github.com/thollander/actions-comment-pull-request) | action | pinDigest | → `24bffb9` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://redirect.github.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). --------- Signed-off-by: Robert Kruszewski Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robert Kruszewski --- .github/actions/alert-incident-io/action.yml | 2 +- .github/actions/setup-rust/action.yml | 4 +- .github/workflows/bench-dispatch.yml | 4 +- .github/workflows/bench-pr.yml | 10 ++--- .github/workflows/bench.yml | 10 ++--- .github/workflows/ci.yml | 40 +++++++++---------- .github/workflows/claude-review.yml | 2 +- .github/workflows/claude-write.yml | 2 +- .../workflows/close-fixed-fuzzer-issues.yml | 2 +- .github/workflows/codspeed.yml | 2 +- .github/workflows/compat-gen-upload.yml | 8 ++-- .github/workflows/compat-validation.yml | 2 +- .github/workflows/cuda.yaml | 10 ++--- .github/workflows/docs.yml | 12 +++--- .github/workflows/fuzz-coverage.yml | 4 +- .github/workflows/fuzzer-fix-automation.yml | 4 +- .github/workflows/labels.yml | 2 +- .../minimize_fuzz_corpus_workflow.yml | 2 +- .github/workflows/package.yml | 22 +++++----- .../workflows/publish-benchmarks-website.yml | 10 ++--- .github/workflows/publish-dry-runs.yml | 14 +++---- .github/workflows/publish.yml | 14 +++---- .github/workflows/release-binaries.yml | 2 +- .github/workflows/release-drafter.yml | 2 +- .github/workflows/report-fuzz-crash.yml | 8 ++-- .github/workflows/reuse.yml | 4 +- .github/workflows/run-fuzzer.yml | 6 +-- .github/workflows/rust-instrumented.yml | 10 ++--- .github/workflows/sql-benchmarks.yml | 14 +++---- .github/workflows/stale.yml | 2 +- .github/workflows/typos.yml | 4 +- .github/workflows/wasm-fuzz.yml | 4 +- .github/workflows/web.yml | 16 ++++---- 33 files changed, 127 insertions(+), 127 deletions(-) diff --git a/.github/actions/alert-incident-io/action.yml b/.github/actions/alert-incident-io/action.yml index 7eb835431dc..e375ffdc5a0 100644 --- a/.github/actions/alert-incident-io/action.yml +++ b/.github/actions/alert-incident-io/action.yml @@ -27,7 +27,7 @@ inputs: runs: using: "composite" steps: - - uses: incident-io/github-action@v1 + - uses: incident-io/github-action@7aa5f85e67679cd8fdf2a19aed3d5450335a004b # v1 with: api-key: ${{ inputs.api-key }} alert-source-id: 01KH4EYTH3HA4PDZPRAPEV1Q10 diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 82ffa9913c4..4f42d38167a 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -36,7 +36,7 @@ runs: - name: Install Mold if: runner.os == 'Linux' - uses: rui314/setup-mold@v1 + uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - name: Check for rustup id: check-rustup @@ -45,7 +45,7 @@ runs: - name: Rust Toolchain id: rust-toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable if: steps.check-rustup.outputs.exists != 'true' with: toolchain: "${{ steps.toolchain-config.outputs.toolchain }}" diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/bench-dispatch.yml index e18bcdca211..48bf6fbe4af 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/bench-dispatch.yml @@ -21,7 +21,7 @@ jobs: timeout-minutes: 10 if: github.event.label.name == 'action/benchmark' steps: - - uses: actions-ecosystem/action-remove-labels@v1 + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' with: labels: action/benchmark @@ -37,7 +37,7 @@ jobs: timeout-minutes: 10 if: github.event.label.name == 'action/benchmark-sql' steps: - - uses: actions-ecosystem/action-remove-labels@v1 + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' with: labels: action/benchmark-sql diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 4e3f6dca730..d50a5f04633 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -38,7 +38,7 @@ jobs: if: github.event.pull_request.head.repo.fork == false with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: ${{ github.event.pull_request.head.sha }} - name: Setup benchmark environment @@ -64,7 +64,7 @@ jobs: - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false - uses: polarsignals/gh-actions-ps-profiling@v0.8.1 + uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" @@ -83,7 +83,7 @@ jobs: - name: Setup AWS CLI if: github.event.pull_request.head.repo.fork == false - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole aws-region: us-east-1 @@ -117,14 +117,14 @@ jobs: - name: Comment PR if: github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: file-path: comment.md comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} - name: Comment PR on failure if: failure() && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: message: | # BENCHMARK FAILED diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0b3874d0c68..a437523b9be 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -16,9 +16,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole aws-region: us-east-1 @@ -54,7 +54,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - uses: ./.github/actions/setup-rust @@ -77,7 +77,7 @@ jobs: cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - name: Setup Polar Signals - uses: polarsignals/gh-actions-ps-profiling@v0.8.1 + uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" @@ -95,7 +95,7 @@ jobs: bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole aws-region: us-east-1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9c7504ca93..7ce22431213 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,14 +29,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 validate-workflow-yaml: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Validate YAML file run: | # Lint the workflows and yamllint's configuration file. @@ -57,7 +57,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild # Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min) - name: Python Lint - Format @@ -87,7 +87,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Pytest - Vortex @@ -125,7 +125,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Docs run: | @@ -168,7 +168,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install wasm32 target if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }} @@ -193,7 +193,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - run: cargo minimal-versions check --direct --workspace --ignore-private @@ -209,7 +209,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install nightly for fmt run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rustfmt @@ -265,7 +265,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: C/C++ Lint - clang-format run: | git ls-files vortex-cuda vortex-cxx vortex-duckdb vortex-ffi \ @@ -286,7 +286,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Rust Lint - Clippy No Default Features shell: bash @@ -305,7 +305,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install nightly for public-api run: rustup toolchain install $NIGHTLY_TOOLCHAIN @@ -348,7 +348,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup (Windows) if: matrix.os == 'windows-x64' run: | @@ -393,7 +393,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - run: ./gradlew test --parallel working-directory: ./java @@ -410,8 +410,8 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - - uses: actions/checkout@v6 - - uses: EmbarkStudios/cargo-deny-action@v2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: EmbarkStudios/cargo-deny-action@91bf2b620e09e18d6eb78b92e7861937469acedb # v2 with: command: check ${{ matrix.checks }} @@ -427,7 +427,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Build and run C++ unit tests run: | @@ -456,7 +456,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Run sqllogictest tests run: | @@ -471,7 +471,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -498,7 +498,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install nightly for cbindgen macro expansion run: rustup toolchain install $NIGHTLY_TOOLCHAIN @@ -537,7 +537,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: "regenerate FFI header file" run: | diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index e274a387756..7802904c99f 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -233,7 +233,7 @@ jobs: actions: read steps: - name: Checkout same-repo PR contents - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: ${{ needs.gate.outputs.checkout_ref }} fetch-depth: 1 diff --git a/.github/workflows/claude-write.yml b/.github/workflows/claude-write.yml index 86e09fd4cff..252bb8f9382 100644 --- a/.github/workflows/claude-write.yml +++ b/.github/workflows/claude-write.yml @@ -168,7 +168,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: ${{ needs.gate.outputs.checkout_ref }} fetch-depth: 0 diff --git a/.github/workflows/close-fixed-fuzzer-issues.yml b/.github/workflows/close-fixed-fuzzer-issues.yml index e660e8a9860..bd365070e3c 100644 --- a/.github/workflows/close-fixed-fuzzer-issues.yml +++ b/.github/workflows/close-fixed-fuzzer-issues.yml @@ -36,7 +36,7 @@ jobs: with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 48ad834f3d1..229f5e1ebf3 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -46,7 +46,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - uses: ./.github/actions/setup-prebuild diff --git a/.github/workflows/compat-gen-upload.yml b/.github/workflows/compat-gen-upload.yml index 6ec81264ba9..2e67341bcee 100644 --- a/.github/workflows/compat-gen-upload.yml +++ b/.github/workflows/compat-gen-upload.yml @@ -38,13 +38,13 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - uses: ./.github/actions/setup-prebuild - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: "arn:aws:iam::245040174862:role/GitHubCompatUploadRole" aws-region: us-east-1 @@ -88,13 +88,13 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - uses: ./.github/actions/setup-prebuild - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: "arn:aws:iam::245040174862:role/GitHubCompatUploadRole" aws-region: us-east-1 diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index da6be24f97f..7be13aec5b3 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -33,7 +33,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Run compat tests run: | diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index c1b33b37746..24eb9add90d 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -30,7 +30,7 @@ jobs: - uses: runs-on/action@v2 with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -64,12 +64,12 @@ jobs: nvidia-smi nvidia-smi -L nvidia-smi -q -d Memory - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install nextest - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@cf525cb33f51aca27cd6fa02034117ab963ff9f1 # v2 with: tool: nextest - name: Rust Tests @@ -116,7 +116,7 @@ jobs: nvidia-smi nvidia-smi -L nvidia-smi -q -d Memory - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -141,7 +141,7 @@ jobs: nvidia-smi nvidia-smi -L nvidia-smi -q -d Memory - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5a41908878a..8cee65e3a9e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,13 +19,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} enable-sccache: "false" - name: Set up JDK 17 - uses: actions/setup-java@v5 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: java-version: "17" distribution: "temurin" @@ -39,7 +39,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install Doxygen - uses: ssciwr/doxygen-install@v2 + uses: ssciwr/doxygen-install@329d88f5a303066a5bd006db7516b1925b86350e # v2 - name: Generate Javadoc for Java projects run: | cd java @@ -55,7 +55,7 @@ jobs: uv run --all-packages make -C docs html - name: Upload static files as artifact id: deployment - uses: actions/upload-pages-artifact@v5 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 with: path: docs/_build/html @@ -71,14 +71,14 @@ jobs: steps: # Note, since we provide the job with a CloudFlare scoped API token, we run it in a separate job that doesn't # execute any repository code. - - uses: actions/download-artifact@v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: github-pages - name: Extract Pages Artifact run: mkdir docs && tar -xvf artifact.tar -C docs shell: bash - name: Upload artifacts to CloudFlare Pages - uses: cloudflare/wrangler-action@v3 + uses: cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd # v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index de1e40891e6..0cc2e211e18 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -26,7 +26,7 @@ jobs: with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: @@ -114,7 +114,7 @@ jobs: --ignore-filename-regex='(\.cargo|rustc|registry)' - name: Upload coverage report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ matrix.fuzz_target }}-coverage path: coverage_html/ diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index 9eca51e865e..b45e0b866a4 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Fetch issue details id: fetch_issue @@ -236,7 +236,7 @@ jobs: CRASH_FILE: ${{ steps.extract.outputs.crash_file }} CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }} ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }} - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@567fe954a4527e81f132d87d1bdbcc94f7737434 # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 5c1489addb2..4934dbad72e 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Get PR Labels from API id: get_labels_api - uses: octokit/request-action@v3.0.0 # Use an action to make API requests + uses: octokit/request-action@b91aabaa861c777dcdb14e2387e30eddf04619ae # v3.0.0 # Use an action to make API requests with: route: GET /repos/{owner}/{repo}/issues/{pull_number}/labels owner: ${{ github.repository_owner }} diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index 5e0fb20900a..e3b2a9c11e6 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -51,7 +51,7 @@ jobs: with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 36e251d1fb6..e720a5d43c9 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -31,7 +31,7 @@ jobs: - { os: ubuntu, runs-on: "ubuntu-latest", target: aarch64-unknown-linux-gnu } - { os: ubuntu, runs-on: "ubuntu-latest", target: x86_64-unknown-linux-gnu } steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 @@ -51,14 +51,14 @@ jobs: cargo set-version --workspace ${{ inputs.version }} - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 # Latest macOS doesn't allow maturin to install stuff into the global Python interpreter if: "${{ matrix.target.runs-on == 'macos-latest' }}" with: python-version: "3.11" - name: Build wheel - uses: PyO3/maturin-action@v1 + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1 with: command: build maturin-version: 1.10.2 @@ -76,7 +76,7 @@ jobs: PYO3_ENVIRONMENT_SIGNATURE: "cpython3" - name: Upload wheels - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: "wheels-${{ matrix.target.target }}.zip" path: dist/ @@ -86,10 +86,10 @@ jobs: runs-on: "macos-latest" timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - - uses: actions/setup-java@v5 + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: "17" @@ -98,7 +98,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} enable-sccache: "false" - run: cargo build --release --package vortex-jni - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: "libvortex_jni_aarch64-apple-darwin.zip" path: "target/release/libvortex_jni.dylib" @@ -121,21 +121,21 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - - uses: actions/setup-java@v5 + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: "17" - uses: ./.github/actions/setup-prebuild - - uses: mlugg/setup-zig@v2.2.1 + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: tool: cargo-zigbuild - run: cargo zigbuild --release --target ${{ matrix.target }}.2.31 --package vortex-jni - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: "libvortex_jni_${{ matrix.target }}.zip" path: "target/${{ matrix.target }}/release/libvortex_jni.so" diff --git a/.github/workflows/publish-benchmarks-website.yml b/.github/workflows/publish-benchmarks-website.yml index e7eeefb8ecc..a85ba0124e1 100644 --- a/.github/workflows/publish-benchmarks-website.yml +++ b/.github/workflows/publish-benchmarks-website.yml @@ -14,23 +14,23 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build and push - uses: docker/build-push-action@v7 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7 with: context: ./benchmarks-website platforms: linux/arm64 diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index c7946ca3fc6..4b834935777 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -27,16 +27,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Rust Dependency Cache - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: save-if: ${{ github.ref_name == 'develop' }} - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} enable-sccache: "false" - - uses: mlugg/setup-zig@v2.2.1 + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install uv uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 with: @@ -79,15 +79,15 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - - uses: actions/setup-java@v5 + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: "17" - uses: ./.github/actions/setup-prebuild - - uses: mlugg/setup-zig@v2.2.1 + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 - name: Install cargo-zigbuild uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 with: @@ -112,7 +112,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install cargo-edit uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 494e3d509e6..1a015fb451f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,7 +23,7 @@ jobs: timeout-minutes: 30 needs: [package] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - uses: ./.github/actions/setup-rust @@ -57,7 +57,7 @@ jobs: name: push-to-pypi url: https://pypi.org/p/vortex-data steps: - - uses: actions/download-artifact@v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: wheels-*.zip # https://github.com/actions/download-artifact?tab=readme-ov-file#download-all-artifacts @@ -67,7 +67,7 @@ jobs: - name: Display structure of downloaded files run: ls -R dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: attestations: true verbose: true @@ -84,13 +84,13 @@ jobs: run: working-directory: ./java steps: - - uses: actions/checkout@v6 - - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: "17" - - uses: gradle/actions/setup-gradle@v6 - - uses: actions/download-artifact@v8 + - uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: libvortex_jni_*.zip - name: Copy native JNI libs diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 8c750fe7085..1e5cd70e69c 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-24.04 archive: tgz steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index e9e0af00017..a4d2bc4e677 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: release-drafter/release-drafter@v7.2.0 + - uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0 with: commitish: ${{ github.ref_name }} prerelease: ${{ github.event_name == 'workflow_dispatch' }} diff --git a/.github/workflows/report-fuzz-crash.yml b/.github/workflows/report-fuzz-crash.yml index dd4026641f6..83d75fcb19e 100644 --- a/.github/workflows/report-fuzz-crash.yml +++ b/.github/workflows/report-fuzz-crash.yml @@ -49,16 +49,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Download fuzzer logs - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ inputs.logs_artifact_name }} path: ./logs - name: Download crash artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ inputs.artifact_name }} path: ./crash_artifacts @@ -118,7 +118,7 @@ jobs: steps.dedup.outputs.duplicate != 'true' || steps.dedup.outputs.confidence != 'exact' continue-on-error: true - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@567fe954a4527e81f132d87d1bdbcc94f7737434 # v1 with: claude_code_oauth_token: ${{ secrets.claude_code_oauth_token }} github_token: ${{ secrets.gh_token }} diff --git a/.github/workflows/reuse.yml b/.github/workflows/reuse.yml index 6003dc47354..6d871fd20d4 100644 --- a/.github/workflows/reuse.yml +++ b/.github/workflows/reuse.yml @@ -17,6 +17,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: REUSE Compliance Check - uses: fsfe/reuse-action@v6 + uses: fsfe/reuse-action@676e2d560c9a403aa252096d99fcab3e1132b0f5 # v6 diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index bd6dcbd6976..55dc3268d3b 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -76,7 +76,7 @@ jobs: with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: @@ -173,7 +173,7 @@ jobs: - name: Archive crash artifacts id: upload_artifacts if: steps.check.outputs.crashes_found == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.FUZZ_NAME }}-crash-artifacts path: fuzz/artifacts @@ -181,7 +181,7 @@ jobs: - name: Archive fuzzer output log if: steps.check.outputs.crashes_found == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.FUZZ_NAME }}-logs path: fuzz_output.log diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index a2c0d0f842d..e9178f04635 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -45,7 +45,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Rust Tests if: ${{ matrix.suite == 'tests' }} @@ -63,7 +63,7 @@ jobs: --ignore benchmarks/* --ignore 'vortex-test/*' \ -o ${{ env.GRCOV_OUTPUT_FILE }} - name: Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 with: name: run-${{ matrix.suite }} files: ${{ env.GRCOV_OUTPUT_FILE }} @@ -105,7 +105,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install Rust nightly toolchain run: | @@ -169,7 +169,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install rustfilt run: | @@ -226,7 +226,7 @@ jobs: if: github.repository == 'vortex-data/vortex' with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-prebuild - name: Install nightly with miri run: rustup toolchain install $NIGHTLY_TOOLCHAIN --component rust-src,rustfmt,clippy,miri diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 81871981788..8be259fa562 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -277,12 +277,12 @@ jobs: if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false with: sccache: s3 - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 if: inputs.mode == 'pr' with: ref: ${{ github.event.pull_request.head.sha }} - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 if: inputs.mode != 'pr' - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh @@ -339,7 +339,7 @@ jobs: - name: Setup AWS CLI if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 with: role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole aws-region: us-east-1 @@ -355,7 +355,7 @@ jobs: - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false - uses: polarsignals/gh-actions-ps-profiling@v0.8.1 + uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" @@ -425,7 +425,7 @@ jobs: - name: Comment PR if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: file-path: comment.md # There is exactly one comment per comment-tag. If a comment with this tag already exists, @@ -478,14 +478,14 @@ jobs: - name: Comment PR with file sizes if: inputs.mode == 'pr' && matrix.remote_storage == null && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: file-path: sizes-comment.md comment-tag: file-sizes-${{ matrix.id }} - name: Comment PR on failure if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 04d44fec222..6f47fb3b8e1 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -11,7 +11,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10 with: # After 30 days of no activity, a PR will be marked as stale days-before-pr-stale: 14 diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index eeabe3c7388..f73ca0e0a0f 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -19,6 +19,6 @@ jobs: timeout-minutes: 10 steps: - name: Checkout Actions Repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Spell Check Repo - uses: crate-ci/typos@v1.45.1 + uses: crate-ci/typos@cf5f1c29a8ac336af8568821ec41919923b05a83 # v1.45.1 diff --git a/.github/workflows/wasm-fuzz.yml b/.github/workflows/wasm-fuzz.yml index 9b6e205a2cc..f076890c5b1 100644 --- a/.github/workflows/wasm-fuzz.yml +++ b/.github/workflows/wasm-fuzz.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 240 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: @@ -85,7 +85,7 @@ jobs: - name: Upload corpus if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: corpus-wasm path: corpus-wasm/ diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 77abe7b9013..67d54c48508 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -28,7 +28,7 @@ jobs: outputs: web: ${{ steps.filter.outputs.web }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4 id: filter with: @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -52,7 +52,7 @@ jobs: - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: . - - uses: actions/setup-node@v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" cache: "npm" @@ -73,7 +73,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -81,7 +81,7 @@ jobs: - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh working-directory: . - - uses: actions/setup-node@v6 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" cache: "npm" @@ -91,7 +91,7 @@ jobs: RUSTFLAGS: --cfg getrandom_backend="unsupported" run: npm run build - name: Upload build artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: vortex-explorer path: vortex-web/dist/ @@ -107,13 +107,13 @@ jobs: name: github-pages url: ${{ steps.deploy.outputs.deployment-url }} steps: - - uses: actions/download-artifact@v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: vortex-explorer path: dist - name: Deploy to Cloudflare Pages id: deploy - uses: cloudflare/wrangler-action@v3 + uses: cloudflare/wrangler-action@9acf94ace14e7dc412b076f2c5c20b8ce93c79cd # v3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} From 85e95fc78033edc6083e31e4fb6dc3eda0ab4842 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:59:45 -0400 Subject: [PATCH 206/250] Rename `ScalarValue::List` to `ScalarValue::Tuple` (#7672) ## Summary This is a better name for what it actually is, where every value can have a different type. And given that we want `ScalarValue::Array` for list scalars soon this makes more sense. ## API Changes Renames the variant. ## Testing N/A since this is a rename. Signed-off-by: Connor Tsui --- encodings/parquet-variant/src/operations.rs | 4 +- vortex-array/public-api.lock | 4 +- vortex-array/src/scalar/arbitrary.rs | 6 +-- vortex-array/src/scalar/constructor.rs | 2 +- .../src/scalar/convert/into_scalar.rs | 2 +- vortex-array/src/scalar/downcast.rs | 43 ++++++++++--------- vortex-array/src/scalar/proto.rs | 8 ++-- vortex-array/src/scalar/scalar_value.rs | 22 +++++----- vortex-array/src/scalar/tests/casting.rs | 6 +-- vortex-array/src/scalar/tests/nested.rs | 4 +- vortex-array/src/scalar/tests/nullability.rs | 6 +-- vortex-array/src/scalar/typed_view/list.rs | 2 +- vortex-array/src/scalar/typed_view/struct_.rs | 8 ++-- vortex-array/src/scalar/validate.rs | 12 +++--- 14 files changed, 66 insertions(+), 63 deletions(-) diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 8a323cce76f..e812e0fcafa 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -214,7 +214,7 @@ fn scalar_from_shredded_object_scalar( let fields = StructFields::new(FieldNames::from(names), dtypes); Scalar::try_new( DType::Struct(fields, Nullability::NonNullable), - Some(ScalarValue::List(field_values)), + Some(ScalarValue::Tuple(field_values)), ) } @@ -330,7 +330,7 @@ fn parquet_variant_to_scalar(variant: PqVariant<'_, '_>) -> VortexResult let fields = StructFields::new(FieldNames::from(names), dtypes); Scalar::try_new( DType::Struct(fields, nn), - Some(ScalarValue::List(field_values)), + Some(ScalarValue::Tuple(field_values)), )? } }) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 2f36c30a472..b3df6c14b09 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -13960,10 +13960,10 @@ pub vortex_array::scalar::ScalarValue::Bool(bool) pub vortex_array::scalar::ScalarValue::Decimal(vortex_array::scalar::DecimalValue) -pub vortex_array::scalar::ScalarValue::List(alloc::vec::Vec>) - pub vortex_array::scalar::ScalarValue::Primitive(vortex_array::scalar::PValue) +pub vortex_array::scalar::ScalarValue::Tuple(alloc::vec::Vec>) + pub vortex_array::scalar::ScalarValue::Utf8(vortex_buffer::string::BufferString) pub vortex_array::scalar::ScalarValue::Variant(alloc::boxed::Box) diff --git a/vortex-array/src/scalar/arbitrary.rs b/vortex-array/src/scalar/arbitrary.rs index 6abf02ffbc3..4b74e726749 100644 --- a/vortex-array/src/scalar/arbitrary.rs +++ b/vortex-array/src/scalar/arbitrary.rs @@ -66,7 +66,7 @@ pub fn random_scalar(u: &mut Unstructured, dtype: &DType) -> Result { .vortex_expect("unable to construct random `Scalar`_"), DType::Struct(sdt, _) => Scalar::try_new( dtype.clone(), - Some(ScalarValue::List( + Some(ScalarValue::Tuple( sdt.fields() .map(|d| random_scalar(u, &d).map(|s| s.into_value())) .collect::>>()?, @@ -75,7 +75,7 @@ pub fn random_scalar(u: &mut Unstructured, dtype: &DType) -> Result { .vortex_expect("unable to construct random `Scalar`_"), DType::List(edt, _) => Scalar::try_new( dtype.clone(), - Some(ScalarValue::List( + Some(ScalarValue::Tuple( iter::from_fn(|| { // Generate elements with 1/4 probability. u.arbitrary() @@ -88,7 +88,7 @@ pub fn random_scalar(u: &mut Unstructured, dtype: &DType) -> Result { .vortex_expect("unable to construct random `Scalar`_"), DType::FixedSizeList(edt, size, _) => Scalar::try_new( dtype.clone(), - Some(ScalarValue::List( + Some(ScalarValue::Tuple( (0..*size) .map(|_| random_scalar(u, edt).map(|s| s.into_value())) .collect::>>()?, diff --git a/vortex-array/src/scalar/constructor.rs b/vortex-array/src/scalar/constructor.rs index faaddc1dcef..2baf53c9528 100644 --- a/vortex-array/src/scalar/constructor.rs +++ b/vortex-array/src/scalar/constructor.rs @@ -166,7 +166,7 @@ impl Scalar { ListKind::FixedSize => DType::FixedSizeList(element_dtype, size, nullability), }; - Self::try_new(dtype, Some(ScalarValue::List(children))) + Self::try_new(dtype, Some(ScalarValue::Tuple(children))) .vortex_expect("unable to construct a list `Scalar`") } diff --git a/vortex-array/src/scalar/convert/into_scalar.rs b/vortex-array/src/scalar/convert/into_scalar.rs index ac804c06dde..f37173d5be7 100644 --- a/vortex-array/src/scalar/convert/into_scalar.rs +++ b/vortex-array/src/scalar/convert/into_scalar.rs @@ -82,7 +82,7 @@ where Scalar: From, { fn from(vec: Vec) -> Self { - ScalarValue::List( + ScalarValue::Tuple( vec.into_iter() .map(|elem| Scalar::from(elem).into_value()) .collect(), diff --git a/vortex-array/src/scalar/downcast.rs b/vortex-array/src/scalar/downcast.rs index ce9996514e3..1276141fc35 100644 --- a/vortex-array/src/scalar/downcast.rs +++ b/vortex-array/src/scalar/downcast.rs @@ -115,12 +115,13 @@ impl Scalar { /// Returns a view of the scalar as a list scalar. /// - /// Note that we use [`ListScalar`] to represent **both** [`List`](crate::dtype::DType::List) and - /// [`FixedSizeList`](crate::dtype::DType::FixedSizeList). + /// Note that we use [`ListScalar`] to represent **both** [`List`](crate::dtype::DType::List) + /// and [`FixedSizeList`](crate::dtype::DType::FixedSizeList). /// /// # Panics /// - /// Panics if the scalar does not have a [`List`](crate::dtype::DType::List) or [`FixedSizeList`](crate::dtype::DType::FixedSizeList) type. + /// Panics if the scalar does not have a [`List`](crate::dtype::DType::List) or + /// [`FixedSizeList`](crate::dtype::DType::FixedSizeList) type. pub fn as_list(&self) -> ListScalar<'_> { self.as_list_opt() .vortex_expect("Failed to convert scalar to list") @@ -128,8 +129,8 @@ impl Scalar { /// Returns a view of the scalar as a list scalar if it has a list type. /// - /// Note that we use [`ListScalar`] to represent **both** [`List`](crate::dtype::DType::List) and - /// [`FixedSizeList`](crate::dtype::DType::FixedSizeList). + /// Note that we use [`ListScalar`] to represent **both** [`List`](crate::dtype::DType::List) + /// and [`FixedSizeList`](crate::dtype::DType::FixedSizeList). pub fn as_list_opt(&self) -> Option> { ListScalar::try_new(self.dtype(), self.value()).ok() } @@ -172,7 +173,7 @@ impl Scalar { } impl ScalarValue { - /// Returns the boolean value, panicking if the value is not a [`Bool`][ScalarValue::Bool]. + /// Returns the boolean value, panicking if the value is not a [`Bool`](ScalarValue::Bool). pub fn as_bool(&self) -> bool { match self { ScalarValue::Bool(b) => *b, @@ -181,7 +182,7 @@ impl ScalarValue { } /// Returns the primitive value, panicking if the value is not a - /// [`Primitive`][ScalarValue::Primitive]. + /// [`Primitive`](ScalarValue::Primitive). pub fn as_primitive(&self) -> &PValue { match self { ScalarValue::Primitive(p) => p, @@ -190,7 +191,7 @@ impl ScalarValue { } /// Returns the decimal value, panicking if the value is not a - /// [`Decimal`][ScalarValue::Decimal]. + /// [`Decimal`](ScalarValue::Decimal). pub fn as_decimal(&self) -> &DecimalValue { match self { ScalarValue::Decimal(d) => d, @@ -198,7 +199,7 @@ impl ScalarValue { } } - /// Returns the UTF-8 string value, panicking if the value is not a [`Utf8`][ScalarValue::Utf8]. + /// Returns the UTF-8 string value, panicking if the value is not a [`Utf8`](ScalarValue::Utf8). pub fn as_utf8(&self) -> &BufferString { match self { ScalarValue::Utf8(s) => s, @@ -206,7 +207,7 @@ impl ScalarValue { } } - /// Returns the binary value, panicking if the value is not a [`Binary`][ScalarValue::Binary]. + /// Returns the binary value, panicking if the value is not a [`Binary`](ScalarValue::Binary). pub fn as_binary(&self) -> &ByteBuffer { match self { ScalarValue::Binary(b) => b, @@ -214,15 +215,15 @@ impl ScalarValue { } } - /// Returns the list elements, panicking if the value is not a [`List`][ScalarValue::List]. + /// Returns the tuple elements, panicking if the value is not a [`Tuple`](ScalarValue::Tuple). pub fn as_list(&self) -> &[Option] { match self { - ScalarValue::List(elements) => elements, - _ => vortex_panic!("ScalarValue is not a List"), + ScalarValue::Tuple(elements) => elements, + _ => vortex_panic!("ScalarValue is not a Tuple"), } } - /// Returns the boolean value, panicking if the value is not a [`Bool`][ScalarValue::Bool]. + /// Returns the boolean value, panicking if the value is not a [`Bool`](ScalarValue::Bool). pub fn into_bool(self) -> bool { match self { ScalarValue::Bool(b) => b, @@ -231,7 +232,7 @@ impl ScalarValue { } /// Returns the primitive value, panicking if the value is not a - /// [`Primitive`][ScalarValue::Primitive]. + /// [`Primitive`](ScalarValue::Primitive). pub fn into_primitive(self) -> PValue { match self { ScalarValue::Primitive(p) => p, @@ -240,7 +241,7 @@ impl ScalarValue { } /// Returns the decimal value, panicking if the value is not a - /// [`Decimal`][ScalarValue::Decimal]. + /// [`Decimal`](ScalarValue::Decimal). pub fn into_decimal(self) -> DecimalValue { match self { ScalarValue::Decimal(d) => d, @@ -248,7 +249,7 @@ impl ScalarValue { } } - /// Returns the UTF-8 string value, panicking if the value is not a [`Utf8`][ScalarValue::Utf8]. + /// Returns the UTF-8 string value, panicking if the value is not a [`Utf8`](ScalarValue::Utf8). pub fn into_utf8(self) -> BufferString { match self { ScalarValue::Utf8(s) => s, @@ -256,7 +257,7 @@ impl ScalarValue { } } - /// Returns the binary value, panicking if the value is not a [`Binary`][ScalarValue::Binary]. + /// Returns the binary value, panicking if the value is not a [`Binary`](ScalarValue::Binary). pub fn into_binary(self) -> ByteBuffer { match self { ScalarValue::Binary(b) => b, @@ -264,11 +265,11 @@ impl ScalarValue { } } - /// Returns the list elements, panicking if the value is not a [`List`][ScalarValue::List]. + /// Returns the tuple elements, panicking if the value is not a [`Tuple`](ScalarValue::Tuple). pub fn into_list(self) -> Vec> { match self { - ScalarValue::List(elements) => elements, - _ => vortex_panic!("ScalarValue is not a List"), + ScalarValue::Tuple(elements) => elements, + _ => vortex_panic!("ScalarValue is not a Tuple"), } } diff --git a/vortex-array/src/scalar/proto.rs b/vortex-array/src/scalar/proto.rs index 2c45a450bfa..938eeb2f778 100644 --- a/vortex-array/src/scalar/proto.rs +++ b/vortex-array/src/scalar/proto.rs @@ -101,7 +101,7 @@ impl From<&ScalarValue> for pb::ScalarValue { ScalarValue::Binary(v) => pb::ScalarValue { kind: Some(Kind::BytesValue(v.to_vec())), }, - ScalarValue::List(v) => { + ScalarValue::Tuple(v) => { let mut values = Vec::with_capacity(v.len()); for elem in v.iter() { values.push(ScalarValue::to_proto(elem.as_ref())); @@ -435,7 +435,7 @@ fn bytes_from_proto(bytes: &[u8], dtype: &DType) -> VortexResult { } } -/// Deserialize a [`ScalarValue::List`] from a protobuf `ListValue`. +/// Deserialize a [`ScalarValue::Tuple`] from a protobuf `ListValue`. fn list_from_proto( v: &ListValue, dtype: &DType, @@ -454,7 +454,7 @@ fn list_from_proto( )?); } - Ok(ScalarValue::List(values)) + Ok(ScalarValue::Tuple(values)) } #[cfg(test)] @@ -531,7 +531,7 @@ mod tests { Arc::new(DType::Primitive(PType::I32, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![ + Some(ScalarValue::Tuple(vec![ Some(ScalarValue::Primitive(42i32.into())), Some(ScalarValue::Primitive(43i32.into())), ])), diff --git a/vortex-array/src/scalar/scalar_value.rs b/vortex-array/src/scalar/scalar_value.rs index da32bafa947..2f104ef1688 100644 --- a/vortex-array/src/scalar/scalar_value.rs +++ b/vortex-array/src/scalar/scalar_value.rs @@ -33,8 +33,10 @@ pub enum ScalarValue { Utf8(BufferString), /// A binary (byte array) value. Binary(ByteBuffer), - /// A list of potentially null scalar values. - List(Vec>), + /// A tuple of potentially null scalar values. + /// + /// Used as the underlying representation for list, fixed-size list, and struct scalars. + Tuple(Vec>), /// A row-specific scalar wrapped by `DType::Variant`. Variant(Box), } @@ -49,17 +51,17 @@ impl ScalarValue { DType::Decimal(dt, ..) => Self::Decimal(DecimalValue::zero(dt)), DType::Utf8(_) => Self::Utf8(BufferString::empty()), DType::Binary(_) => Self::Binary(ByteBuffer::empty()), - DType::List(..) => Self::List(vec![]), + DType::List(..) => Self::Tuple(vec![]), DType::FixedSizeList(edt, size, _) => { let elements = (0..*size).map(|_| Some(Self::zero_value(edt))).collect(); - Self::List(elements) + Self::Tuple(elements) } DType::Struct(fields, _) => { let field_values = fields .fields() .map(|f| Some(Self::zero_value(&f))) .collect(); - Self::List(field_values) + Self::Tuple(field_values) } DType::Extension(ext_dtype) => { // Since we have no way to define a "zero" extension value (since we have no idea @@ -89,14 +91,14 @@ impl ScalarValue { DType::Decimal(dt, ..) => Self::Decimal(DecimalValue::zero(dt)), DType::Utf8(_) => Self::Utf8(BufferString::empty()), DType::Binary(_) => Self::Binary(ByteBuffer::empty()), - DType::List(..) => Self::List(vec![]), + DType::List(..) => Self::Tuple(vec![]), DType::FixedSizeList(edt, size, _) => { let elements = (0..*size).map(|_| Self::default_value(edt)).collect(); - Self::List(elements) + Self::Tuple(elements) } DType::Struct(fields, _) => { let field_values = fields.fields().map(|f| Self::default_value(&f)).collect(); - Self::List(field_values) + Self::Tuple(field_values) } DType::Extension(ext_dtype) => { // Since we have no way to define a "default" extension value (since we have no idea @@ -117,7 +119,7 @@ impl PartialOrd for ScalarValue { (ScalarValue::Decimal(a), ScalarValue::Decimal(b)) => a.partial_cmp(b), (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => a.partial_cmp(b), (ScalarValue::Binary(a), ScalarValue::Binary(b)) => a.partial_cmp(b), - (ScalarValue::List(a), ScalarValue::List(b)) => a.partial_cmp(b), + (ScalarValue::Tuple(a), ScalarValue::Tuple(b)) => a.partial_cmp(b), (ScalarValue::Variant(a), ScalarValue::Variant(b)) => a.partial_cmp(b), // (ScalarValue::Extension(a), ScalarValue::Extension(b)) => a.partial_cmp(b), _ => None, @@ -156,7 +158,7 @@ impl Display for ScalarValue { write!(f, "{}", to_hex(b)) } } - ScalarValue::List(elements) => { + ScalarValue::Tuple(elements) => { write!(f, "[")?; for (i, element) in elements.iter().enumerate() { if i > 0 { diff --git a/vortex-array/src/scalar/tests/casting.rs b/vortex-array/src/scalar/tests/casting.rs index b73d4c83818..ae63232330b 100644 --- a/vortex-array/src/scalar/tests/casting.rs +++ b/vortex-array/src/scalar/tests/casting.rs @@ -152,7 +152,7 @@ mod tests { Some(ScalarValue::Primitive(PValue::F32(f32_value))), ]; - let scalar = Scalar::new(struct_dtype, Some(ScalarValue::List(field_values))); + let scalar = Scalar::new(struct_dtype, Some(ScalarValue::Tuple(field_values))); let struct_scalar = scalar.as_struct(); let fields: Vec<_> = (0..3) @@ -209,7 +209,7 @@ mod tests { ))), ]; - let scalar = Scalar::new(list_dtype, Some(ScalarValue::List(elements))); + let scalar = Scalar::new(list_dtype, Some(ScalarValue::Tuple(elements))); let list_scalar = scalar.as_list(); let elements = list_scalar.elements().unwrap(); @@ -353,7 +353,7 @@ mod tests { let scalar = Scalar::new( DType::Extension(ext_dtype.erased()), - Some(ScalarValue::List(field_values)), + Some(ScalarValue::Tuple(field_values)), ); // Verify the struct field was coerced diff --git a/vortex-array/src/scalar/tests/nested.rs b/vortex-array/src/scalar/tests/nested.rs index 0aadd4ce2c9..d02bf43c631 100644 --- a/vortex-array/src/scalar/tests/nested.rs +++ b/vortex-array/src/scalar/tests/nested.rs @@ -311,7 +311,7 @@ mod tests { Arc::from(DType::Primitive(PType::U16, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![ + Some(ScalarValue::Tuple(vec![ Some(ScalarValue::Primitive(PValue::U16(6))), Some(ScalarValue::Primitive(PValue::U16(100))), ])), @@ -350,7 +350,7 @@ mod tests { Arc::from(DType::Primitive(PType::U16, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![ + Some(ScalarValue::Tuple(vec![ Some(ScalarValue::Primitive(PValue::U16(100))), Some(ScalarValue::Primitive(PValue::U16(256))), // Too large for U8 Some(ScalarValue::Primitive(PValue::U16(1000))), // Too large for U8 diff --git a/vortex-array/src/scalar/tests/nullability.rs b/vortex-array/src/scalar/tests/nullability.rs index 4d2bff418a5..d862d08d837 100644 --- a/vortex-array/src/scalar/tests/nullability.rs +++ b/vortex-array/src/scalar/tests/nullability.rs @@ -54,7 +54,7 @@ mod tests { Arc::from(DType::Primitive(PType::U16, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![Some(ScalarValue::Primitive( + Some(ScalarValue::Tuple(vec![Some(ScalarValue::Primitive( PValue::U16(6), ))])), ); @@ -97,7 +97,7 @@ mod tests { Arc::from(DType::Primitive(PType::U16, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![ + Some(ScalarValue::Tuple(vec![ Some(ScalarValue::Primitive(PValue::U16(6))), None, Some(ScalarValue::Primitive(PValue::U16(10))), @@ -200,7 +200,7 @@ mod tests { Arc::from(DType::Primitive(PType::U16, Nullability::Nullable)), Nullability::Nullable, ), - Some(ScalarValue::List(vec![ + Some(ScalarValue::Tuple(vec![ Some(ScalarValue::Primitive(PValue::U16(6))), None, ])), diff --git a/vortex-array/src/scalar/typed_view/list.rs b/vortex-array/src/scalar/typed_view/list.rs index 3a158dd59ca..905db80dbd8 100644 --- a/vortex-array/src/scalar/typed_view/list.rs +++ b/vortex-array/src/scalar/typed_view/list.rs @@ -207,7 +207,7 @@ impl<'a> ListScalar<'a> { Scalar::try_new( dtype.clone(), - Some(ScalarValue::List( + Some(ScalarValue::Tuple( self.elements .ok_or_else(|| vortex_err!("nullness should be handled in Scalar::cast"))? .iter() diff --git a/vortex-array/src/scalar/typed_view/struct_.rs b/vortex-array/src/scalar/typed_view/struct_.rs index 93b796f055f..6e167f3b0f4 100644 --- a/vortex-array/src/scalar/typed_view/struct_.rs +++ b/vortex-array/src/scalar/typed_view/struct_.rs @@ -236,7 +236,7 @@ impl<'a> StructScalar<'a> { .map(|s| s.into_value()) }) .collect::>>()?; - Scalar::try_new(dtype.clone(), Some(ScalarValue::List(fields))) + Scalar::try_new(dtype.clone(), Some(ScalarValue::Tuple(fields))) } else { Ok(Scalar::null(dtype.clone())) } @@ -261,7 +261,7 @@ impl<'a> StructScalar<'a> { return Ok(Scalar::null(projected_dtype)); }; - let new_fields = ScalarValue::List( + let new_fields = ScalarValue::Tuple( projection .iter() .map(|name| { @@ -306,7 +306,7 @@ impl Scalar { } let value_children: Vec<_> = children.into_iter().map(|x| x.into_value()).collect(); - Self::try_new(dtype, Some(ScalarValue::List(value_children))) + Self::try_new(dtype, Some(ScalarValue::Tuple(value_children))) .vortex_expect("unable to construct a struct `Scalar`") } @@ -323,7 +323,7 @@ impl Scalar { children: impl IntoIterator, ) -> Self { let value_children: Vec<_> = children.into_iter().map(|s| s.into_value()).collect(); - unsafe { Self::new_unchecked(dtype, Some(ScalarValue::List(value_children))) } + unsafe { Self::new_unchecked(dtype, Some(ScalarValue::Tuple(value_children))) } } } diff --git a/vortex-array/src/scalar/validate.rs b/vortex-array/src/scalar/validate.rs index 2696dc1f804..4df38187a9a 100644 --- a/vortex-array/src/scalar/validate.rs +++ b/vortex-array/src/scalar/validate.rs @@ -74,8 +74,8 @@ impl Scalar { ); } DType::List(elem_dtype, _) => { - let ScalarValue::List(elements) = value else { - vortex_bail!("list dtype expected List value, got {value}"); + let ScalarValue::Tuple(elements) = value else { + vortex_bail!("list dtype expected Tuple value, got {value}"); }; for (i, element) in elements.iter().enumerate() { @@ -84,8 +84,8 @@ impl Scalar { } } DType::FixedSizeList(elem_dtype, size, _) => { - let ScalarValue::List(elements) = value else { - vortex_bail!("fixed-size list dtype expected List value, got {value}",); + let ScalarValue::Tuple(elements) = value else { + vortex_bail!("fixed-size list dtype expected Tuple value, got {value}",); }; let len = elements.len(); @@ -102,8 +102,8 @@ impl Scalar { } } DType::Struct(fields, _) => { - let ScalarValue::List(values) = value else { - vortex_bail!("struct dtype expected List value, got {value}"); + let ScalarValue::Tuple(values) = value else { + vortex_bail!("struct dtype expected Tuple value, got {value}"); }; let nfields = fields.nfields(); From 27b1825d5b0252eaeb23fdde5fbf6432dbda076e Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:00:03 -0400 Subject: [PATCH 207/250] make all scalar view types `Copy` (#7673) ## Summary Closes: https://github.com/vortex-data/vortex/issues/4844 ## API Changes Adds `Copy` impl for all scalar view types. ## Testing N/A Signed-off-by: Connor Tsui --- encodings/sparse/src/canonical.rs | 2 +- vortex-array/public-api.lock | 12 ++++++++++++ vortex-array/src/scalar/typed_view/binary.rs | 2 +- vortex-array/src/scalar/typed_view/bool.rs | 2 +- vortex-array/src/scalar/typed_view/extension/mod.rs | 2 +- vortex-array/src/scalar/typed_view/list.rs | 2 +- vortex-array/src/scalar/typed_view/struct_.rs | 2 +- vortex-array/src/scalar/typed_view/utf8.rs | 2 +- 8 files changed, 19 insertions(+), 7 deletions(-) diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index 0dcd8d533b1..8a27e72aed4 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -214,7 +214,7 @@ fn execute_sparse_lists_inner( } else { // Set with the fill value. builder - .append_value(fill_value.clone()) + .append_value(fill_value) .vortex_expect("Failed to append fill value"); } } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index b3df6c14b09..dd7f4c0bea0 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -14248,6 +14248,8 @@ impl<'a> core::hash::Hash for vortex_array::scalar::BinaryScalar<'a> pub fn vortex_array::scalar::BinaryScalar<'a>::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl<'a> core::marker::Copy for vortex_array::scalar::BinaryScalar<'a> + pub struct vortex_array::scalar::BoolScalar<'a> impl<'a> vortex_array::scalar::BoolScalar<'a> @@ -14298,6 +14300,8 @@ impl<'a> core::hash::Hash for vortex_array::scalar::BoolScalar<'a> pub fn vortex_array::scalar::BoolScalar<'a>::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl<'a> core::marker::Copy for vortex_array::scalar::BoolScalar<'a> + pub struct vortex_array::scalar::DecimalScalar<'a> impl<'a> vortex_array::scalar::DecimalScalar<'a> @@ -14464,6 +14468,8 @@ impl<'a> core::fmt::Debug for vortex_array::scalar::ExtScalar<'a> pub fn vortex_array::scalar::ExtScalar<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for vortex_array::scalar::ExtScalar<'a> + pub struct vortex_array::scalar::ListScalar<'a> impl<'a> vortex_array::scalar::ListScalar<'a> @@ -14516,6 +14522,8 @@ impl<'a> core::fmt::Debug for vortex_array::scalar::ListScalar<'a> pub fn vortex_array::scalar::ListScalar<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for vortex_array::scalar::ListScalar<'a> + pub struct vortex_array::scalar::PrimitiveScalar<'a> impl<'a> vortex_array::scalar::PrimitiveScalar<'a> @@ -15310,6 +15318,8 @@ impl<'a> core::fmt::Debug for vortex_array::scalar::StructScalar<'a> pub fn vortex_array::scalar::StructScalar<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for vortex_array::scalar::StructScalar<'a> + pub struct vortex_array::scalar::Utf8Scalar<'a> impl<'a> vortex_array::scalar::Utf8Scalar<'a> @@ -15360,6 +15370,8 @@ impl<'a> core::hash::Hash for vortex_array::scalar::Utf8Scalar<'a> pub fn vortex_array::scalar::Utf8Scalar<'a>::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl<'a> core::marker::Copy for vortex_array::scalar::Utf8Scalar<'a> + pub struct vortex_array::scalar::VariantScalar<'a> impl<'a> vortex_array::scalar::VariantScalar<'a> diff --git a/vortex-array/src/scalar/typed_view/binary.rs b/vortex-array/src/scalar/typed_view/binary.rs index 9235d4e17ef..cb107e3613c 100644 --- a/vortex-array/src/scalar/typed_view/binary.rs +++ b/vortex-array/src/scalar/typed_view/binary.rs @@ -20,7 +20,7 @@ use crate::scalar::ScalarValue; /// /// This type provides a view into a binary scalar value, which can be either /// a valid byte buffer or null. -#[derive(Debug, Clone, Hash)] +#[derive(Debug, Clone, Copy, Hash)] pub struct BinaryScalar<'a> { /// The data type of this scalar. dtype: &'a DType, diff --git a/vortex-array/src/scalar/typed_view/bool.rs b/vortex-array/src/scalar/typed_view/bool.rs index 10f8cd47d9f..c971d1169c6 100644 --- a/vortex-array/src/scalar/typed_view/bool.rs +++ b/vortex-array/src/scalar/typed_view/bool.rs @@ -19,7 +19,7 @@ use crate::scalar::ScalarValue; /// /// This type provides a view into a boolean scalar value, which can be either /// true, false, or null. -#[derive(Debug, Clone, Hash, Eq)] +#[derive(Debug, Clone, Copy, Hash, Eq)] pub struct BoolScalar<'a> { /// The data type of this scalar. dtype: &'a DType, diff --git a/vortex-array/src/scalar/typed_view/extension/mod.rs b/vortex-array/src/scalar/typed_view/extension/mod.rs index dfbc25f936d..3508883fc92 100644 --- a/vortex-array/src/scalar/typed_view/extension/mod.rs +++ b/vortex-array/src/scalar/typed_view/extension/mod.rs @@ -19,7 +19,7 @@ use crate::scalar::ScalarValue; /// A scalar value representing an extension type. /// /// Extension types allow wrapping a storage type with custom semantics. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct ExtScalar<'a> { /// A reference to the `DType` of the extension type. This **must** be the [`DType::Extension`] /// variant. diff --git a/vortex-array/src/scalar/typed_view/list.rs b/vortex-array/src/scalar/typed_view/list.rs index 905db80dbd8..15de731089a 100644 --- a/vortex-array/src/scalar/typed_view/list.rs +++ b/vortex-array/src/scalar/typed_view/list.rs @@ -29,7 +29,7 @@ use crate::scalar::ScalarValue; /// number of `elements` is equal to the `size` field of the [`FixedSizeList`]. /// /// [`FixedSizeList`]: DType::FixedSizeList -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct ListScalar<'a> { /// The data type of this scalar. dtype: &'a DType, diff --git a/vortex-array/src/scalar/typed_view/struct_.rs b/vortex-array/src/scalar/typed_view/struct_.rs index 6e167f3b0f4..d8069bab77c 100644 --- a/vortex-array/src/scalar/typed_view/struct_.rs +++ b/vortex-array/src/scalar/typed_view/struct_.rs @@ -27,7 +27,7 @@ use crate::scalar::ScalarValue; /// /// This type provides a view into a struct scalar value, which can contain /// named fields with different types, or be null. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct StructScalar<'a> { /// The data type of this scalar. dtype: &'a DType, diff --git a/vortex-array/src/scalar/typed_view/utf8.rs b/vortex-array/src/scalar/typed_view/utf8.rs index acf0ab9f407..5a9ad84064b 100644 --- a/vortex-array/src/scalar/typed_view/utf8.rs +++ b/vortex-array/src/scalar/typed_view/utf8.rs @@ -22,7 +22,7 @@ use crate::scalar::ScalarValue; /// /// This type provides a view into a UTF-8 string scalar value, which can be either /// a valid UTF-8 string or null. -#[derive(Debug, Clone, Hash, Eq)] +#[derive(Debug, Clone, Copy, Hash, Eq)] pub struct Utf8Scalar<'a> { /// The data type of this scalar. dtype: &'a DType, From 40c2c7f93af0ab0d23cbfb27fdb7ead3439e249c Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 27 Apr 2026 12:02:09 -0400 Subject: [PATCH 208/250] Upgrade all vortex-web dependencies (#7669) Signed-off-by: Robert Kruszewski --- vortex-web/eslint.config.ts | 3 +- vortex-web/package-lock.json | 1639 ++++++++++------------------------ vortex-web/package.json | 34 +- 3 files changed, 474 insertions(+), 1202 deletions(-) diff --git a/vortex-web/eslint.config.ts b/vortex-web/eslint.config.ts index ecc2fe28df0..8c6a8e00920 100644 --- a/vortex-web/eslint.config.ts +++ b/vortex-web/eslint.config.ts @@ -21,7 +21,8 @@ export default tseslint.config({ ignores: ["dist", "storybook-static"] }, { "react-refresh": reactRefresh, }, rules: { - ...reactHooks.configs.recommended.rules, + "react-hooks/rules-of-hooks": "error", + "react-hooks/exhaustive-deps": "warn", "react-refresh/only-export-components": [ "warn", { allowConstantExport: true }, diff --git a/vortex-web/package-lock.json b/vortex-web/package-lock.json index 2dccff8bdd4..53ccf765e89 100644 --- a/vortex-web/package-lock.json +++ b/vortex-web/package-lock.json @@ -16,26 +16,26 @@ "react-dom": "^19.1.0" }, "devDependencies": { - "@eslint/js": "^9.25.0", - "@storybook/addon-docs": "^10.3.3", - "@storybook/react-vite": "^10.3.3", - "@tailwindcss/vite": "^4.1.4", + "@eslint/js": "^10.0.1", + "@storybook/addon-docs": "^10.3.5", + "@storybook/react-vite": "^10.3.5", + "@tailwindcss/vite": "^4.2.4", "@types/d3-hierarchy": "^3.1.7", - "@types/react": "^19.1.2", - "@types/react-dom": "^19.1.2", - "@vitejs/plugin-react": "^4.4.1", - "eslint": "^9.25.0", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "eslint": "^10.2.1", "eslint-config-prettier": "^10.1.8", - "eslint-plugin-react-hooks": "^5.2.0", - "eslint-plugin-react-refresh": "^0.5.0", - "eslint-plugin-storybook": "^10.3.3", - "globals": "^16.0.0", - "prettier": "^3.8.1", - "storybook": "^10.3.3", - "tailwindcss": "^4.1.4", + "eslint-plugin-react-hooks": "^7.1.1", + "eslint-plugin-react-refresh": "^0.5.2", + "eslint-plugin-storybook": "^10.3.5", + "globals": "^17.5.0", + "prettier": "^3.8.3", + "storybook": "^10.3.5", + "tailwindcss": "^4.2.4", "typescript": "~6.0.0", - "typescript-eslint": "^8.30.1", - "vite": "^6.3.2" + "typescript-eslint": "^8.59.0", + "vite": "^8.0.10" } }, "node_modules/@adobe/css-tools": { @@ -177,16 +177,6 @@ "@babel/core": "^7.0.0" } }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/helper-string-parser": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", @@ -247,38 +237,6 @@ "node": ">=6.0.0" } }, - "node_modules/@babel/plugin-transform-react-jsx-self": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.27.1.tgz", - "integrity": "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-transform-react-jsx-source": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.27.1.tgz", - "integrity": "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.27.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, "node_modules/@babel/runtime": { "version": "7.29.2", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", @@ -338,6 +296,40 @@ "node": ">=6.9.0" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", @@ -823,118 +815,89 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", - "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^2.1.7", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", - "minimatch": "^3.1.5" + "minimatch": "^10.2.4" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.5.5.tgz", + "integrity": "sha512-eIJYKTCECbP/nsKaaruF6LW967mtbQbsw4JTtSVkUQc9MneSkbrgPJAbKl9nWr0ZeowV8BfsarBmPpBzGelA2w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0" + "@eslint/core": "^1.2.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", - "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.14.0", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", - "minimatch": "^3.1.5", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/js": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", - "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", "dev": true, "license": "MIT", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } } }, "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.1.tgz", + "integrity": "sha512-rZAP3aVgB9ds9KOeUSL+zZ21hPmo8dh6fnIFwRQj5EAZl9gzR7wxYbYXYysAM8CTqGmUGyp2S4kUdV17MnGuWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0", + "@eslint/core": "^1.2.1", "levn": "^0.4.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@humanfs/core": { @@ -1091,54 +1054,39 @@ "react": ">=16" } }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.27", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", - "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@rollup/pluginutils": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", - "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@types/estree": "^1.0.0", - "estree-walker": "^2.0.2", - "picomatch": "^4.0.2" + "@tybys/wasm-util": "^0.10.1" }, - "engines": { - "node": ">=14.0.0" + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" }, "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" - }, - "peerDependenciesMeta": { - "rollup": { - "optional": true - } + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" } }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.2.tgz", - "integrity": "sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==", - "cpu": [ - "arm" - ], + "node_modules/@oxc-project/types": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", + "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", "dev": true, "license": "MIT", - "optional": true, - "os": [ - "android" - ] + "funding": { + "url": "https://github.com/sponsors/Boshen" + } }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.2.tgz", - "integrity": "sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-s70pVGhw4zqGeFnXWvAzJDlvxhlRollagdCCKRgOsgUOH3N1l0LIxf83AtGzmb5SiVM4Hjl5HyarMRfdfj3DaQ==", "cpu": [ "arm64" ], @@ -1147,12 +1095,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.2.tgz", - "integrity": "sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-4ksWc9n0mhlZpZ9PMZgTGjeOPRu8MB1Z3Tz0Mo02eWfWCHMW1zN82Qz/pL/rC+yQa+8ZnutMF0JjJe7PjwasYw==", "cpu": [ "arm64" ], @@ -1161,12 +1112,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.2.tgz", - "integrity": "sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-SUSDOI6WwUVNcWxd02QEBjLdY1VPHvlEkw6T/8nYG322iYWCTxRb1vzk4E+mWWYehTp7ERibq54LSJGjmouOsw==", "cpu": [ "x64" ], @@ -1175,26 +1129,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.2.tgz", - "integrity": "sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.2.tgz", - "integrity": "sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-hwnz3nw9dbJ05EDO/PvcjaaewqqDy7Y1rn1UO81l8iIK1GjenME75dl16ajbvSSMfv66WXSRCYKIqfgq2KCfxw==", "cpu": [ "x64" ], @@ -1203,46 +1146,32 @@ "optional": true, "os": [ "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.2.tgz", - "integrity": "sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.2.tgz", - "integrity": "sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.17.tgz", + "integrity": "sha512-IS+W7epTcwANmFSQFrS1SivEXHtl1JtuQA9wlxrZTcNi6mx+FDOYrakGevvvTwgj2JvWiK8B29/qD9BELZPyXQ==", "cpu": [ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.2.tgz", - "integrity": "sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-e6usGaHKW5BMNZOymS1UcEYGowQMWcgZ71Z17Sl/h2+ZziNJ1a9n3Zvcz6LdRyIW5572wBCTH/Z+bKuZouGk9Q==", "cpu": [ "arm64" ], @@ -1254,12 +1183,15 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.2.tgz", - "integrity": "sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-b/CgbwAJpmrRLp02RPfhbudf5tZnN9nsPWK82znefso832etkem8H7FSZwxrOI9djcdTP7U6YfNhbRnh7djErg==", "cpu": [ "arm64" ], @@ -1271,46 +1203,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.2.tgz", - "integrity": "sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.2.tgz", - "integrity": "sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "musl" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.2.tgz", - "integrity": "sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-4EII1iNGRUN5WwGbF/kOh/EIkoDN9HsupgLQoXfY+D1oyJm7/F4t5PYU5n8SWZgG0FEwakyM8pGgwcBYruGTlA==", "cpu": [ "ppc64" ], @@ -1322,31 +1223,17 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.2.tgz", - "integrity": "sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "libc": [ - "musl" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.2.tgz", - "integrity": "sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-AH8oq3XqQo4IibpVXvPeLDI5pzkpYn0WiZAfT05kFzoJ6tQNzwRdDYQ45M8I/gslbodRZwW8uxLhbSBbkv96rA==", "cpu": [ - "riscv64" + "s390x" ], "dev": true, "libc": [ @@ -1356,31 +1243,17 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.2.tgz", - "integrity": "sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "musl" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.2.tgz", - "integrity": "sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-cLnjV3xfo7KslbU41Z7z8BH/E1y5mzUYzAqih1d1MDaIGZRCMqTijqLv76/P7fyHuvUcfGsIpqCdddbxLLK9rA==", "cpu": [ - "s390x" + "x64" ], "dev": true, "libc": [ @@ -1390,60 +1263,71 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.2.tgz", - "integrity": "sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-0phclDw1spsL7dUB37sIARuis2tAgomCJXAHZlpt8PXZ4Ba0dRP1e+66lsRqrfhISeN9bEGNjQs+T/Fbd7oYGw==", "cpu": [ "x64" ], "dev": true, "libc": [ - "glibc" + "musl" ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.2.tgz", - "integrity": "sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-0ag/hEgXOwgw4t8QyQvUCxvEg+V0KBcA6YuOx9g0r02MprutRF5dyljgm3EmR02O292UX7UeS6HzWHAl6KgyhA==", "cpu": [ - "x64" + "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ - "linux" - ] + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.2.tgz", - "integrity": "sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==", + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.17.tgz", + "integrity": "sha512-LEXei6vo0E5wTGwpkJ4KoT3OZJRnglwldt5ziLzOlc6qqb55z4tWNq2A+PFqCJuvWWdP53CVhG1Z9NtToDPJrA==", "cpu": [ - "x64" + "wasm32" ], "dev": true, "license": "MIT", "optional": true, - "os": [ - "openbsd" - ] + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.2.tgz", - "integrity": "sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==", + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-gUmyzBl3SPMa6hrqFUth9sVfcLBlYsbMzBx5PlexMroZStgzGqlZ26pYG89rBb45Mnia+oil6YAIFeEWGWhoZA==", "cpu": [ "arm64" ], @@ -1451,64 +1335,58 @@ "license": "MIT", "optional": true, "os": [ - "openharmony" - ] + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.2.tgz", - "integrity": "sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-3hkiolcUAvPB9FLb3UZdfjVVNWherN1f/skkGWJP/fgSQhYUZpSIRr0/I8ZK9TkF3F7kxvJAk0+IcKvPHk9qQg==", "cpu": [ - "arm64" + "x64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] - }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.2.tgz", - "integrity": "sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==", - "cpu": [ - "ia32" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.2.tgz", - "integrity": "sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==", - "cpu": [ - "x64" - ], + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.7", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.7.tgz", + "integrity": "sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==", "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "license": "MIT" }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.2.tgz", - "integrity": "sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==", - "cpu": [ - "x64" - ], + "node_modules/@rollup/pluginutils": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz", + "integrity": "sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==", "dev": true, "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "@types/estree": "^1.0.0", + "estree-walker": "^2.0.2", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } }, "node_modules/@storybook/addon-docs": { "version": "10.3.5", @@ -2156,6 +2034,17 @@ "@testing-library/dom": ">=7.21.4" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", + "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -2253,6 +2142,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2498,45 +2394,6 @@ "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -2592,38 +2449,30 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/@vitejs/plugin-react": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", - "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.1.tgz", + "integrity": "sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/core": "^7.28.0", - "@babel/plugin-transform-react-jsx-self": "^7.27.1", - "@babel/plugin-transform-react-jsx-source": "^7.27.1", - "@rolldown/pluginutils": "1.0.0-beta.27", - "@types/babel__core": "^7.20.5", - "react-refresh": "^0.17.0" + "@rolldown/pluginutils": "1.0.0-rc.7" }, "engines": { - "node": "^14.18.0 || >=16.0.0" + "node": "^20.19.0 || >=22.12.0" }, "peerDependencies": { - "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + } } }, "node_modules/@vitest/expect": { @@ -2715,9 +2564,9 @@ } }, "node_modules/ajv": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", - "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", "dependencies": { @@ -2777,13 +2626,6 @@ "arrow2csv": "bin/arrow2csv.js" } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -2827,11 +2669,14 @@ } }, "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } }, "node_modules/baseline-browser-mapping": { "version": "2.10.21", @@ -2847,14 +2692,16 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", - "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" } }, "node_modules/browserslist": { @@ -2907,16 +2754,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/caniuse-lite": { "version": "1.0.30001790", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001790.tgz", @@ -3052,13 +2889,6 @@ "node": ">=12.20.0" } }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -3330,33 +3160,30 @@ } }, "node_modules/eslint": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", - "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.2", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.5", - "@eslint/js": "9.39.4", - "@eslint/plugin-kit": "^0.4.1", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", - "chalk": "^4.0.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", @@ -3366,8 +3193,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.5", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -3375,7 +3201,7 @@ "eslint": "bin/eslint.js" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" @@ -3406,16 +3232,23 @@ } }, "node_modules/eslint-plugin-react-hooks": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-5.2.0.tgz", - "integrity": "sha512-+f15FfK64YQwZdJNELETdn5ibXEUQmW1DZL6KXhNnc2heoy/sg9VJJeT7n8TlMWouzWqSWavFkIhHyIbIAEapg==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", + "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", "dev": true, "license": "MIT", + "dependencies": { + "@babel/core": "^7.24.4", + "@babel/parser": "^7.24.4", + "hermes-parser": "^0.25.1", + "zod": "^3.25.0 || ^4.0.0", + "zod-validation-error": "^3.5.0 || ^4.0.0" + }, "engines": { - "node": ">=10" + "node": ">=18" }, "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "node_modules/eslint-plugin-react-refresh": { @@ -3443,48 +3276,50 @@ } }, "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.15.0", + "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" + "eslint-visitor-keys": "^5.0.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" @@ -3736,49 +3571,10 @@ "node": ">=10.13.0" } }, - "node_modules/glob/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/globals": { - "version": "16.5.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-16.5.0.tgz", - "integrity": "sha512-c/c15i26VrJ4IRt5Z89DnIzCGDn9EcebibhAOjw5ibqEHsE1wLUgkPn9RDmNcUKyU87GeaL633nyJ+pplFR2ZQ==", + "version": "17.5.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.5.0.tgz", + "integrity": "sha512-qoV+HK2yFl/366t2/Cb3+xxPUo5BuMynomoDmiaZBIdbs+0pYbjfZU+twLhGKp4uCZ/+NbtpVepH5bGCxRyy2g==", "dev": true, "license": "MIT", "engines": { @@ -3817,6 +3613,23 @@ "node": ">= 0.4" } }, + "node_modules/hermes-estree": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", + "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", + "dev": true, + "license": "MIT" + }, + "node_modules/hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", + "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hermes-estree": "0.25.1" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -3827,23 +3640,6 @@ "node": ">= 4" } }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -3978,19 +3774,6 @@ "dev": true, "license": "MIT" }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", @@ -4365,13 +4148,6 @@ "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "license": "MIT" - }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -4421,16 +4197,19 @@ } }, "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.5" }, "engines": { - "node": "*" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minimist": { @@ -4562,19 +4341,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4815,16 +4581,6 @@ "license": "MIT", "peer": true }, - "node_modules/react-refresh": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", - "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/recast": { "version": "0.23.11", "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.11.tgz", @@ -4891,60 +4647,46 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/rollup": { - "version": "4.60.2", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.2.tgz", - "integrity": "sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==", + "node_modules/rolldown": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.17.tgz", + "integrity": "sha512-ZrT53oAKrtA4+YtBWPQbtPOxIbVDbxT0orcYERKd63VJTF13zPcgXTvD4843L8pcsI7M6MErt8QtON6lrB9tyA==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.8" + "@oxc-project/types": "=0.127.0", + "@rolldown/pluginutils": "1.0.0-rc.17" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.60.2", - "@rollup/rollup-android-arm64": "4.60.2", - "@rollup/rollup-darwin-arm64": "4.60.2", - "@rollup/rollup-darwin-x64": "4.60.2", - "@rollup/rollup-freebsd-arm64": "4.60.2", - "@rollup/rollup-freebsd-x64": "4.60.2", - "@rollup/rollup-linux-arm-gnueabihf": "4.60.2", - "@rollup/rollup-linux-arm-musleabihf": "4.60.2", - "@rollup/rollup-linux-arm64-gnu": "4.60.2", - "@rollup/rollup-linux-arm64-musl": "4.60.2", - "@rollup/rollup-linux-loong64-gnu": "4.60.2", - "@rollup/rollup-linux-loong64-musl": "4.60.2", - "@rollup/rollup-linux-ppc64-gnu": "4.60.2", - "@rollup/rollup-linux-ppc64-musl": "4.60.2", - "@rollup/rollup-linux-riscv64-gnu": "4.60.2", - "@rollup/rollup-linux-riscv64-musl": "4.60.2", - "@rollup/rollup-linux-s390x-gnu": "4.60.2", - "@rollup/rollup-linux-x64-gnu": "4.60.2", - "@rollup/rollup-linux-x64-musl": "4.60.2", - "@rollup/rollup-openbsd-x64": "4.60.2", - "@rollup/rollup-openharmony-arm64": "4.60.2", - "@rollup/rollup-win32-arm64-msvc": "4.60.2", - "@rollup/rollup-win32-ia32-msvc": "4.60.2", - "@rollup/rollup-win32-x64-gnu": "4.60.2", - "@rollup/rollup-win32-x64-msvc": "4.60.2", - "fsevents": "~2.3.2" - } + "@rolldown/binding-android-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-x64": "1.0.0-rc.17", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.17", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.17", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.17", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.17", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.17", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.17", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.17" + } + }, + "node_modules/rolldown/node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.17.tgz", + "integrity": "sha512-n8iosDOt6Ig1UhJ2AYqoIhHWh/isz0xpicHTzpKBeotdVsTEcxsSA/i3EVM7gQAj0rU27OLAxCjzlj15IWY7bg==", + "dev": true, + "license": "MIT" }, "node_modules/run-applescript": { "version": "7.1.0", @@ -5091,19 +4833,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -5385,24 +5114,23 @@ } }, "node_modules/vite": { - "version": "6.4.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", - "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", + "version": "8.0.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.10.tgz", + "integrity": "sha512-rZuUu9j6J5uotLDs+cAA4O5H4K1SfPliUlQwqa6YEwSrWDZzP4rhm00oJR5snMewjxF5V/K3D4kctsUTsIU9Mw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.25.0", - "fdir": "^6.4.4", - "picomatch": "^4.0.2", - "postcss": "^8.5.3", - "rollup": "^4.34.9", - "tinyglobby": "^0.2.13" + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.10", + "rolldown": "1.0.0-rc.17", + "tinyglobby": "^0.2.16" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -5411,14 +5139,15 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.0", + "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" @@ -5427,13 +5156,16 @@ "@types/node": { "optional": true }, - "jiti": { + "@vitejs/devtools": { "optional": true }, - "less": { + "esbuild": { + "optional": true + }, + "jiti": { "optional": true }, - "lightningcss": { + "less": { "optional": true }, "sass": { @@ -5459,490 +5191,6 @@ } } }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, "node_modules/webpack-virtual-modules": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", @@ -6042,6 +5290,29 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", + "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-validation-error": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", + "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } } } } diff --git a/vortex-web/package.json b/vortex-web/package.json index b0b1d825faa..9f21562641d 100644 --- a/vortex-web/package.json +++ b/vortex-web/package.json @@ -27,25 +27,25 @@ "react-dom": "^19.1.0" }, "devDependencies": { - "@eslint/js": "^9.25.0", - "@storybook/addon-docs": "^10.3.3", - "@storybook/react-vite": "^10.3.3", - "@tailwindcss/vite": "^4.1.4", + "@eslint/js": "^10.0.1", + "@storybook/addon-docs": "^10.3.5", + "@storybook/react-vite": "^10.3.5", + "@tailwindcss/vite": "^4.2.4", "@types/d3-hierarchy": "^3.1.7", - "@types/react": "^19.1.2", - "@types/react-dom": "^19.1.2", - "@vitejs/plugin-react": "^4.4.1", - "eslint": "^9.25.0", + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "eslint": "^10.2.1", "eslint-config-prettier": "^10.1.8", - "eslint-plugin-react-hooks": "^5.2.0", - "eslint-plugin-react-refresh": "^0.5.0", - "eslint-plugin-storybook": "^10.3.3", - "globals": "^16.0.0", - "prettier": "^3.8.1", - "storybook": "^10.3.3", - "tailwindcss": "^4.1.4", + "eslint-plugin-react-hooks": "^7.1.1", + "eslint-plugin-react-refresh": "^0.5.2", + "eslint-plugin-storybook": "^10.3.5", + "globals": "^17.5.0", + "prettier": "^3.8.3", + "storybook": "^10.3.5", + "tailwindcss": "^4.2.4", "typescript": "~6.0.0", - "typescript-eslint": "^8.30.1", - "vite": "^6.3.2" + "typescript-eslint": "^8.59.0", + "vite": "^8.0.10" } } From ea31cc2402ac2c535e5cad0ccc61ddee9798de98 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:10:35 +0000 Subject: [PATCH 209/250] Update dependency concurrently to v9 (#7656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [concurrently](https://redirect.github.com/open-cli-tools/concurrently) | [`^8.2.2` → `^9.0.0`](https://renovatebot.com/diffs/npm/concurrently/8.2.2/9.2.1) | ![age](https://developer.mend.io/api/mc/badges/age/npm/concurrently/9.2.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/concurrently/8.2.2/9.2.1?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes
open-cli-tools/concurrently (concurrently) ### [`v9.2.1`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.2.1) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.2.0...v9.2.1) #### What's Changed - chore: update eslint-plugin-simple-import-sort from v10 to v12 by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​551](https://redirect.github.com/open-cli-tools/concurrently/pull/551) - chore: update eslint-config-prettier from v9 to v10 by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​552](https://redirect.github.com/open-cli-tools/concurrently/pull/552) - Remove lodash by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​555](https://redirect.github.com/open-cli-tools/concurrently/pull/555) - chore: update coveralls-next from v4 to v5 by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​557](https://redirect.github.com/open-cli-tools/concurrently/pull/557) - Replace jest with vitest by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​554](https://redirect.github.com/open-cli-tools/concurrently/pull/554) - Upgrade to pnpm v10 by [@​paescuj](https://redirect.github.com/paescuj) in [#​558](https://redirect.github.com/open-cli-tools/concurrently/pull/558) - chore: remove unused eslint-plugin-jest by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​559](https://redirect.github.com/open-cli-tools/concurrently/pull/559) - Minor dependency updates by [@​paescuj](https://redirect.github.com/paescuj) in [#​560](https://redirect.github.com/open-cli-tools/concurrently/pull/560) - Migrate to ESLint v9 by [@​paescuj](https://redirect.github.com/paescuj) in [#​561](https://redirect.github.com/open-cli-tools/concurrently/pull/561) - Update shell-quote to 1.8.3 by [@​paescuj](https://redirect.github.com/paescuj) in [#​562](https://redirect.github.com/open-cli-tools/concurrently/pull/562) - Full coverage by [@​paescuj](https://redirect.github.com/paescuj) in [#​563](https://redirect.github.com/open-cli-tools/concurrently/pull/563) - Update GH actions/workflows, enable NPM provenance by [@​paescuj](https://redirect.github.com/paescuj) in [#​564](https://redirect.github.com/open-cli-tools/concurrently/pull/564) **Full Changelog**: ### [`v9.2.0`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.2.0) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.1.2...v9.2.0) #### What's Changed - Bump esbuild from 0.23.1 to 0.25.0 in the npm\_and\_yarn group by [@​dependabot](https://redirect.github.com/dependabot) in [#​528](https://redirect.github.com/open-cli-tools/concurrently/pull/528) - fix: don't throw when there are no commands by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​532](https://redirect.github.com/open-cli-tools/concurrently/pull/532) - docs: nicer quotes by [@​IsaacLeeWebDev](https://redirect.github.com/IsaacLeeWebDev) in [#​537](https://redirect.github.com/open-cli-tools/concurrently/pull/537) - Add `--kill-timeout` by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​540](https://redirect.github.com/open-cli-tools/concurrently/pull/540) - docs: fix typo by [@​ldeveber](https://redirect.github.com/ldeveber) in [#​542](https://redirect.github.com/open-cli-tools/concurrently/pull/542) - fix: correct typos in comments and documentation by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​544](https://redirect.github.com/open-cli-tools/concurrently/pull/544) - refactor: use startsWith & simplify boolean expression by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​543](https://redirect.github.com/open-cli-tools/concurrently/pull/543) - refactor: use optional chaining by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​545](https://redirect.github.com/open-cli-tools/concurrently/pull/545) - Handle SIGPIPEs by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​547](https://redirect.github.com/open-cli-tools/concurrently/pull/547) - refactor: fix map and reduce as return values are not used by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​546](https://redirect.github.com/open-cli-tools/concurrently/pull/546) - docs: fix typos in docs by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​548](https://redirect.github.com/open-cli-tools/concurrently/pull/548) - chore: update jest from v29 to v30 by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​549](https://redirect.github.com/open-cli-tools/concurrently/pull/549) - chore: update [@​types/jest](https://redirect.github.com/types/jest) from v29 to v30 by [@​noritaka1166](https://redirect.github.com/noritaka1166) in [#​550](https://redirect.github.com/open-cli-tools/concurrently/pull/550) #### New Contributors - [@​IsaacLeeWebDev](https://redirect.github.com/IsaacLeeWebDev) made their first contribution in [#​537](https://redirect.github.com/open-cli-tools/concurrently/pull/537) - [@​ldeveber](https://redirect.github.com/ldeveber) made their first contribution in [#​542](https://redirect.github.com/open-cli-tools/concurrently/pull/542) - [@​noritaka1166](https://redirect.github.com/noritaka1166) made their first contribution in [#​544](https://redirect.github.com/open-cli-tools/concurrently/pull/544) **Full Changelog**: ### [`v9.1.2`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.1.2) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.1.1...v9.1.2) #### What's Changed - Add ability to have custom logger by [@​mwood23](https://redirect.github.com/mwood23) in [#​522](https://redirect.github.com/open-cli-tools/concurrently/pull/522) #### New Contributors - [@​mwood23](https://redirect.github.com/mwood23) made their first contribution in [#​522](https://redirect.github.com/open-cli-tools/concurrently/pull/522) **Full Changelog**: ### [`v9.1.1`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.1.1) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.1.0...v9.1.1) #### What's Changed - fix: support Deno's JSON with comments configuration by [@​mahtaran](https://redirect.github.com/mahtaran) in [#​523](https://redirect.github.com/open-cli-tools/concurrently/pull/523) **Full Changelog**: ### [`v9.1.0`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.1.0) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.0.1...v9.1.0) #### What's Changed - Remove signal event listeners on finish by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​512](https://redirect.github.com/open-cli-tools/concurrently/pull/512) - Add support for Deno shortcuts and wildcards by [@​mahtaran](https://redirect.github.com/mahtaran) in [#​508](https://redirect.github.com/open-cli-tools/concurrently/pull/508) - bin: show help when no args are passed by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​513](https://redirect.github.com/open-cli-tools/concurrently/pull/513) #### New Contributors - [@​mahtaran](https://redirect.github.com/mahtaran) made their first contribution in [#​508](https://redirect.github.com/open-cli-tools/concurrently/pull/508) **Full Changelog**: ### [`v9.0.1`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.0.1) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v9.0.0...v9.0.1) #### What's Changed - Don't set up more than 1 abort signal listener by [@​gustavohenke](https://redirect.github.com/gustavohenke) in [#​503](https://redirect.github.com/open-cli-tools/concurrently/pull/503) **Full Changelog**: ### [`v9.0.0`](https://redirect.github.com/open-cli-tools/concurrently/releases/tag/v9.0.0) [Compare Source](https://redirect.github.com/open-cli-tools/concurrently/compare/v8.2.2...v9.0.0) ### 💥 Breaking Changes - **Dropped support for Node.js < 18.0.0**. Older Node.js version have reached end-of-life, and certain features require new-ish JS APIs. - **Pending commands no longer run when `--max-processes` is set and an interruption/kill signal is sent** - [#​433](https://redirect.github.com/open-cli-tools/concurrently/issues/433), [#​452](https://redirect.github.com/open-cli-tools/concurrently/issues/452), [#​460](https://redirect.github.com/open-cli-tools/concurrently/issues/460) Before v9.0.0, pressing Ctrl+C when `--max-processes` is set meant that only those commands would receive it. Once these stopped, the pending commands would start, which in turn meant that another Ctrl+C press would be necessary to stop them, and so on. Similar situation applied to combining `--max-processes` with `--kill-others`/`--kill-others-on-fail`. Starting with v9.0.0, this no longer happens, as these flags and/or key presses send a signal to stop the running commands in addition to preventing new commands from running. - **The `concurrently` and default exports are now the same** - [#​399](https://redirect.github.com/open-cli-tools/concurrently/issues/399) It's reasonable to expect that `import { concurrently } from 'concurrently'` would work the same as `import concurrently from 'concurrently'`, however this has not been the case. The former used to be an unconfigured version of concurrently, lacking all features that you'd get from the CLI, which was seen as a "footgun". Both are now the same function. If you'd like to access the unconfigured function, use `import { createConcurrently } from 'concurrently'` instead. ### ✨ New Features - Exponential back-off support for process restarting - [#​265](https://redirect.github.com/open-cli-tools/concurrently/issues/265), [#​462](https://redirect.github.com/open-cli-tools/concurrently/issues/462) Use `--restart-after exponential`. Restarts happen at `2^N` seconds. - Add prefix padding via new `--pad-prefix` flag - [#​166](https://redirect.github.com/open-cli-tools/concurrently/issues/166), [#​417](https://redirect.github.com/open-cli-tools/concurrently/issues/417), [#​495](https://redirect.github.com/open-cli-tools/concurrently/issues/495) - Specify teardown commands via new `--teardown` flag - [#​472](https://redirect.github.com/open-cli-tools/concurrently/issues/472), [#​500](https://redirect.github.com/open-cli-tools/concurrently/issues/500) - Expand `node:

The set of paths belongs to each {@link VortexFilePartition} — the factory itself is + * stateless across partitions. For every input partition, {@link VortexPartitionReader} + * opens a single {@code Session}, {@code DataSource} and {@code Scan} spanning that + * partition's paths and consumes every Vortex partition produced by that scan before + * returning. + */ +public final class VortexPartitionReaderFactory implements PartitionReaderFactory, Serializable { + private static final long serialVersionUID = 1L; + + private final ImmutableList dataColumnNames; + private final ImmutableMap formatOptions; + + public VortexPartitionReaderFactory(List dataColumnNames, java.util.Map formatOptions) { + this.dataColumnNames = ImmutableList.copyOf(dataColumnNames); + this.formatOptions = ImmutableMap.copyOf(formatOptions); + } + + @Override + public PartitionReader createReader(InputPartition partition) { + throw new UnsupportedOperationException("row-based reads are not supported"); + } + + @Override + public PartitionReader createColumnarReader(InputPartition partition) { + VortexFilePartition spark = (VortexFilePartition) partition; + return new VortexPartitionReader(spark, dataColumnNames, formatOptions); + } + + @Override + public boolean supportColumnarReads(InputPartition partition) { + return true; + } +} diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java index 1ec1a0aeb7d..d20cd719135 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java @@ -3,8 +3,8 @@ package dev.vortex.spark.read; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; import org.apache.spark.sql.connector.catalog.CatalogV2Util; import org.apache.spark.sql.connector.catalog.Column; import org.apache.spark.sql.connector.read.Batch; @@ -16,20 +16,18 @@ */ public final class VortexScan implements Scan { - private final ImmutableList paths; - private final ImmutableList readColumns; - private final ImmutableMap formatOptions; + private final List paths; + private final List readColumns; + private final Map formatOptions; /** - * Creates a new VortexScan for the specified file paths and columns. + * Creates a new VortexScan for the specified file paths and columns. The caller is + * responsible for passing immutable collections; the constructor does not copy. * - * @param paths the list of Vortex file paths to scan + * @param paths the list of Vortex file paths to scan * @param readColumns the list of columns to read from the files */ - public VortexScan( - ImmutableList paths, - ImmutableList readColumns, - ImmutableMap formatOptions) { + public VortexScan(List paths, List readColumns, Map formatOptions) { this.paths = paths; this.readColumns = readColumns; this.formatOptions = formatOptions; diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index 0ea690b6406..6155f54d386 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -6,7 +6,6 @@ import static com.google.common.base.Preconditions.checkState; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -31,7 +30,7 @@ public final class VortexScanBuilder implements ScanBuilder, SupportsPushDownReq public VortexScanBuilder(Map formatOptions) { this.paths = ImmutableList.builder(); this.columns = new ArrayList<>(); - this.formatOptions = formatOptions; + this.formatOptions = Map.copyOf(formatOptions); } /** @@ -89,14 +88,12 @@ public VortexScanBuilder addAllColumns(Iterable columns) { @Override public Scan build() { var paths = this.paths.build(); - var columns = ImmutableList.copyOf(this.columns); - var formatOptions = ImmutableMap.copyOf(this.formatOptions); checkState(!paths.isEmpty(), "paths cannot be empty"); // Allow empty columns for operations like count() that don't need actual column data // If no columns are specified, we'll read the minimal schema needed - return new VortexScan(paths, columns, formatOptions); + return new VortexScan(paths, List.copyOf(this.columns), this.formatOptions); } /** diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexBatchWrite.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexBatchWrite.java index be0b516e81e..ebb18bd9777 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexBatchWrite.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexBatchWrite.java @@ -3,7 +3,8 @@ package dev.vortex.spark.write; -import dev.vortex.jni.NativeFileMethods; +import dev.vortex.jni.NativeFiles; +import dev.vortex.spark.VortexSparkSession; import java.io.IOException; import java.io.Serializable; import java.nio.file.Files; @@ -83,9 +84,10 @@ public BatchWrite toBatch() { public DataWriterFactory createBatchWriterFactory(PhysicalWriteInfo info) { // Handle overwrite cleanup BEFORE writing starts if (overwrite) { - var uris = NativeFileMethods.listVortexFiles(outputPath, options); + var session = VortexSparkSession.get(options); + var uris = NativeFiles.listFiles(session, outputPath, options); log.info("truncating table with {} files", uris.size()); - NativeFileMethods.delete(uris.toArray(new String[0]), options); + NativeFiles.delete(session, uris.toArray(new String[0]), options); log.warn("overwrite currently does not do anything for vortex format"); } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java index df66c4ca254..fc88889eb04 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java @@ -3,6 +3,7 @@ package dev.vortex.spark.write; +import dev.vortex.api.Session; import dev.vortex.api.VortexWriter; import dev.vortex.relocated.org.apache.arrow.c.ArrowArray; import dev.vortex.relocated.org.apache.arrow.c.ArrowSchema; @@ -14,7 +15,7 @@ import dev.vortex.relocated.org.apache.arrow.vector.VectorSchemaRoot; import dev.vortex.relocated.org.apache.arrow.vector.complex.ListVector; import dev.vortex.relocated.org.apache.arrow.vector.complex.StructVector; -import dev.vortex.spark.SparkTypes; +import dev.vortex.spark.VortexSparkSession; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -49,6 +50,7 @@ public final class VortexDataWriter implements DataWriter, AutoClos private final CaseInsensitiveStringMap options; private final int batchSize; + private Session session; private VortexWriter vortexWriter; private BufferAllocator allocator; private VectorSchemaRoot vectorSchemaRoot; @@ -89,17 +91,12 @@ public final class VortexDataWriter implements DataWriter, AutoClos } try { - // Initialize Arrow components this.allocator = new RootAllocator(); - - // Convert Spark schema to Vortex and Arrow schemas. - var writeSchema = SparkTypes.toDType(schema); var arrowSchema = SparkToArrowSchema.convert(schema); - // Create Vortex writer - this.vortexWriter = VortexWriter.create(filePath, writeSchema, options.asCaseSensitiveMap()); - - // Create VectorSchemaRoot for batching rows + this.session = VortexSparkSession.get(options.asCaseSensitiveMap()); + this.vortexWriter = + VortexWriter.create(session, filePath, arrowSchema, options.asCaseSensitiveMap(), allocator); this.vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, allocator); logger.debug("Initialized VortexDataWriter for {}", filePath); @@ -171,7 +168,7 @@ private void writeBatch() throws IOException { try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator); ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator)) { Data.exportVectorSchemaRoot(allocator, vectorSchemaRoot, null, arrowArray, arrowSchema); - vortexWriter.writeBatchFfi(arrowArray.memoryAddress(), arrowSchema.memoryAddress()); + vortexWriter.writeBatch(arrowArray.memoryAddress(), arrowSchema.memoryAddress()); } vectorSchemaRoot.clear(); @@ -298,7 +295,7 @@ public WriterCommitMessage commit() throws IOException { } // The Arrow C Data Interface export (Data.exportVectorSchemaRoot) creates structural - // allocations from this allocator. When writeBatchFfi passes the ArrowArray to Rust, + // allocations from this allocator. When writeBatch passes the ArrowArray to Rust, // FFI_ArrowArray::from_raw() takes ownership and nullifies the release callback on // the Java side. The Rust side calls release asynchronously on its own thread, so // small structural allocations may still be outstanding when the allocator is closed. @@ -312,6 +309,10 @@ public WriterCommitMessage commit() throws IOException { allocator = null; } + // Session is the JVM-wide singleton held by VortexSparkSession; we just + // drop our local handle to it here. + session = null; + closed = true; // Throw any exception that occurred during cleanup @@ -358,6 +359,10 @@ public void abort() throws IOException { allocator = null; } + // Session is the JVM-wide singleton held by VortexSparkSession; we just + // drop our local handle to it here. + session = null; + // Delete the partial file if it exists try { Files.deleteIfExists(Paths.get(filePath)); diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/SparkTypesTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/SparkTypesTest.java deleted file mode 100644 index 6c80ab6b694..00000000000 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/SparkTypesTest.java +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -package dev.vortex.spark; - -import static org.junit.jupiter.api.Assertions.*; - -import dev.vortex.api.DType; -import dev.vortex.jni.NativeLoader; -import org.apache.spark.sql.types.ArrayType; -import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public final class SparkTypesTest { - - @BeforeAll - public static void loadLibrary() { - NativeLoader.loadJni(); - } - - @Test - @DisplayName("toDataType should convert FIXED_SIZE_LIST to Spark ArrayType") - public void testFixedSizeListToDataType() { - var elementType = DType.newInt(false); - var fslType = DType.newFixedSizeList(elementType, 3, true); - var sparkType = SparkTypes.toDataType(fslType); - assertInstanceOf(ArrayType.class, sparkType); - ArrayType arrayType = (ArrayType) sparkType; - assertEquals(DataTypes.IntegerType, arrayType.elementType()); - } - - @Test - @DisplayName("toDataType should convert LIST to Spark ArrayType") - public void testListToDataType() { - var elementType = DType.newDouble(false); - var listType = DType.newList(elementType, true); - var sparkType = SparkTypes.toDataType(listType); - assertInstanceOf(ArrayType.class, sparkType); - ArrayType arrayType = (ArrayType) sparkType; - assertEquals(DataTypes.DoubleType, arrayType.elementType()); - } -} diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceBasicTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceBasicTest.java index 81b0ae3613a..f4440e7512a 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceBasicTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceBasicTest.java @@ -6,6 +6,8 @@ import static org.junit.jupiter.api.Assertions.*; import dev.vortex.relocated.org.apache.arrow.vector.types.pojo.ArrowType; +import dev.vortex.spark.write.SparkToArrowSchema; +import dev.vortex.spark.write.VortexWriterCommitMessage; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -36,7 +38,7 @@ public void testSparkToArrowSchemaConversion() { }); // Convert to Arrow schema - var arrowSchema = dev.vortex.spark.write.SparkToArrowSchema.convert(sparkSchema); + var arrowSchema = SparkToArrowSchema.convert(sparkSchema); // Verify conversion assertNotNull(arrowSchema, "Arrow schema should not be null"); @@ -66,7 +68,7 @@ public void testNestedSparkToArrowSchemaConversion() { }); // Convert to Arrow schema - var arrowSchema = dev.vortex.spark.write.SparkToArrowSchema.convert(sparkSchema); + var arrowSchema = SparkToArrowSchema.convert(sparkSchema); // Verify conversion assertNotNull(arrowSchema, "Arrow schema should not be null"); @@ -98,7 +100,7 @@ public void testWriterCommitMessage() { long recordCount = 1000; long bytesWritten = 50000; - var message = new dev.vortex.spark.write.VortexWriterCommitMessage(testPath, recordCount, bytesWritten); + var message = new VortexWriterCommitMessage(testPath, recordCount, bytesWritten); assertEquals(testPath, message.filePath()); assertEquals(recordCount, message.recordCount()); diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java index be3d2e5da59..ae238aa46b1 100644 --- a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceS3MockTest.java @@ -47,6 +47,7 @@ public void setUp() { .config("spark.sql.adaptive.enabled", "false") .config("spark.driver.host", "127.0.0.1") .config("spark.ui.enabled", "false") + .config("spark.driver.host", "127.0.0.1") // S3A configuration for S3Mock. // This should be propagated into our reader .config("spark.hadoop.fs.s3a.endpoint", s3Endpoint) diff --git a/vortex-duckdb/cpp/file_system.cpp b/vortex-duckdb/cpp/file_system.cpp index 4e5c6cab414..11083ad6b86 100644 --- a/vortex-duckdb/cpp/file_system.cpp +++ b/vortex-duckdb/cpp/file_system.cpp @@ -150,3 +150,19 @@ extern "C" duckdb_state duckdb_vx_fs_sync(duckdb_vx_file_handle handle, duckdb_v } return DuckDBSuccess; } + +extern "C" duckdb_state +duckdb_vx_fs_remove(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out) { + if (!ctx || !path) { + return SetError(error_out, "Invalid arguments to fs_remove"); + } + + auto *client_context = reinterpret_cast(ctx); + + try { + FileSystem::GetFileSystem(*client_context).RemoveFile(path); + } catch (const std::exception &e) { + return SetError(error_out, e.what()); + } + return DuckDBSuccess; +} diff --git a/vortex-duckdb/cpp/include/duckdb_vx/file_system.h b/vortex-duckdb/cpp/include/duckdb_vx/file_system.h index bfe31b12aff..567b3f7163f 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/file_system.h +++ b/vortex-duckdb/cpp/include/duckdb_vx/file_system.h @@ -75,6 +75,9 @@ duckdb_state duckdb_vx_fs_write(duckdb_vx_file_handle handle, // Flush pending writes to storage. duckdb_state duckdb_vx_fs_sync(duckdb_vx_file_handle handle, duckdb_vx_error *error_out); +// Delete a file using DuckDB's filesystem. +duckdb_state duckdb_vx_fs_remove(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out); + #ifdef __cplusplus /* End C ABI */ } #endif diff --git a/vortex-duckdb/src/filesystem.rs b/vortex-duckdb/src/filesystem.rs index e6d5c7c9750..8c09ed6b147 100644 --- a/vortex-duckdb/src/filesystem.rs +++ b/vortex-duckdb/src/filesystem.rs @@ -149,6 +149,27 @@ impl FileSystem for DuckDbFileSystem { let reader = unsafe { DuckDbFsReader::open_url(self.ctx.as_ptr(), &url)? }; Ok(Arc::new(reader)) } + + async fn delete(&self, path: &str) -> VortexResult<()> { + let mut url = self.base_url.clone(); + url.set_path(path); + let c_path = CString::new(url.as_str()).map_err(|e| vortex_err!("Invalid URL: {e}"))?; + let ctx = self.ctx; + + RUNTIME + .handle() + .spawn_blocking(move || { + let mut err: cpp::duckdb_vx_error = ptr::null_mut(); + let status = unsafe { + cpp::duckdb_vx_fs_remove(ctx.as_ptr(), c_path.as_ptr(), &raw mut err) + }; + if status != cpp::duckdb_state::DuckDBSuccess { + return Err(fs_error(err)); + } + Ok::<_, VortexError>(()) + }) + .await + } } /// Recursively list all files under `directory`, stripping `base_path` from each diff --git a/vortex-io/Cargo.toml b/vortex-io/Cargo.toml index dcb9377e55e..45ba0d90f9d 100644 --- a/vortex-io/Cargo.toml +++ b/vortex-io/Cargo.toml @@ -23,7 +23,6 @@ async-trait = { workspace = true } bytes = { workspace = true } futures = { workspace = true, features = ["std", "executor"] } glob = { workspace = true } -handle = "1.0.2" kanal = { workspace = true } object_store = { workspace = true, optional = true, features = ["fs"] } oneshot = { workspace = true } diff --git a/vortex-io/public-api.lock b/vortex-io/public-api.lock index 08d74ffc830..d611f71f785 100644 --- a/vortex-io/public-api.lock +++ b/vortex-io/public-api.lock @@ -12,6 +12,8 @@ impl<'__pin, T> core::marker::Unpin for vortex_io::compat::Compat where pin_p impl vortex_io::filesystem::FileSystem for vortex_io::compat::Compat +pub fn vortex_io::compat::Compat::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + pub fn vortex_io::compat::Compat::list(&self, prefix: &str) -> futures_core::stream::BoxStream<'_, vortex_error::VortexResult> pub fn vortex_io::compat::Compat::open_read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait @@ -50,6 +52,40 @@ pub type vortex_io::compat::Compat::Output = ::poll(self: core::pin::Pin<&mut Self>, cx: &mut core::task::wake::Context<'_>) -> core::task::poll::Poll +impl core::fmt::Display for vortex_io::compat::Compat + +pub fn vortex_io::compat::Compat::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl object_store::ObjectStore for vortex_io::compat::Compat + +pub fn vortex_io::compat::Compat::copy_opts<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, from: &'life1 object_store::path::Path, to: &'life2 object_store::path::Path, options: object_store::CopyOptions) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait + +pub fn vortex_io::compat::Compat::delete_stream(&self, locations: futures_core::stream::BoxStream<'static, object_store::Result>) -> futures_core::stream::BoxStream<'static, object_store::Result> + +pub fn vortex_io::compat::Compat::get_opts<'life0, 'life1, 'async_trait>(&'life0 self, location: &'life1 object_store::path::Path, options: object_store::GetOptions) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + +pub fn vortex_io::compat::Compat::get_ranges<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, location: &'life1 object_store::path::Path, ranges: &'life2 [core::ops::range::Range]) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait + +pub fn vortex_io::compat::Compat::list(&self, prefix: core::option::Option<&object_store::path::Path>) -> futures_core::stream::BoxStream<'static, object_store::Result> + +pub fn vortex_io::compat::Compat::list_with_delimiter<'life0, 'life1, 'async_trait>(&'life0 self, prefix: core::option::Option<&'life1 object_store::path::Path>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + +pub fn vortex_io::compat::Compat::list_with_offset(&self, prefix: core::option::Option<&object_store::path::Path>, offset: &object_store::path::Path) -> futures_core::stream::BoxStream<'static, object_store::Result> + +pub fn vortex_io::compat::Compat::put_multipart_opts<'life0, 'life1, 'async_trait>(&'life0 self, location: &'life1 object_store::path::Path, opts: object_store::PutMultipartOptions) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + +pub fn vortex_io::compat::Compat::put_opts<'life0, 'life1, 'async_trait>(&'life0 self, location: &'life1 object_store::path::Path, payload: object_store::payload::PutPayload, opts: object_store::PutOptions) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + +pub fn vortex_io::compat::Compat::rename_opts<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, from: &'life1 object_store::path::Path, to: &'life2 object_store::path::Path, options: object_store::RenameOptions) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait + +impl object_store::upload::MultipartUpload for vortex_io::compat::Compat + +pub fn vortex_io::compat::Compat::abort<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait + +pub fn vortex_io::compat::Compat::complete<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait + +pub fn vortex_io::compat::Compat::put_part(&mut self, data: object_store::payload::PutPayload) -> object_store::upload::UploadPart + impl core::ops::drop::Drop for vortex_io::compat::Compat pub fn vortex_io::compat::Compat::drop(&mut self) @@ -96,18 +132,24 @@ impl core::marker::StructuralPartialEq for vortex_io::filesystem::FileListing pub trait vortex_io::filesystem::FileSystem: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn vortex_io::filesystem::FileSystem::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + pub fn vortex_io::filesystem::FileSystem::list(&self, prefix: &str) -> futures_core::stream::BoxStream<'_, vortex_error::VortexResult> pub fn vortex_io::filesystem::FileSystem::open_read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait impl vortex_io::filesystem::FileSystem for vortex_io::object_store::ObjectStoreFileSystem +pub fn vortex_io::object_store::ObjectStoreFileSystem::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + pub fn vortex_io::object_store::ObjectStoreFileSystem::list(&self, prefix: &str) -> futures_core::stream::BoxStream<'_, vortex_error::VortexResult> pub fn vortex_io::object_store::ObjectStoreFileSystem::open_read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait impl vortex_io::filesystem::FileSystem for vortex_io::compat::Compat +pub fn vortex_io::compat::Compat::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + pub fn vortex_io::compat::Compat::list(&self, prefix: &str) -> futures_core::stream::BoxStream<'_, vortex_error::VortexResult> pub fn vortex_io::compat::Compat::open_read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait @@ -140,6 +182,8 @@ pub fn vortex_io::object_store::ObjectStoreFileSystem::fmt(&self, f: &mut core:: impl vortex_io::filesystem::FileSystem for vortex_io::object_store::ObjectStoreFileSystem +pub fn vortex_io::object_store::ObjectStoreFileSystem::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait + pub fn vortex_io::object_store::ObjectStoreFileSystem::list(&self, prefix: &str) -> futures_core::stream::BoxStream<'_, vortex_error::VortexResult> pub fn vortex_io::object_store::ObjectStoreFileSystem::open_read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait diff --git a/vortex-io/src/compat/filesystem.rs b/vortex-io/src/compat/filesystem.rs index e9f9dbf292f..9f9902b6a08 100644 --- a/vortex-io/src/compat/filesystem.rs +++ b/vortex-io/src/compat/filesystem.rs @@ -25,4 +25,8 @@ impl FileSystem for Compat { let read_at = Compat::new(self.inner().open_read(path)).await?; Ok(Arc::new(Compat::new(read_at))) } + + async fn delete(&self, path: &str) -> VortexResult<()> { + Compat::new(self.inner().delete(path)).await + } } diff --git a/vortex-io/src/compat/mod.rs b/vortex-io/src/compat/mod.rs index eb31d05b2ff..843c4cfdcb7 100644 --- a/vortex-io/src/compat/mod.rs +++ b/vortex-io/src/compat/mod.rs @@ -11,6 +11,8 @@ //! and async-compat only supports the latter. mod filesystem; +#[cfg(feature = "object_store")] +mod obj_store; mod read_at; mod write; diff --git a/vortex-io/src/compat/obj_store.rs b/vortex-io/src/compat/obj_store.rs new file mode 100644 index 00000000000..8ff9cd5e94b --- /dev/null +++ b/vortex-io/src/compat/obj_store.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; +use std::ops::Range; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use object_store::CopyOptions; +use object_store::GetOptions; +use object_store::GetResult; +use object_store::ListResult; +use object_store::MultipartUpload; +use object_store::ObjectMeta; +use object_store::ObjectStore; +use object_store::PutMultipartOptions; +use object_store::PutOptions; +use object_store::PutPayload; +use object_store::PutResult; +use object_store::RenameOptions; +use object_store::Result; +use object_store::UploadPart; +use object_store::path::Path; +use smol::future::FutureExt; +use smol::stream::StreamExt; + +use crate::compat::Compat; + +impl Display for Compat { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "Compat<{}>", self.inner()) + } +} + +#[async_trait] +impl ObjectStore for Compat { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> Result { + Compat::new(self.inner().put_opts(location, payload, opts)).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> Result> { + Ok(Box::new(Compat::new( + Compat::new(self.inner().put_multipart_opts(location, opts)).await?, + ))) + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> Result { + Compat::new(self.inner().get_opts(location, options)).await + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> Result> { + Compat::new(self.inner().get_ranges(location, ranges)).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, Result>, + ) -> BoxStream<'static, Result> { + Compat::new(self.inner().delete_stream(locations)).boxed() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result> { + Compat::new(self.inner().list(prefix)).boxed() + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, Result> { + Compat::new(self.inner().list_with_offset(prefix, offset)).boxed() + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + Compat::new(self.inner().list_with_delimiter(prefix)).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> { + Compat::new(self.inner().copy_opts(from, to, options)).await + } + + async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> { + Compat::new(self.inner().rename_opts(from, to, options)).await + } +} + +#[async_trait] +impl MultipartUpload for Compat { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + Compat::new(self.inner_mut().put_part(data)).boxed() + } + + async fn complete(&mut self) -> Result { + Compat::new(self.inner_mut().complete()).await + } + + async fn abort(&mut self) -> Result<()> { + Compat::new(self.inner_mut().abort()).await + } +} diff --git a/vortex-io/src/compat/read_at.rs b/vortex-io/src/compat/read_at.rs index 03f47bf58b3..4fc49785d28 100644 --- a/vortex-io/src/compat/read_at.rs +++ b/vortex-io/src/compat/read_at.rs @@ -29,7 +29,7 @@ impl VortexReadAt for Compat { } fn size(&self) -> BoxFuture<'static, VortexResult> { - self.inner().size() + Compat::new(self.inner().size()).boxed() } fn read_at( diff --git a/vortex-io/src/filesystem/glob.rs b/vortex-io/src/filesystem/glob.rs index adb6da3c394..6bbe26fa5dc 100644 --- a/vortex-io/src/filesystem/glob.rs +++ b/vortex-io/src/filesystem/glob.rs @@ -106,6 +106,10 @@ mod tests { async fn open_read(&self, _path: &str) -> VortexResult> { vortex_panic!("open_read() should not be called") } + + async fn delete(&self, _path: &str) -> VortexResult<()> { + vortex_panic!("delete() should not be called") + } } #[tokio::test] diff --git a/vortex-io/src/filesystem/mod.rs b/vortex-io/src/filesystem/mod.rs index 8749d87d8ae..97535909d28 100644 --- a/vortex-io/src/filesystem/mod.rs +++ b/vortex-io/src/filesystem/mod.rs @@ -53,4 +53,6 @@ pub trait FileSystem: Debug + Send + Sync { /// Open a file for reading at the given path. async fn open_read(&self, path: &str) -> VortexResult>; + + async fn delete(&self, path: &str) -> VortexResult<()>; } diff --git a/vortex-io/src/filesystem/prefix.rs b/vortex-io/src/filesystem/prefix.rs index d194b30960d..5f6304430aa 100644 --- a/vortex-io/src/filesystem/prefix.rs +++ b/vortex-io/src/filesystem/prefix.rs @@ -57,6 +57,12 @@ impl FileSystem for PrefixFileSystem { .open_read(&format!("{}{}", self.prefix, path.trim_start_matches('/'))) .await } + + async fn delete(&self, path: &str) -> VortexResult<()> { + self.inner + .delete(&format!("{}{}", self.prefix, path.trim_start_matches('/'))) + .await + } } impl dyn FileSystem + 'static { diff --git a/vortex-io/src/object_store/filesystem.rs b/vortex-io/src/object_store/filesystem.rs index 6fdbc23caa3..b8c2aa39ce2 100644 --- a/vortex-io/src/object_store/filesystem.rs +++ b/vortex-io/src/object_store/filesystem.rs @@ -11,8 +11,10 @@ use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; use object_store::ObjectStore; +use object_store::ObjectStoreExt; use object_store::path::Path; use vortex_error::VortexResult; +use vortex_error::vortex_err; use crate::VortexReadAt; use crate::filesystem::FileListing; @@ -81,4 +83,14 @@ impl FileSystem for ObjectStoreFileSystem { self.handle.clone(), ))) } + + async fn delete(&self, path: &str) -> VortexResult<()> { + self.store + .delete( + &Path::from_url_path(path) + .map_err(|_| vortex_err!("invalid path for url {path}"))?, + ) + .await?; + Ok(()) + } } diff --git a/vortex-jni/Cargo.toml b/vortex-jni/Cargo.toml index d6901514b49..66dd5f6254b 100644 --- a/vortex-jni/Cargo.toml +++ b/vortex-jni/Cargo.toml @@ -18,22 +18,19 @@ categories = { workspace = true } [dependencies] arrow-array = { workspace = true, features = ["ffi"] } -arrow-ipc = { workspace = true } arrow-schema = { workspace = true } futures = { workspace = true } -jni = "0.21.1" +jni = { workspace = true } object_store = { workspace = true, features = ["aws", "azure", "gcp"] } parking_lot = { workspace = true } -prost = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread"] } tracing = { workspace = true, features = ["std", "log"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true } -vortex = { workspace = true, features = ["object_store", "files", "tokio"] } +vortex = { workspace = true, features = ["object_store", "files"] } [dev-dependencies] -jni = { version = "0.21.1", features = ["invocation"] } +jni = { workspace = true, features = ["invocation"] } [lib] crate-type = ["staticlib", "cdylib"] diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs deleted file mode 100644 index 18af64bd2c8..00000000000 --- a/vortex-jni/src/array.rs +++ /dev/null @@ -1,525 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::sync::Arc; - -use arrow_array::ffi::FFI_ArrowArray; -use arrow_array::ffi::FFI_ArrowSchema; -use arrow_schema::DataType; -use arrow_schema::FieldRef; -use arrow_schema::Fields; -use jni::JNIEnv; -use jni::objects::JClass; -use jni::objects::JIntArray; -use jni::objects::JLongArray; -use jni::objects::JObject; -use jni::objects::JValue; -use jni::sys::JNI_FALSE; -use jni::sys::JNI_TRUE; -use jni::sys::jboolean; -use jni::sys::jbyte; -use jni::sys::jbyteArray; -use jni::sys::jdouble; -use jni::sys::jfloat; -use jni::sys::jint; -use jni::sys::jlong; -use jni::sys::jobject; -use jni::sys::jshort; -use jni::sys::jstring; -use vortex::array::ArrayRef; -use vortex::array::ArrayView; -use vortex::array::LEGACY_SESSION; -use vortex::array::VortexSessionExecute; -use vortex::array::arrays::ExtensionArray; -use vortex::array::arrays::StructArray; -use vortex::array::arrays::VarBin; -use vortex::array::arrays::VarBinView; -use vortex::array::arrays::extension::ExtensionArrayExt; -use vortex::array::arrays::struct_::StructArrayExt; -use vortex::array::arrays::varbin::VarBinArrayExt; -use vortex::array::arrow::ArrowArrayExecutor; -use vortex::dtype::DType; -use vortex::dtype::i256; -use vortex::error::VortexError; -use vortex::error::VortexExpect; -use vortex::error::vortex_err; -use vortex::scalar::DecimalValue; - -use crate::SESSION; -use crate::errors::JNIError; -use crate::errors::try_or_throw; - -pub struct NativeArray { - inner: ArrayRef, - is_extension: bool, -} - -impl NativeArray { - pub fn new(array_ref: ArrayRef) -> Box { - Box::new(NativeArray { - is_extension: array_ref.dtype().is_extension(), - inner: array_ref, - }) - } - - pub fn into_raw(self: Box) -> jlong { - Box::into_raw(self) as jlong - } - - /// Reconstruct a boxed `NativeArray` from a raw heap pointer. - pub unsafe fn from_raw(pointer: jlong) -> Box { - // SAFETY: caller must ensure that the pointer is valid and points to a `NativeArray`. - unsafe { Box::from_raw(pointer as *mut NativeArray) } - } - - #[expect( - clippy::expect_used, - reason = "JNI contract guarantees non-null pointer" - )] - pub unsafe fn from_ptr<'a>(pointer: jlong) -> &'a Self { - unsafe { - (pointer as *const NativeArray) - .as_ref() - .expect("Pointer should never be null") - } - } -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_free( - _env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) { - drop(unsafe { NativeArray::from_raw(array_ptr) }); -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_nbytes( - _env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) -> jlong { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - array_ref.inner.nbytes() as jlong -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_exportToArrow<'local>( - mut env: JNIEnv<'local>, - _class: JClass, - array_ptr: jlong, - arrow_schema_ptr: JLongArray<'local>, - arrow_array_ptr: JLongArray<'local>, -) { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - - try_or_throw(&mut env, |env| { - let preferred_arrow_type = array_ref.inner.dtype().to_arrow_dtype()?; - let viewless_arrow_type = data_type_no_views(preferred_arrow_type); - - let arrow_array = array_ref.inner.clone().execute_arrow( - Some(&viewless_arrow_type), - &mut LEGACY_SESSION.create_execution_ctx(), - )?; - let (ffi_array, ffi_schema) = - arrow_array::ffi::to_ffi(&arrow_array.to_data()).map_err(VortexError::from)?; - - let ffi_schema_ptr = Box::into_raw(Box::new(ffi_schema)); - let ffi_array_ptr = Box::into_raw(Box::new(ffi_array)); - - // Return native Arrow FFI pointers to caller. - env.set_long_array_region(arrow_schema_ptr, 0, &[ffi_schema_ptr as jlong])?; - env.set_long_array_region(arrow_array_ptr, 0, &[ffi_array_ptr as jlong])?; - Ok(()) - }); -} - -/// Visit the potentially nested DataType, replacing all instances of Utf8View and BinaryView -/// with non-Viewable equivalents. This is necessary because Spark and Iceberg do not support -/// Utf8View. -fn data_type_no_views(data_type: DataType) -> DataType { - match data_type { - DataType::BinaryView => DataType::Binary, - DataType::Utf8View => DataType::Utf8, - // List - DataType::List(inner) | DataType::ListView(inner) => { - let new_inner = (*inner) - .clone() - .with_data_type(data_type_no_views(inner.data_type().clone())); - DataType::List(FieldRef::new(new_inner)) - } - // LargeList - DataType::LargeList(inner) | DataType::LargeListView(inner) => { - let new_inner = (*inner) - .clone() - .with_data_type(data_type_no_views(inner.data_type().clone())); - DataType::LargeList(FieldRef::new(new_inner)) - } - DataType::Struct(fields) => { - let viewless_fields: Vec = fields - .iter() - .map(|field_ref| { - let field = (*Arc::clone(field_ref)).clone(); - let data_type = field.data_type().clone(); - let field = field.with_data_type(data_type_no_views(data_type)); - FieldRef::new(field) - }) - .collect(); - DataType::Struct(Fields::from(viewless_fields)) - } - DataType::Decimal128(precision, scale) => DataType::Decimal128(precision, scale), - DataType::Decimal256(precision, scale) => DataType::Decimal256(precision, scale), - DataType::FixedSizeList(inner, size) => { - let new_inner = (*inner) - .clone() - .with_data_type(data_type_no_views(inner.data_type().clone())); - DataType::FixedSizeList(FieldRef::new(new_inner), size) - } - DataType::Union(..) => unreachable!("Vortex never returns Union"), - DataType::Dictionary(..) => unreachable!("Vortex never returns Dictionary"), - DataType::Map(..) => unreachable!("Vortex never returns Map"), - DataType::RunEndEncoded(..) => unreachable!("Vortex never returns RunEndEncoded"), - // The non-nested non-view types stay the same. - dt => dt, - } -} - -/// Drop the native memory holding an Arrow FFI Schema behind the pointer. -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_dropArrowSchema( - _env: JNIEnv, - _class: JClass, - schema_ptr: jlong, -) { - drop(unsafe { Box::from_raw(schema_ptr as *mut FFI_ArrowSchema) }); -} - -/// Drop FFI_ArrowArray behind the pointer. -/// -/// Note that this doesn't not free the memory backing the arrow data buffers, those -/// are still backed by Vortex Buffers. -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_dropArrowArray( - _env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) { - drop(unsafe { Box::from_raw(array_ptr as *mut FFI_ArrowArray) }); -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getLen( - _env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) -> jlong { - unsafe { NativeArray::from_ptr(array_ptr) }.inner.len() as jlong -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getDataType( - _env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) -> jlong { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - let dtype_ptr = array_ref.inner.dtype(); - // Return a pointer to the DType. - (dtype_ptr as *const DType).addr() as jlong -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getField( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - index: jint, -) -> jlong { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - - try_or_throw(&mut env, |_| { - let mut ctx = SESSION.create_execution_ctx(); - let struct_array = array_ref.inner.clone().execute::(&mut ctx)?; - let idx = index as usize; - if idx >= struct_array.struct_fields().nfields() { - return Err(vortex_err!("Field index out of bounds").into()); - } - let field = struct_array.unmasked_field(idx).clone(); - Ok(NativeArray::new(field).into_raw()) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_slice( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - start: jint, - end: jint, -) -> jlong { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - - try_or_throw(&mut env, |_| { - let sliced_array = array_ref.inner.slice(start as usize..end as usize)?; - Ok(NativeArray::new(sliced_array).into_raw()) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNull( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - index: jint, -) -> jboolean { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |_| { - let mut ctx = SESSION.create_execution_ctx(); - let is_null = array_ref.inner.is_invalid(index as usize, &mut ctx)?; - if is_null { Ok(JNI_TRUE) } else { Ok(JNI_FALSE) } - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNullCount( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, -) -> jint { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |_| { - let mut ctx = SESSION.create_execution_ctx(); - let count = array_ref.inner.invalid_count(&mut ctx)?; - Ok(jint::try_from(count).unwrap_or(-1)) - }) -} - -macro_rules! get_primitive { - ($name:ident, $native:ty, $jtype:ty) => { - #[unsafe(no_mangle)] - pub extern "system" fn $name( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - index: jint, - ) -> $jtype { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |_| { - let mut ctx = SESSION.create_execution_ctx(); - let scalar_value = if array_ref.is_extension { - let ext = array_ref - .inner - .clone() - .execute::(&mut ctx)?; - ext.storage_array() - .execute_scalar(index as usize, &mut ctx)? - } else { - array_ref.inner.execute_scalar(index as usize, &mut ctx)? - }; - - Ok(scalar_value - .as_primitive() - .as_::<$native>() - .unwrap_or_default()) - }) - } - }; -} - -get_primitive!(Java_dev_vortex_jni_NativeArrayMethods_getByte, i8, jbyte); -get_primitive!(Java_dev_vortex_jni_NativeArrayMethods_getShort, i16, jshort); -get_primitive!(Java_dev_vortex_jni_NativeArrayMethods_getInt, i32, jint); -get_primitive!(Java_dev_vortex_jni_NativeArrayMethods_getLong, i64, jlong); -get_primitive!(Java_dev_vortex_jni_NativeArrayMethods_getFloat, f32, jfloat); -get_primitive!( - Java_dev_vortex_jni_NativeArrayMethods_getDouble, - f64, - jdouble -); - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBigDecimal( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - index: jint, -) -> jobject { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |env| { - let mut ctx = SESSION.create_execution_ctx(); - let scalar_value = if array_ref.is_extension { - let ext = array_ref - .inner - .clone() - .execute::(&mut ctx)?; - ext.storage_array() - .execute_scalar(index as usize, &mut ctx)? - } else { - array_ref.inner.execute_scalar(index as usize, &mut ctx)? - }; - - let decimal_scalar = scalar_value.as_decimal(); - let DType::Decimal(decimal_type, ..) = decimal_scalar.dtype() else { - return Err(vortex_err!("Expected Decimal type").into()); - }; - let scale = decimal_type.scale(); - if let Some(v) = decimal_scalar.decimal_value() { - match v { - DecimalValue::I8(v) => bigdecimal_i8(env, v, scale), - DecimalValue::I16(v) => bigdecimal_i16(env, v, scale), - DecimalValue::I32(v) => bigdecimal_i32(env, v, scale), - DecimalValue::I64(v) => bigdecimal_i64(env, v, scale), - DecimalValue::I128(v) => bigdecimal_i128(env, v, scale), - DecimalValue::I256(v) => bigdecimal_i256(env, v, scale), - } - } else { - Ok(JObject::null().into_raw()) - } - }) -} - -static BIGDECIMAL_CLASS: &str = "java/math/BigDecimal"; -static BIGINT_CLASS: &str = "java/math/BigInteger"; - -macro_rules! bigdecimal_from_bytes { - ($typ:ty, $name:ident) => { - fn $name(env: &mut JNIEnv, value: $typ, scale: i8) -> Result { - // NOTE: BigInteger constructor expects big-endian bytes. - let be_bytes = value.to_be_bytes(); - let bytearray = env.byte_array_from_slice(&be_bytes)?; - let bigint = env.new_object(BIGINT_CLASS, "([B)V", &[JValue::from(&bytearray)])?; - - // Create the BigDecimal from a BigInteger + scale - let bigdecimal = env.new_object( - BIGDECIMAL_CLASS, - "(Ljava/math/BigInteger;I)V", - &[JValue::from(&bigint), JValue::from(scale as jint)], - )?; - - Ok(bigdecimal.into_raw()) - } - }; -} - -bigdecimal_from_bytes!(i8, bigdecimal_i8); -bigdecimal_from_bytes!(i16, bigdecimal_i16); -bigdecimal_from_bytes!(i32, bigdecimal_i32); -bigdecimal_from_bytes!(i64, bigdecimal_i64); -bigdecimal_from_bytes!(i128, bigdecimal_i128); -bigdecimal_from_bytes!(i256, bigdecimal_i256); - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBool( - mut env: JNIEnv, - _class: JClass, - array_ptr: jlong, - index: jint, -) -> jboolean { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |_| { - let mut ctx = SESSION.create_execution_ctx(); - let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; - match value.as_bool().value() { - None => Ok(JNI_FALSE), - Some(b) => { - if b { - Ok(JNI_TRUE) - } else { - Ok(JNI_FALSE) - } - } - } - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getUTF8<'local>( - mut env: JNIEnv<'local>, - _class: JClass<'local>, - array_ptr: jlong, - index: jint, -) -> jstring { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |env| { - let mut ctx = SESSION.create_execution_ctx(); - let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; - match value.as_utf8().value() { - None => Ok(JObject::null().into_raw()), - Some(buf_str) => Ok(env.new_string(buf_str.as_str())?.into_raw()), - } - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getUTF8_1ptr_1len<'local>( - mut env: JNIEnv<'local>, - _class: JClass<'local>, - array_ptr: jlong, - index: jint, - out_ptr: JLongArray<'local>, - out_len: JIntArray<'local>, -) { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - - try_or_throw(&mut env, |env| { - if !array_ref.inner.dtype().is_utf8() { - throw_runtime!("getUTF8_ptr_len expected UTF8 array"); - } - - if let Some(varbin) = array_ref.inner.as_opt::() { - let (ptr, len) = get_ptr_len_varbin(index, varbin); - env.set_long_array_region(&out_ptr, 0, &[ptr as jlong])?; - env.set_int_array_region(&out_len, 0, &[len as jint])?; - } else if let Some(varbinview) = array_ref.inner.as_opt::() { - let (ptr, len) = get_ptr_len_view(index, varbinview); - env.set_long_array_region(&out_ptr, 0, &[ptr as jlong])?; - env.set_int_array_region(&out_len, 0, &[len as jint])?; - } else { - throw_runtime!("getUTF8_ptr_len expected VarBin or VarBinView"); - } - Ok(()) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getBinary<'local>( - mut env: JNIEnv<'local>, - _class: JClass<'local>, - array_ptr: jlong, - index: jint, -) -> jbyteArray { - let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; - try_or_throw(&mut env, |env| { - let mut ctx = SESSION.create_execution_ctx(); - let value = array_ref.inner.execute_scalar(index as usize, &mut ctx)?; - match value.as_binary().value() { - None => Ok(JObject::null().into_raw()), - Some(buf) => Ok(env.byte_array_from_slice(buf.as_slice())?.into_raw()), - } - }) -} - -/// Get a raw pointer + len to pass back to Java to avoid copying across the boundary. -/// -/// Panics if the index is out of bounds. -fn get_ptr_len_varbin(index: jint, array: ArrayView) -> (*const u8, u32) { - // TODO: propagate this error up instead of expecting - let bytes = array.bytes_at(usize::try_from(index).vortex_expect("index must fit in usize")); - ( - bytes.as_ptr(), - // TODO: propagate this error up instead of expecting - u32::try_from(bytes.len()).vortex_expect("string length must fit in u32"), - ) -} - -/// Get a raw pointer + len to pass back to Java to avoid copying across the boundary. -fn get_ptr_len_view(index: jint, array: ArrayView) -> (*const u8, u32) { - // TODO: propagate this error up instead of expecting - let bytes = array.bytes_at(usize::try_from(index).vortex_expect("index must fit in usize")); - ( - bytes.as_ptr(), - // TODO: propagate this error up instead of expecting - u32::try_from(bytes.len()).vortex_expect("string length must fit in u32"), - ) -} diff --git a/vortex-jni/src/array_iter.rs b/vortex-jni/src/array_iter.rs deleted file mode 100644 index 0306fed8950..00000000000 --- a/vortex-jni/src/array_iter.rs +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use jni::JNIEnv; -use jni::objects::JClass; -use jni::sys::jlong; -use vortex::array::iter::ArrayIterator; -use vortex::dtype::DType; - -use crate::array::NativeArray; -use crate::errors::try_or_throw; - -/// Blocking JNI bridge to a Vortex [`ArrayIterator`]. -pub struct NativeArrayIterator { - inner: Option>, -} - -impl NativeArrayIterator { - pub fn new(stream: Box) -> Box { - Box::new(Self { - inner: Some(stream), - }) - } - - pub fn into_raw(self: Box) -> jlong { - Box::into_raw(self) as jlong - } - - #[expect( - clippy::expect_used, - reason = "JNI contract guarantees non-null pointer" - )] - pub unsafe fn from_ptr<'a>(pointer: jlong) -> &'a Self { - unsafe { - (pointer as *const NativeArrayIterator) - .as_ref() - .expect("Pointer should never be null") - } - } - - #[expect( - clippy::expect_used, - reason = "JNI contract guarantees non-null pointer" - )] - pub unsafe fn from_ptr_mut<'a>(pointer: jlong) -> &'a mut Self { - unsafe { - (pointer as *mut NativeArrayIterator) - .as_mut() - .expect("Pointer should never be null") - } - } - - pub unsafe fn from_raw(pointer: jlong) -> Box { - unsafe { Box::from_raw(pointer as *mut NativeArrayIterator) } - } -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayIteratorMethods_free( - _env: JNIEnv, - _class: JClass, - pointer: jlong, -) { - drop(unsafe { NativeArrayIterator::from_raw(pointer) }); -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayIteratorMethods_take( - mut env: JNIEnv, - _class: JClass, - pointer: jlong, -) -> jlong { - let iter = unsafe { NativeArrayIterator::from_ptr_mut(pointer) }; - - try_or_throw(&mut env, |_| { - if let Some(mut inner) = iter.inner.take() { - let result = match inner.next() { - Some(result) => { - match result { - Ok(array_ref) => { - // Successfully got the next array - let ptr = NativeArray::new(array_ref).into_raw(); - Ok(ptr) - } - Err(e) => { - // Error occurred, but we still need to restore the iterator - Err(e.into()) - } - } - } - None => { - // Iterator is exhausted - Ok(-1) - } - }; - - // Always restore the iterator, even on error - iter.inner = Some(inner); - result - } else { - throw_runtime!("attempted to take() on a closed ArrayIter"); - } - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeArrayIteratorMethods_getDType( - mut env: JNIEnv, - _class: JClass, - pointer: jlong, -) -> jlong { - let iter = unsafe { NativeArrayIterator::from_ptr(pointer) }; - - try_or_throw(&mut env, |_| { - if let Some(ref inner) = iter.inner { - let dtype = inner.dtype(); - Ok(dtype as *const DType as jlong) - } else { - throw_runtime!("NativeArrayMethods.getDType: closed stream"); - } - }) -} diff --git a/vortex-jni/src/data_source.rs b/vortex-jni/src/data_source.rs new file mode 100644 index 00000000000..d62b583a4a9 --- /dev/null +++ b/vortex-jni/src/data_source.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! JNI bindings for [`vortex::scan::DataSource`] (see the equivalent types in +//! `vortex-ffi/src/data_source.rs`). +//! +//! Glob handling mirrors `vortex-duckdb`'s `VortexMultiFileScan`: +//! * full URLs (`s3://...`, `file:///...`) are used as-is, +//! * bare file paths are made absolute and have `.`/`..` components normalized, +//! * filesystems are cached per base URL so repeated globs against the same bucket share +//! a single client. + +use std::path::Component; +use std::path::Path; +use std::path::PathBuf; +use std::path::absolute; +use std::sync::Arc; + +use jni::EnvUnowned; +use jni::objects::JClass; +use jni::objects::JLongArray; +use jni::objects::JObject; +use jni::objects::JObjectArray; +use jni::objects::JString; +use jni::sys::jlong; +use url::Url; +use vortex::error::VortexResult; +use vortex::error::vortex_err; +use vortex::expr::stats::Precision; +use vortex::file::multi::MultiFileDataSource; +use vortex::io::filesystem::FileSystemRef; +use vortex::io::runtime::BlockingRuntime; +use vortex::io::session::RuntimeSessionExt; +use vortex::scan::DataSourceRef; +use vortex::utils::aliases::hash_map::HashMap; + +use crate::RUNTIME; +use crate::dtype::export_dtype_to_arrow; +use crate::errors::try_or_throw; +use crate::file::extract_properties; +use crate::object_store::object_store_fs; +use crate::session::session_ref; + +/// Wraps an `Arc` behind a single pointer. +pub(crate) struct NativeDataSource { + inner: DataSourceRef, +} + +impl NativeDataSource { + fn into_raw(self: Box) -> jlong { + Box::into_raw(self) as jlong + } + + /// SAFETY: pointer must have been returned from [`Self::into_raw`]. + pub(crate) unsafe fn from_ptr<'a>(ptr: jlong) -> &'a Self { + debug_assert!(ptr != 0, "null data source pointer"); + unsafe { &*(ptr as *const Self) } + } + + pub(crate) fn inner(&self) -> &DataSourceRef { + &self.inner + } +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_open( + mut env: EnvUnowned, + _class: JClass, + session_ptr: jlong, + uris: JObjectArray, + options: JObject, +) -> jlong { + try_or_throw(&mut env, |env| { + let session = unsafe { session_ref(session_ptr) }; + let properties = extract_properties(env, &options)?; + + let mut glob_strings = Vec::new(); + let uri_count = uris.len(env)?; + for idx in 0..uri_count { + let uri = uris.get_element(env, idx)?; + let uri = env.cast_local::(uri)?; + let uri: String = uri.try_to_string(env)?; + let uri = uri.trim(); + if !uri.is_empty() { + glob_strings.push(uri.to_owned()); + } + } + if glob_strings.is_empty() { + return Err(vortex_err!("no paths provided").into()); + } + + let glob_urls: Vec = glob_strings + .iter() + .map(|g| parse_glob_url(g.as_str())) + .collect::>()?; + + let mut fs_cache: HashMap = HashMap::new(); + for glob_url in &glob_urls { + let base = base_url(glob_url); + if !fs_cache.contains_key(&base) { + let fs = object_store_fs(glob_url, &properties, session.handle())?; + fs_cache.insert(base, fs); + } + } + + let mut builder = MultiFileDataSource::new(session.clone()); + for glob_url in &glob_urls { + let base = base_url(glob_url); + let fs = fs_cache + .get(&base) + .cloned() + .unwrap_or_else(|| unreachable!("fs cached for every base url")); + builder = builder.with_glob(glob_url.path(), Some(fs)); + } + + let inner = RUNTIME + .block_on(builder.build()) + .map(|ds| Arc::new(ds) as DataSourceRef)?; + Ok(Box::new(NativeDataSource { inner }).into_raw()) + }) +} + +/// Parse a glob string into a [`Url`]. Accepts full URLs and bare (relative or absolute) +/// file paths — see the module docs for details. +fn parse_glob_url(glob: &str) -> VortexResult { + // `Url::parse` accepts Windows absolute paths like `C:\foo` as a URL with a + // single-letter scheme (`c`). No real URL scheme is one character, so treat any + // single-letter scheme as a filesystem path instead. + if let Ok(url) = Url::parse(glob) + && url.scheme().len() > 1 + { + return Ok(url); + } + let path = + absolute(Path::new(glob)).map_err(|e| vortex_err!("failed to absolutize {glob}: {e}"))?; + let path = normalize_path(path); + Url::from_file_path(path).map_err(|_| vortex_err!("neither URL nor path: {glob}")) +} + +/// Normalize `.` and `..` without touching the filesystem. +fn normalize_path(path: PathBuf) -> PathBuf { + let mut out = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + c => out.push(c), + } + } + out +} + +/// URL with the path cleared, used as a cache key for filesystem reuse. +fn base_url(url: &Url) -> Url { + let mut base = url.clone(); + base.set_path(""); + base +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_free( + _env: EnvUnowned, + _class: JClass, + pointer: jlong, +) { + if pointer == 0 { + return; + } + drop(unsafe { Box::from_raw(pointer as *mut NativeDataSource) }); +} + +/// Export the data source's schema into the Arrow C Data Interface schema struct at +/// `schema_addr`. +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_arrowSchema( + mut env: EnvUnowned, + _class: JClass, + pointer: jlong, + schema_addr: jlong, +) { + try_or_throw(&mut env, |_| { + if schema_addr == 0 { + throw_runtime!("null arrow schema address"); + } + let ds = unsafe { NativeDataSource::from_ptr(pointer) }; + export_dtype_to_arrow(ds.inner.dtype(), schema_addr)?; + Ok(()) + }); +} + +/// Write the row count into the two-slot jlong pair `out`: +/// `out[0]` receives the row count (0 when unknown), `out[1]` the cardinality (0=unknown, 1=estimate, 2=exact). +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_rowCount( + mut env: EnvUnowned, + _class: JClass, + pointer: jlong, + out: JLongArray, +) { + try_or_throw(&mut env, |env| { + let ds = unsafe { NativeDataSource::from_ptr(pointer) }; + let (rows, cardinality) = match ds.inner.row_count() { + Some(Precision::Exact(r)) => (r as jlong, 2), + Some(Precision::Inexact(r)) => (r as jlong, 1), + None => (0, 0), + }; + out.set_region(env, 0, &[rows, cardinality])?; + Ok(()) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_glob_url_full_url() { + let url = parse_glob_url("s3://bucket/prefix/*.vortex").unwrap(); + assert_eq!(url.scheme(), "s3"); + assert_eq!(url.host_str(), Some("bucket")); + assert_eq!(url.path(), "/prefix/*.vortex"); + } + + #[test] + fn test_parse_glob_url_absolute_path() { + // Use a drive-prefixed input on Windows so `absolute()` doesn't inject the cwd drive + // and the expected URL path is predictable. + #[cfg(unix)] + let (input, expected_path) = ("/tmp/data/*.vortex", "/tmp/data/*.vortex"); + #[cfg(windows)] + let (input, expected_path) = (r"C:\tmp\data\*.vortex", "/C:/tmp/data/*.vortex"); + let url = parse_glob_url(input).unwrap(); + assert_eq!(url.scheme(), "file"); + assert_eq!(url.path(), expected_path); + } + + #[test] + fn test_parse_glob_url_normalizes_dots() { + #[cfg(unix)] + let (input, expected_path) = ("/a/b/../c/./d", "/a/c/d"); + #[cfg(windows)] + let (input, expected_path) = (r"C:\a\b\..\c\.\d", "/C:/a/c/d"); + let url = parse_glob_url(input).unwrap(); + assert_eq!(url.path(), expected_path); + } + + #[test] + fn test_parse_glob_url_single_letter_scheme_is_path() { + // Regression: `Url::parse("C:\\tmp")` succeeds with scheme="c"; the function must + // treat that as a filesystem path, not a URL. Exercised on all platforms because + // the check lives in `parse_glob_url`, not in an OS-specific branch. + let url = parse_glob_url(r"C:\tmp\data\*.vortex").unwrap(); + assert_eq!(url.scheme(), "file"); + assert_ne!(url.scheme(), "c"); + } + + #[test] + fn test_base_url_strips_path() { + let url = Url::parse("s3://bucket/a/b/c").unwrap(); + let base = base_url(&url); + assert_eq!(base.scheme(), "s3"); + assert_eq!(base.host_str(), Some("bucket")); + assert_eq!(base.path(), ""); + } +} diff --git a/vortex-jni/src/dtype.rs b/vortex-jni/src/dtype.rs index 3c8666127ae..f748135ade1 100644 --- a/vortex-jni/src/dtype.rs +++ b/vortex-jni/src/dtype.rs @@ -1,658 +1,78 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::sync::Arc; +//! Bridge functions between a Vortex [`DType`] and Arrow's C Data Interface +//! [`FFI_ArrowSchema`]. Java receives DType information exclusively as Arrow schema. -use jni::JNIEnv; -use jni::objects::JClass; -use jni::objects::JLongArray; -use jni::objects::JObject; -use jni::objects::JObjectArray; -use jni::objects::JString; -use jni::objects::JValue; -use jni::sys::JNI_FALSE; -use jni::sys::JNI_TRUE; -use jni::sys::jboolean; -use jni::sys::jbyte; -use jni::sys::jint; -use jni::sys::jlong; -use jni::sys::jobject; -use jni::sys::jstring; -use vortex::dtype::DType; -use vortex::dtype::DecimalDType; -use vortex::dtype::Nullability; -use vortex::dtype::PType; -use vortex::dtype::StructFields; -use vortex::error::vortex_err; -use vortex::extension::datetime::AnyTemporal; -use vortex::extension::datetime::Date; -use vortex::extension::datetime::Time; -use vortex::extension::datetime::TimeUnit; -use vortex::extension::datetime::Timestamp; - -use crate::errors::JNIError; -use crate::errors::try_or_throw; - -pub const DTYPE_NULL: jbyte = 0; -pub const DTYPE_BOOL: jbyte = 1; -pub const DTYPE_PRIMITIVE_U8: jbyte = 2; -pub const DTYPE_PRIMITIVE_U16: jbyte = 3; -pub const DTYPE_PRIMITIVE_U32: jbyte = 4; -pub const DTYPE_PRIMITIVE_U64: jbyte = 5; -pub const DTYPE_PRIMITIVE_I8: jbyte = 6; -pub const DTYPE_PRIMITIVE_I16: jbyte = 7; -pub const DTYPE_PRIMITIVE_I32: jbyte = 8; -pub const DTYPE_PRIMITIVE_I64: jbyte = 9; -pub const DTYPE_PRIMITIVE_F16: jbyte = 10; -pub const DTYPE_PRIMITIVE_F32: jbyte = 11; -pub const DTYPE_PRIMITIVE_F64: jbyte = 12; -pub const DTYPE_UTF8: jbyte = 13; -pub const DTYPE_BINARY: jbyte = 14; -pub const DTYPE_STRUCT: jbyte = 15; -pub const DTYPE_LIST: jbyte = 16; -pub const DTYPE_EXTENSION: jbyte = 17; -pub const DTYPE_DECIMAL: jbyte = 18; -pub const DTYPE_FIXED_SIZE_LIST: jbyte = 19; - -static LONG_CLASS: &str = "java/lang/Long"; - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_free( - _env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) { - // SAFETY: caller must ensure that the pointer is valid and points to a `DType`. - drop(unsafe { Box::from_raw(dtype_ptr as *mut DType) }); -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_getVariant( - _env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jbyte { - // SAFETY: caller must ensure that the pointer is valid and points to a `DType`. - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - match dtype { - DType::Null => DTYPE_NULL, - DType::Bool(_) => DTYPE_BOOL, - DType::Primitive(ptype, _) => match ptype { - PType::U8 => DTYPE_PRIMITIVE_U8, - PType::U16 => DTYPE_PRIMITIVE_U16, - PType::U32 => DTYPE_PRIMITIVE_U32, - PType::U64 => DTYPE_PRIMITIVE_U64, - PType::I8 => DTYPE_PRIMITIVE_I8, - PType::I16 => DTYPE_PRIMITIVE_I16, - PType::I32 => DTYPE_PRIMITIVE_I32, - PType::I64 => DTYPE_PRIMITIVE_I64, - PType::F16 => DTYPE_PRIMITIVE_F16, - PType::F32 => DTYPE_PRIMITIVE_F32, - PType::F64 => DTYPE_PRIMITIVE_F64, - }, - DType::Decimal(..) => DTYPE_DECIMAL, - DType::Utf8(_) => DTYPE_UTF8, - DType::Binary(_) => DTYPE_BINARY, - DType::Struct(..) => DTYPE_STRUCT, - DType::List(..) => DTYPE_LIST, - DType::FixedSizeList(..) => DTYPE_FIXED_SIZE_LIST, - DType::Extension(_) => DTYPE_EXTENSION, - DType::Variant(_) => unimplemented!("Variant DType is not supported in JNI yet"), - } -} +use std::ptr; -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_isNullable( - _env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jboolean { - // SAFETY: caller must ensure that the pointer is valid and points to a `DType`. - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - if dtype.is_nullable() { - JNI_TRUE - } else { - JNI_FALSE +use arrow_array::ffi::FFI_ArrowSchema; +use arrow_schema::DataType; +use arrow_schema::FieldRef; +use arrow_schema::Fields; +use vortex::dtype::DType; +use vortex::dtype::arrow::FromArrowType; +use vortex::error::VortexResult; + +/// Export a Vortex [`DType`] to the Arrow C Data Interface struct at `schema_addr`. Views +/// (Utf8View/BinaryView) are downgraded to regular Utf8/Binary so Spark and other consumers +/// without view support can read them. +pub(crate) fn export_dtype_to_arrow(dtype: &DType, schema_addr: i64) -> VortexResult<()> { + let arrow_schema = dtype.to_arrow_schema()?; + let viewless = strip_views(DataType::Struct(arrow_schema.fields().clone())); + let fields = match viewless { + DataType::Struct(fields) => fields, + _ => unreachable!("Vortex DType always exports as a struct"), + }; + let schema = arrow_schema::Schema::new(fields); + let ffi_schema = FFI_ArrowSchema::try_from(&schema)?; + unsafe { + ptr::write(schema_addr as *mut FFI_ArrowSchema, ffi_schema); } -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_getFieldNames( - mut env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jobject { - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - - try_or_throw(&mut env, |env| { - let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; - let field_names = env.get_list(&array_list)?; - let Some(struct_dtype) = dtype.as_struct_fields_opt() else { - throw_runtime!("DType should be STRUCT, was {dtype}"); - }; - - for name in struct_dtype.names().iter() { - let field = env.new_string(name)?; - field_names.add(env, field.as_ref())?; + Ok(()) +} + +/// Replace view-based Arrow types with their non-view counterparts throughout the tree. +pub(crate) fn strip_views(data_type: DataType) -> DataType { + match data_type { + DataType::BinaryView => DataType::Binary, + DataType::Utf8View => DataType::Utf8, + DataType::List(inner) | DataType::ListView(inner) => { + let new_inner = (*inner) + .clone() + .with_data_type(strip_views(inner.data_type().clone())); + DataType::List(FieldRef::new(new_inner)) } - - Ok::(array_list.into_raw()) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_getFieldTypes( - mut env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jobject { - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - - try_or_throw(&mut env, |env| { - let array_list = env - .new_object("java/util/ArrayList", "()V", &[]) - .map_err(|e| JNIError::Vortex(vortex_err!("failure constructing ArrayList: {e}")))?; - let field_types = env.get_list(&array_list)?; - let Some(struct_dtype) = dtype.as_struct_fields_opt() else { - throw_runtime!("DType should be STRUCT, was {dtype}"); - }; - - for field_dtype in struct_dtype.fields() { - let ptr: *mut DType = Box::into_raw(Box::new(field_dtype)); - let boxed = env - .call_static_method( - LONG_CLASS, - "valueOf", - "(J)Ljava/lang/Long;", - &[JValue::Long(ptr.addr() as jlong)], - )? - .l()?; - field_types.add(env, &boxed)?; + DataType::LargeList(inner) | DataType::LargeListView(inner) => { + let new_inner = (*inner) + .clone() + .with_data_type(strip_views(inner.data_type().clone())); + DataType::LargeList(FieldRef::new(new_inner)) } - - Ok(array_list.into_raw()) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_getElementType( - mut env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jlong { - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - - try_or_throw(&mut env, |_| { - let element_type = dtype - .as_list_element_opt() - .or_else(|| dtype.as_fixed_size_list_element_opt()); - let Some(element_type) = element_type else { - throw_runtime!("DType should be LIST or FIXED_SIZE_LIST, was {dtype}"); - }; - - Ok(element_type.as_ref() as *const DType as jlong) - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_isDate( - mut env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jboolean { - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - - try_or_throw(&mut env, |_| { - let DType::Extension(ext_dtype) = dtype else { - throw_runtime!("DType should be an EXTENSION, was {dtype}"); - }; - - if ext_dtype.is::() { - Ok(JNI_TRUE) - } else { - Ok(JNI_FALSE) + DataType::Struct(fields) => { + let viewless_fields: Vec = fields + .iter() + .map(|field_ref| { + let field = (**field_ref).clone(); + let data_type = field.data_type().clone(); + FieldRef::new(field.with_data_type(strip_views(data_type))) + }) + .collect(); + DataType::Struct(Fields::from(viewless_fields)) } - }) -} - -#[unsafe(no_mangle)] -pub extern "system" fn Java_dev_vortex_jni_NativeDTypeMethods_isTime( - mut env: JNIEnv, - _class: JClass, - dtype_ptr: jlong, -) -> jboolean { - let dtype = unsafe { &*(dtype_ptr as *const DType) }; - - try_or_throw(&mut env, |_| { - let DType::Extension(ext_dtype) = dtype else { - throw_runtime!("DType should be an EXTENSION, was {dtype}"); - }; - - if ext_dtype.is:: { row_range: Option>, ) -> VortexResult> + Send + 'static + use> { use futures::StreamExt; - let num_workers = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); + let num_workers = get_available_parallelism().unwrap_or(1); let concurrency = self.concurrency * num_workers; let handle = self.session.handle(); diff --git a/vortex-layout/src/scan/scan_builder.rs b/vortex-layout/src/scan/scan_builder.rs index d6a372ca19e..bdf5d0bfb11 100644 --- a/vortex-layout/src/scan/scan_builder.rs +++ b/vortex-layout/src/scan/scan_builder.rs @@ -38,6 +38,7 @@ use vortex_io::session::RuntimeSessionExt; use vortex_metrics::MetricsRegistry; use vortex_scan::selection::Selection; use vortex_session::VortexSession; +use vortex_utils::parallelism::get_available_parallelism; use crate::LayoutReader; use crate::LayoutReaderRef; @@ -367,9 +368,7 @@ impl Stream for LazyScanStream { LazyScanState::Builder(builder) => { let builder = builder.take().vortex_expect("polled after completion"); let ordered = builder.ordered; - let num_workers = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); + let num_workers = get_available_parallelism().unwrap_or(1); let concurrency = builder.concurrency * num_workers; let handle = builder.session.handle(); let task = handle.spawn_blocking(move || { diff --git a/vortex-utils/public-api.lock b/vortex-utils/public-api.lock index 185029d5627..e7d259551d1 100644 --- a/vortex-utils/public-api.lock +++ b/vortex-utils/public-api.lock @@ -83,3 +83,7 @@ impl vortex_uti pub fn I::reduce_balanced(self, combine: F) -> core::option::Option where Self::Item: core::clone::Clone, F: core::ops::function::Fn(Self::Item, Self::Item) -> Self::Item pub fn I::try_reduce_balanced(self, combine: F) -> core::result::Result, E> where Self::Item: core::clone::Clone, F: core::ops::function::Fn(Self::Item, Self::Item) -> core::result::Result + +pub mod vortex_utils::parallelism + +pub fn vortex_utils::parallelism::get_available_parallelism() -> core::option::Option diff --git a/vortex-utils/src/lib.rs b/vortex-utils/src/lib.rs index cc650072eab..31decd406ff 100644 --- a/vortex-utils/src/lib.rs +++ b/vortex-utils/src/lib.rs @@ -10,3 +10,4 @@ pub mod debug_with; #[cfg(feature = "dyn-traits")] pub mod dyn_traits; pub mod iter; +pub mod parallelism; diff --git a/vortex-utils/src/parallelism.rs b/vortex-utils/src/parallelism.rs new file mode 100644 index 00000000000..14b251e8bfb --- /dev/null +++ b/vortex-utils/src/parallelism.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Useful utilities for discovering the desired level of parallelism + +use std::sync::LazyLock; + +/// Estimates the degree of parallelism the program should use, caching the result after the first call. +/// +/// This is currently implemented using [`std::thread::available_parallelism`], but might change in the future. +/// +/// Returns `None` if the underlying functions fails. +pub fn get_available_parallelism() -> Option { + #[allow(clippy::disallowed_methods)] + static PARALLELISM: LazyLock> = + LazyLock::new(|| std::thread::available_parallelism().ok().map(|n| n.get())); + + *PARALLELISM +} From a83c9b33fe4ab57fc275783ef8de372f2e8a9bbc Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 24 Apr 2026 13:16:09 +0100 Subject: [PATCH 183/250] Try and improve the perf of natural file splits in DF (#7609) ## Summary Try and re-capture some of the performance we lost in #7591, only doing the extra work when its actually required. --------- Signed-off-by: Adam Gutglick --- vortex-datafusion/src/persistent/opener.rs | 66 +++++++++++++++------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index 0719b023881..acff2a22806 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -307,13 +307,7 @@ impl FileOpener for VortexOpener { } }; - let natural_split_ranges = natural_split_ranges_for_file( - natural_split_ranges.as_ref(), - &file.object_meta.location, - &layout_reader, - )?; - - let mut scan_builder = ScanBuilder::new(session.clone(), layout_reader); + let mut scan_builder = ScanBuilder::new(session.clone(), Arc::clone(&layout_reader)); if let Some(extensions) = file.extensions && let Some(vortex_plan) = extensions.downcast_ref::() @@ -328,16 +322,25 @@ impl FileOpener for VortexOpener { end: u64::try_from(file_range.end) .map_err(|_| exec_datafusion_err!("Vortex file range end is negative"))?, }; - - let Some(row_range) = split_aligned_row_range( - byte_range, - file.object_meta.size, - natural_split_ranges.as_ref(), - ) else { - return Ok(stream::empty().boxed()); - }; - - scan_builder = scan_builder.with_row_range(row_range); + if byte_range.start != 0 || byte_range.end != file.object_meta.size { + // Full-file scans already cover every natural split. Only translate the + // byte range back into row boundaries when DataFusion has trimmed the file. + let natural_split_ranges = natural_split_ranges_for_file( + natural_split_ranges.as_ref(), + &file.object_meta.location, + &layout_reader, + )?; + + let Some(row_range) = split_aligned_row_range( + byte_range, + file.object_meta.size, + natural_split_ranges.as_ref(), + ) else { + return Ok(stream::empty().boxed()); + }; + + scan_builder = scan_builder.with_row_range(row_range); + } } let filter = filter @@ -477,6 +480,8 @@ fn compute_natural_split_ranges(layout_reader: &dyn LayoutReader) -> DFResult, total_size: u64, @@ -491,10 +496,13 @@ fn split_aligned_row_range( return None; } - let mut owned_splits = split_ranges.iter().filter(|split_range| { - let midpoint_byte = split_midpoint_to_byte(split_range, row_count, total_size); - byte_range.contains(&midpoint_byte) - }); + let mut owned_splits = split_ranges + .iter() + .enumerate() + .filter_map(|(idx, split_range)| { + let assignment_byte = split_assignment_byte(idx, split_range, row_count, total_size); + byte_range.contains(&assignment_byte).then_some(split_range) + }); let first_split = owned_splits.next()?; let mut row_range = first_split.start..first_split.end; @@ -505,6 +513,21 @@ fn split_aligned_row_range( Some(row_range) } +fn split_assignment_byte( + idx: usize, + split_range: &Range, + row_count: u64, + total_size: u64, +) -> u64 { + if idx == 0 && split_range.start == 0 { + // Byte 0 is the only stable representative for the leading split. A midpoint can fall + // into the next DataFusion byte range and leave the first range with no rows to read. + 0 + } else { + split_midpoint_to_byte(split_range, row_count, total_size) + } +} + fn split_midpoint_to_byte(split_range: &Range, row_count: u64, total_size: u64) -> u64 { let midpoint_row = split_range.start + (split_range.end - split_range.start) / 2; let midpoint_byte = (u128::from(midpoint_row) * u128::from(total_size)) / u128::from(row_count); @@ -566,6 +589,7 @@ mod tests { #[case(3..7, 10, vec![0..2, 2..5, 5..10], Some(2..5))] #[case(1..8, 10, vec![0..1, 1..9, 9..10], Some(1..9))] #[case(1..4, 16, vec![0..1, 1..2, 2..3, 3..4], None)] + #[case(0..1, 10, vec![0..2, 2..10], Some(0..2))] fn test_split_aligned_row_range( #[case] byte_range: Range, #[case] total_size: u64, From 50531abf25549c47eb85bfe2344483d3a5311676 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 24 Apr 2026 14:11:39 +0100 Subject: [PATCH 184/250] feat: add a runner field to all runners (#7622) Adds a runner field to all benchmark runner which could be later used to diff runs from different runners --------- Signed-off-by: Joe Isaacs --- .github/workflows/sql-benchmarks.yml | 2 + bench-orchestrator/bench_orchestrator/cli.py | 5 +++ .../bench_orchestrator/runner/executor.py | 5 +++ benchmarks/datafusion-bench/src/main.rs | 4 ++ benchmarks/duckdb-bench/src/main.rs | 4 ++ benchmarks/lance-bench/src/main.rs | 4 ++ vortex-bench/src/datasets/mod.rs | 11 +++++ vortex-bench/src/measurements.rs | 9 ++++ vortex-bench/src/runner.rs | 41 +++++++++++++++++++ 9 files changed, 85 insertions(+) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 5f57862937a..e2da59838e9 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -377,6 +377,7 @@ jobs: --targets-json '${{ steps.targets.outputs.targets_json }}' \ --output results.json \ --no-build \ + --runner "ec2_${{ inputs.machine_type }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} @@ -395,6 +396,7 @@ jobs: --targets-json '${{ steps.targets.outputs.targets_json }}' \ --output results.json \ --no-build \ + --runner "ec2_${{ inputs.machine_type }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ --opt remote-data-dir=${{ matrix.remote_storage }} \ ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index 8ee8a7ef055..d497d85ed13 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -202,6 +202,10 @@ def run( str | None, typer.Option("--targets-json", help="Exact benchmark targets as a JSON array"), ] = None, + runner: Annotated[ + str | None, + typer.Option("--runner", help="Benchmark runner ID (e.g., ec2_c6id.8xlarge)"), + ] = None, output: Annotated[ Path | None, typer.Option("--output", help="Optional path for compatibility JSONL output"), @@ -289,6 +293,7 @@ def run( samply=samply, sample_rate=sample_rate, tracing=tracing, + runner=runner, on_result=lambda line, store_writer=ctx.write_raw_json, compatibility=compatibility_file: ( write_result_line( line, diff --git a/bench-orchestrator/bench_orchestrator/runner/executor.py b/bench-orchestrator/bench_orchestrator/runner/executor.py index 4fa87076d64..b895afdc2e1 100644 --- a/bench-orchestrator/bench_orchestrator/runner/executor.py +++ b/bench-orchestrator/bench_orchestrator/runner/executor.py @@ -39,6 +39,7 @@ def build_command( samply: bool = False, sample_rate: int | None = None, tracing: bool = False, + runner: str | None = None, ) -> list[str]: """Build the command used to execute a benchmark binary.""" cmd = [ @@ -64,6 +65,8 @@ def build_command( cmd.append("--track-memory") if tracing: cmd.append("--tracing") + if runner: + cmd.extend(["--runner", runner]) if options: for key, value in options.items(): cmd.extend(["--opt", f"{key}={value}"]) @@ -94,6 +97,7 @@ def run( samply: bool = False, sample_rate: int | None = None, tracing: bool = False, + runner: str | None = None, on_result: Callable[[str], None] | None = None, ) -> list[str]: """ @@ -123,6 +127,7 @@ def run( samply=samply, sample_rate=sample_rate, tracing=tracing, + runner=runner, ) if self.verbose: diff --git a/benchmarks/datafusion-bench/src/main.rs b/benchmarks/datafusion-bench/src/main.rs index ad0df8ea74a..745d8371303 100644 --- a/benchmarks/datafusion-bench/src/main.rs +++ b/benchmarks/datafusion-bench/src/main.rs @@ -94,6 +94,9 @@ struct Args { #[arg(long, default_value_t = false)] track_memory: bool, + #[arg(long, default_value = "unknown")] + runner: String, + #[arg(long, default_value_t = false)] explain: bool, @@ -149,6 +152,7 @@ async fn main() -> anyhow::Result<()> { let mut runner = SqlBenchmarkRunner::new( &*benchmark, Engine::DataFusion, + args.runner.clone(), args.formats.clone(), args.track_memory, args.hide_progress_bar, diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index 7ab8f1ac7ab..d8a3306b224 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -64,6 +64,9 @@ struct Args { #[arg(long, default_value_t = false)] hide_progress_bar: bool, + #[arg(long, default_value = "unknown")] + runner: String, + #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))] formats: Vec, @@ -142,6 +145,7 @@ fn main() -> anyhow::Result<()> { let mut runner = SqlBenchmarkRunner::new( &*benchmark, Engine::DuckDB, + args.runner.clone(), args.formats.clone(), args.track_memory, args.hide_progress_bar, diff --git a/benchmarks/lance-bench/src/main.rs b/benchmarks/lance-bench/src/main.rs index 73fa8426ffe..6cce97d2548 100644 --- a/benchmarks/lance-bench/src/main.rs +++ b/benchmarks/lance-bench/src/main.rs @@ -65,6 +65,9 @@ struct Args { #[arg(long, default_value_t = false)] track_memory: bool, + #[arg(long, default_value = "unknown")] + runner: String, + #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))] options: Vec, } @@ -93,6 +96,7 @@ async fn main() -> anyhow::Result<()> { let mut runner = SqlBenchmarkRunner::new( &*benchmark, Engine::DataFusion, + args.runner.clone(), vec![Format::Lance], args.track_memory, args.hide_progress_bar, diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs index 9429e06fd8f..7136d5451e1 100644 --- a/vortex-bench/src/datasets/mod.rs +++ b/vortex-bench/src/datasets/mod.rs @@ -22,6 +22,17 @@ pub mod tpch_l_comment; use std::path::PathBuf; +pub(crate) const DEFAULT_BENCHMARK_RUNNER_ID: &str = "unknown"; + +pub(crate) fn normalize_benchmark_runner_id(benchmark_runner: &str) -> String { + let benchmark_runner = benchmark_runner.trim().replace('/', "_"); + if benchmark_runner.is_empty() { + DEFAULT_BENCHMARK_RUNNER_ID.to_string() + } else { + benchmark_runner + } +} + #[async_trait] pub trait Dataset { fn name(&self) -> &str; diff --git a/vortex-bench/src/measurements.rs b/vortex-bench/src/measurements.rs index f49349cd95e..91036db3f5e 100644 --- a/vortex-bench/src/measurements.rs +++ b/vortex-bench/src/measurements.rs @@ -243,6 +243,7 @@ pub struct QueryMeasurement { pub query_idx: usize, pub target: Target, pub benchmark_dataset: BenchmarkDataset, + pub benchmark_runner: String, /// The storage backend against which this test was run. One of: s3, gcs, nvme. pub storage: String, pub runs: Vec, @@ -279,6 +280,8 @@ pub struct QueryMeasurementJson { pub name: String, pub storage: String, pub dataset: BenchmarkDataset, + /// The cloud runner used to run this + pub runner: String, pub unit: String, pub value: u128, pub all_runtimes: Vec, @@ -310,6 +313,7 @@ impl ToJson for QueryMeasurement { name, storage: self.storage.clone(), dataset: self.benchmark_dataset.clone(), + runner: self.benchmark_runner.clone(), unit: "ns".to_string(), value: self.median_run().as_nanos(), all_runtimes: self.runs.iter().map(|r| r.as_nanos()).collect_vec(), @@ -430,6 +434,7 @@ pub struct MemoryMeasurement { pub query_idx: usize, pub target: Target, pub benchmark_dataset: BenchmarkDataset, + pub benchmark_runner: String, pub storage: String, pub physical_memory_delta: i64, pub virtual_memory_delta: i64, @@ -442,6 +447,7 @@ impl MemoryMeasurement { query_idx: usize, target: Target, benchmark_dataset: BenchmarkDataset, + benchmark_runner: String, storage: String, memory_result: MemoryMeasurementResult, ) -> Self { @@ -449,6 +455,7 @@ impl MemoryMeasurement { query_idx, target, benchmark_dataset, + benchmark_runner, storage, physical_memory_delta: memory_result.physical_memory_delta, virtual_memory_delta: memory_result.virtual_memory_delta, @@ -474,6 +481,7 @@ impl ToJson for MemoryMeasurement { name, storage: self.storage.clone(), dataset: self.benchmark_dataset.clone(), + runner: self.benchmark_runner.clone(), physical_memory_delta: self.physical_memory_delta, virtual_memory_delta: self.virtual_memory_delta, peak_physical_memory: self.peak_physical_memory, @@ -507,6 +515,7 @@ pub struct MemoryMeasurementJson { pub name: String, pub storage: String, pub dataset: BenchmarkDataset, + pub runner: String, pub physical_memory_delta: i64, pub virtual_memory_delta: i64, pub peak_physical_memory: u64, diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index dc7d729232e..3885bace2af 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -20,6 +20,8 @@ use crate::BenchmarkDataset; use crate::Engine; use crate::Format; use crate::Target; +use crate::datasets::DEFAULT_BENCHMARK_RUNNER_ID; +use crate::datasets::normalize_benchmark_runner_id; /// Controls whether queries are benchmarked or explained. pub enum BenchmarkMode { @@ -66,6 +68,7 @@ pub struct BenchmarkResults { pub struct SqlBenchmarkRunner { engine: Engine, benchmark_dataset: BenchmarkDataset, + benchmark_runner: String, storage: String, expected_row_counts: Option>, /// Deduplicated, preserving insertion order. @@ -81,6 +84,7 @@ impl SqlBenchmarkRunner { pub fn new( benchmark: &B, engine: Engine, + benchmark_runner: String, formats: impl IntoIterator, track_memory: bool, hide_progress_bar: bool, @@ -88,12 +92,15 @@ impl SqlBenchmarkRunner { let mut seen = HashSet::new(); let formats: Vec = formats.into_iter().filter(|f| seen.insert(*f)).collect(); let storage = url_scheme_to_storage(benchmark.data_url())?; + let benchmark_runner = normalize_benchmark_runner_id(&benchmark_runner); + validate_benchmark_runner_id(&benchmark_runner, is_ci())?; let memory_tracker = track_memory.then(BenchmarkMemoryTracker::new); Ok(Self { engine, benchmark_dataset: benchmark.dataset(), + benchmark_runner, storage, expected_row_counts: benchmark.expected_row_counts().map(|s| s.to_vec()), formats, @@ -168,6 +175,7 @@ impl SqlBenchmarkRunner { query_idx, target, benchmark_dataset: self.benchmark_dataset.clone(), + benchmark_runner: self.benchmark_runner.clone(), storage: self.storage.clone(), runs, }); @@ -192,6 +200,7 @@ impl SqlBenchmarkRunner { query_idx, target, self.benchmark_dataset.clone(), + self.benchmark_runner.clone(), self.storage.clone(), memory_result, )); @@ -411,6 +420,18 @@ impl SqlBenchmarkRunner { } } +fn is_ci() -> bool { + matches!(std::env::var("CI").as_deref(), Ok("true")) +} + +fn validate_benchmark_runner_id(benchmark_runner: &str, is_ci: bool) -> anyhow::Result<()> { + anyhow::ensure!( + !is_ci || benchmark_runner != DEFAULT_BENCHMARK_RUNNER_ID, + "benchmark runner must not be unknown in CI; pass --runner" + ); + Ok(()) +} + pub fn export_results( queries: Vec, memory: Vec, @@ -460,3 +481,23 @@ pub fn filter_queries( }) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ci_rejects_unknown_benchmark_runner() { + assert!(validate_benchmark_runner_id("unknown", true).is_err()); + } + + #[test] + fn ci_accepts_explicit_benchmark_runner() { + assert!(validate_benchmark_runner_id("ec2_c6id.8xlarge", true).is_ok()); + } + + #[test] + fn local_accepts_unknown_benchmark_runner() { + assert!(validate_benchmark_runner_id("unknown", false).is_ok()); + } +} From 0597991429a5d51ca58597395c3f47911e97330f Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 24 Apr 2026 14:37:02 +0100 Subject: [PATCH 185/250] DuckDB backend for benchmarks website (#7491) ## Summary This PR started as a joke with @joseph-isaacs, but I actually think the result is quite nice (even considering the diff size). This PR changes the benchmarks website backend from having a specialized ETL in JS, to processing the raw JSON into DuckDB tables, and using SQL to serve all endpoints. ### Tooling Moving to SQL also makes it easier to explore data locally, and this change includes tools and documentation on how to get the data into a local DuckDB instance, including generating all required SQL statements and downloading the data. ### Docs This PR includes new docs about the benchmarks website: - README.md includes general setup, development and how to explore data locally. - SCHEMA.md describes the various tables and views, and how the API uses them - ETL.md attempts at describing the full lifecycle of the data, from being generated by the post-merge CI run, how various fields actually look and the processing steps it goes through before reaching the API. --------- Signed-off-by: Adam Gutglick --- benchmarks-website/Dockerfile | 5 +- benchmarks-website/ETL.md | 302 +++++++ benchmarks-website/README.md | 206 +++++ benchmarks-website/SCHEMA.md | 298 +++++++ benchmarks-website/package-lock.json | 102 +++ benchmarks-website/package.json | 4 +- .../scripts/export-bootstrap-sql.js | 89 ++ benchmarks-website/server.js | 830 +++--------------- benchmarks-website/src/App.jsx | 32 +- benchmarks-website/src/api.js | 70 +- .../src/components/ChartContainer.jsx | 2 +- benchmarks-website/src/components/Header.jsx | 5 +- benchmarks-website/src/components/Modal.jsx | 2 +- .../src/hooks/useBenchmarkMetadata.js | 75 ++ benchmarks-website/store/benchmark-store.js | 354 ++++++++ benchmarks-website/store/cache.js | 174 ++++ benchmarks-website/store/constants.js | 20 + benchmarks-website/store/db.js | 33 + benchmarks-website/store/downsample.js | 68 ++ benchmarks-website/store/metadata.js | 401 +++++++++ benchmarks-website/store/sql.js | 479 ++++++++++ benchmarks-website/store/utils.js | 3 + 22 files changed, 2820 insertions(+), 734 deletions(-) create mode 100644 benchmarks-website/ETL.md create mode 100644 benchmarks-website/README.md create mode 100644 benchmarks-website/SCHEMA.md create mode 100644 benchmarks-website/scripts/export-bootstrap-sql.js create mode 100644 benchmarks-website/src/hooks/useBenchmarkMetadata.js create mode 100644 benchmarks-website/store/benchmark-store.js create mode 100644 benchmarks-website/store/cache.js create mode 100644 benchmarks-website/store/constants.js create mode 100644 benchmarks-website/store/db.js create mode 100644 benchmarks-website/store/downsample.js create mode 100644 benchmarks-website/store/metadata.js create mode 100644 benchmarks-website/store/sql.js create mode 100644 benchmarks-website/store/utils.js diff --git a/benchmarks-website/Dockerfile b/benchmarks-website/Dockerfile index 1f87a7148b5..094d9a9a822 100644 --- a/benchmarks-website/Dockerfile +++ b/benchmarks-website/Dockerfile @@ -1,16 +1,17 @@ -FROM node:24-alpine AS build +FROM node:24-bookworm-slim AS build WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci COPY . . RUN npm run build -FROM node:24-alpine +FROM node:24-bookworm-slim WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev COPY --from=build /app/dist ./dist COPY server.js . +COPY store ./store COPY src/config.js ./src/config.js EXPOSE 3000 CMD ["node", "server.js"] diff --git a/benchmarks-website/ETL.md b/benchmarks-website/ETL.md new file mode 100644 index 00000000000..3087e683d60 --- /dev/null +++ b/benchmarks-website/ETL.md @@ -0,0 +1,302 @@ +# Benchmark Website ETL + +For the exact normalized table definitions, see [SCHEMA.md](./SCHEMA.md). + +## Source Artifacts + +The website consumes two append-only JSON artifacts from S3: + +- `commits.json` +- `data.json.gz` + +Those artifacts are produced by GitHub Actions on every push to `develop`: + +- [`../.github/workflows/bench.yml`](../.github/workflows/bench.yml) + runs the random-access and compression benchmarks +- [`../.github/workflows/sql-benchmarks.yml`](../.github/workflows/sql-benchmarks.yml) + runs ClickBench, TPC-H, TPC-DS, StatPopGen, PolarSignals, and other SQL workloads + +Both workflows append into the S3 objects with [`../scripts/cat-s3.sh`](../scripts/cat-s3.sh), +which uses S3 `ETag` compare-and-swap to preserve append-only semantics under concurrent writers. + +## Artifact Generation + +### `commits.json` + +`commits.json` is generated by [`../scripts/commit-json.sh`](../scripts/commit-json.sh). + +Important detail: `timestamp` comes from the Git **committer date**: + +```bash +git log -1 --format=%cd --date=iso-strict "$GITHUB_SHA" +``` + +So the website orders commits by committer timestamp, not author timestamp. + +Logical emitted schema: + +```sql +create table commits_json_artifact ( + id varchar primary key, + message varchar, + timestamp varchar, -- ISO 8601 with timezone, from git %cd + author json, + committer json, + tree_id varchar, + url varchar +); +``` + +Representative record: + +```json +{ + "author": { "email": "alice@example.com", "name": "Alice" }, + "committer": { "email": "alice@example.com", "name": "Alice" }, + "id": "abc123...", + "message": "Improve Vortex scan planning", + "timestamp": "2026-04-17T09:41:52+00:00", + "tree_id": "def456...", + "url": "https://github.com/vortex-data/vortex/commit/abc123..." +} +``` + +### `data.json.gz` + +`data.json.gz` is gzipped newline-delimited JSON. Each line is one benchmark sample emitted by +the benchmark runners in `vortex-bench`. + +The website intentionally projects only the subset it needs: + +```sql +create table benchmark_json_projection ( + name varchar, + unit varchar, + value double, + storage varchar, + dataset json, + commit json, + commit_id varchar +); +``` + +Fields present upstream but ignored by the website include `target`, `time`, `bytes`, +`all_runtimes`, and `env_triple`. See +[`../vortex-bench/src/measurements.rs`](../vortex-bench/src/measurements.rs). + +Two raw fields carry most of the semantic structure: + +- `name` + encodes benchmark family, query number, chart identity, and series identity +- `dataset` + is used for suite fan-out such as TPC-H / TPC-DS scale factors + +Representative naming patterns: + +```text +random-access/// +clickbench_q01/: +tpch_q07/: +compress time/ +vortex size/ +``` + +## Refresh Pipeline + +At refresh time, the server: + +1. downloads or reuses cached copies of `commits.json` and `data.json.gz` +2. exposes them to DuckDB as NDJSON views +3. materializes the normalized relational model in `:memory:` DuckDB +4. computes metadata and latest-value projections +5. serves chart slices and summaries from those tables + +The refresh uses conditional HTTP requests against the cached `etag` and `last-modified` values. +If both files are unchanged and the server already has an in-memory state, the rebuild is skipped. + +## Unicode Relation Diagrams + +```text + Refresh / Materialization Graph + +┌──────────────────────────────┐ ┌──────────────────────────────┐ +│ raw_commits │ │ raw_benchmarks │ +│ view over commits.json │ │ view over data.json.gz │ +│ ──────────────────────────── │ │ ──────────────────────────── │ +│ id │ │ benchmark_row │ +│ message │ │ name │ +│ timestamp │ │ unit │ +│ author │ │ value │ +│ url │ │ storage │ +└──────────────┬───────────────┘ │ dataset │ + │ │ commit │ + │ order by timestamp │ commit_id │ + ▼ └──────────────┬───────────────┘ +┌──────────────────────────────┐ │ +│ commit_dim │ │ split name / keep raw JSON +│ materialized table │ ▼ +│ ──────────────────────────── │ ┌──────────────────────────────┐ +│ PK commit_idx │ │ benchmarks_base │ +│ UK id │ │ materialized table │ +│ message │ │ ──────────────────────────── │ +│ timestamp_text │ │ PK benchmark_row │ +│ commit_ts │ │ name / name_lower │ +│ author │ │ part1..part4 / part_count │ +│ url │ │ unit / raw_value │ +└──────────────┬───────────────┘ │ storage │ + │ │ dataset_json │ + │ join on id = │ commit_json │ + │ resolved_commit_id│ commit_id │ + │ └──────────────┬───────────────┘ + │ │ + │ ┌─────────────────────┼─────────────────────┐ + │ │ │ │ + │ ▼ ▼ ▼ + │ ┌───────────────────┐ ┌───────────────────┐ ┌──────────────────────┐ + │ │ query_suites │ │ engine_renames │ │ valid_groups │ + │ │ config table │ │ config table │ │ config table │ + │ │ prefix │ │ src │ │ PK group_name │ + │ │ display_name │ │ dst │ └──────────┬───────────┘ + │ │ query_prefix │ └─────────┬─────────┘ │ allowlist + │ │ dataset_key │ │ rename series │ + │ │ fan_out / skip │ │ │ + │ └─────────┬─────────┘ │ │ + │ │ match on prefix │ │ + │ ▼ │ │ + │ ┌──────────────────────────────┐ │ │ + │ │ matched_suites │ │ │ + │ │ materialized table │ │ │ + │ │ PK benchmark_row │ │ │ + │ │ prefix / display_name │ │ │ + │ │ query_prefix │ │ │ + │ │ dataset_key / fan_out │ │ │ + │ └──────────────┬───────────────┘ │ │ + │ │ │ │ + │ └────────┬────────┴──────────────┐ │ + │ ▼ │ │ + │ ┌──────────────────────────────────────────────┐ │ │ + │ │ classified_benchmarks │◄┘ │ + │ │ materialized table │ │ + │ │ ──────────────────────────────────────────── │ │ + │ │ PK benchmark_row │ │ + │ │ name │ │ + │ │ resolved_commit_id │ │ + │ │ group_name │ │ + │ │ chart_name │ │ + │ │ series_name │ │ + │ │ sort_position │ │ + │ │ unit / value │ │ + │ └──────────────────────┬─────────────────────────┘ │ + │ │ │ + │ │ join commit + apply valid_groups │ + │ ▼ │ + │ ┌──────────────────────────────────────────────┐◄───────────┘ + │ │ benchmark_points │ + │ │ first fact table │ + │ │ ──────────────────────────────────────────── │ + │ │ group_name │ + │ │ chart_name │ + │ │ series_name │ + │ │ sort_position │ + │ │ unit / value │ + │ │ FK commit_idx -> commit_dim.commit_idx │ + │ └──────────────────────┬───────────────────────┘ + │ │ trim leading empty commits + │ ▼ + │ ┌──────────────────────────────────────────────┐ + │ │ active_commits │ + │ │ rebased commit axis │ + │ │ ──────────────────────────────────────────── │ + │ │ PK original_commit_idx │ + │ │ UK commit_idx │ + │ │ id / message / timestamp / author / url │ + │ └──────────────────────┬───────────────────────┘ + │ │ join on original_commit_idx + └──────────────────────────────┴─────────────────────────────────────┐ + ▼ + ┌──────────────────────────────────────────────┐ + │ benchmark_points_active │ + │ canonical chart fact table │ + │ ──────────────────────────────────────────── │ + │ group_name │ + │ chart_name │ + │ series_name │ + │ sort_position │ + │ unit / value │ + │ commit_idx │ + │ practical key: │ + │ (group_name, chart_name, series_name, │ + │ commit_idx) │ + └──────────────────────────────────────────────┘ +``` + +```text + Serving / Derived Projection Graph + + ┌──────────────────────────────────────────────┐ + │ benchmark_points_active │ + │ group_name, chart_name, series_name, │ + │ sort_position, unit, value, commit_idx │ + └───────┬──────────────────────┬───────────────┘ + │ │ + group by chart │ │ arg_max(value, commit_idx) + │ │ + ▼ ▼ + ┌──────────────────────────────┐ ┌──────────────────────────────┐ + │ chart_defs │ │ chart_series_latest_values │ + │ PK (group_name, chart_name) │ │ PK (group_name, chart_name, │ + │ sort_position │ │ series_name) │ + │ unit │ │ latest_value │ + └──────────────┬──────────────┘ └──────────────┬───────────────┘ + │ │ + │ latest commit per chart │ + ▼ │ + ┌──────────────────────────────┐ │ + │ chart_latest_idx │ │ + │ PK (group_name, chart_name) │ │ + │ latest_commit_idx │ │ + └──────────────┬──────────────┘ │ + │ join on latest_commit_idx │ + ▼ │ + ┌──────────────────────────────┐ │ + │ chart_latest_values │──────────────────────┘ + │ group_name │ + │ chart_name │ + │ series_name │ + │ value │ + └──────────────┬──────────────┘ + │ + │ + active_commits + ▼ + ┌──────────────────────────────┐ + │ /api/metadata │ + │ groups, charts, summaries, │ + │ commits, lastUpdated │ + └──────────────────────────────┘ + + ┌──────────────────────────────┐ + │ active_commits │ + │ commit_idx, id, message, │ + │ timestamp, author, url │ + └──────────────┬──────────────┘ + │ range slice + ▼ + ┌──────────────────────────────┐ + │ /api/data/:group/:chart │ + │ dense series arrays over │ + │ requested commit range │ + └──────────────────────────────┘ +``` + +## Transformation Notes + +- `commit_dim` is the first place timestamps become typed and ordered. +- `benchmarks_base` is the string-parsing layer over benchmark `name`. +- `classified_benchmarks` is the semantic step where the row becomes: + - a group + - a chart + - a series + - a normalized display unit/value +- `benchmark_points_active` is the canonical long-form chart fact table. +- The API never stores dense charts on disk; dense series arrays are produced on demand from the + long-form fact table. diff --git a/benchmarks-website/README.md b/benchmarks-website/README.md new file mode 100644 index 00000000000..2b67382e7e3 --- /dev/null +++ b/benchmarks-website/README.md @@ -0,0 +1,206 @@ +# Benchmarks Website + +This directory contains the benchmark website frontend, the Node HTTP server, and the DuckDB-based +refresh pipeline that turns the raw benchmark artifacts into chartable time series. + +For the data model and table relationships, start with [SCHEMA.md](./SCHEMA.md). +For the upstream artifact generation and refresh/materialization flow, see [ETL.md](./ETL.md). + +## Prerequisites + +- Node.js `>=18` +- npm +- Optional: DuckDB CLI, if you want to query the cached artifacts directly + +Install dependencies: + +```bash +cd benchmarks-website +npm install +``` + +## Development Server + +Run the frontend and backend together: + +```bash +cd benchmarks-website +npm run dev +``` + +That starts: + +- Vite on `http://localhost:5173` +- the API/static server on `http://localhost:3000` + +Useful endpoints: + +- `http://localhost:3000/api/metadata` +- `http://localhost:3000/api/health` + +The backend refreshes from these artifact URLs by default: + +- `https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz` +- `https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json` + +Relevant environment variables: + +```bash +PORT=3000 +REFRESH_INTERVAL=300000 +DATA_URL=https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz +COMMITS_URL=https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json +CACHE_DIR=/path/to/local/cache +``` + +`CACHE_DIR` is the most useful one during development. If it is unset, the server uses a temp +directory under `os.tmpdir()`. + +## Pull The Data Locally + +If you want a predictable local copy for exploration, populate a cache directory yourself and point +the server at it. + +```bash +cd benchmarks-website +mkdir -p .cache/benchmarks + +curl -L \ + https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz \ + -o .cache/benchmarks/data.json.gz + +curl -L \ + https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json \ + -o .cache/benchmarks/commits.json +``` + +Then start the dev server against that cache: + +```bash +cd benchmarks-website +CACHE_DIR="$PWD/.cache/benchmarks" npm run dev +``` + +On first startup, the server will use the cached files immediately and then asynchronously +revalidate them against S3. + +## Explore The Cached Data Directly + +Once `data.json.gz` and `commits.json` exist locally, you can query them with DuckDB without +running the website. + +Example with the DuckDB CLI: + +```sql +create view raw_commits as +select * +from read_json( + '.cache/benchmarks/commits.json', + format = 'newline_delimited', + compression = 'auto_detect', + columns = { + id: 'VARCHAR', + message: 'VARCHAR', + timestamp: 'VARCHAR', + author: 'JSON', + url: 'VARCHAR' + } +); + +create view raw_benchmarks as +select * +from read_json( + '.cache/benchmarks/data.json.gz', + format = 'newline_delimited', + compression = 'auto_detect', + columns = { + name: 'VARCHAR', + unit: 'VARCHAR', + value: 'DOUBLE', + storage: 'VARCHAR', + dataset: 'JSON', + commit: 'JSON', + commit_id: 'VARCHAR' + } +); +``` + +Useful starter queries: + +```sql +select count(*) as commit_count from raw_commits; + +select count(*) as benchmark_count from raw_benchmarks; + +select split_part(name, '/', 1) as prefix, count(*) as rows +from raw_benchmarks +group by 1 +order by 2 desc +limit 20; + +select + coalesce(json_extract_string(commit, '$.id'), commit_id) as resolved_commit_id, + count(*) as rows +from raw_benchmarks +group by 1 +order by 2 desc +limit 20; +``` + +If you want the normalized relational model rather than the raw JSON views, follow the pipeline in +[SCHEMA.md](./SCHEMA.md) and [`store/sql.js`](./store/sql.js). + +## Export The Full Bootstrap SQL + +If you want the exact SQL that the server uses to create all config tables, raw views, normalized +tables, and derived projections, export it from the shared SQL builder: + +```bash +cd benchmarks-website +npm run export-sql -- \ + --data-path "$PWD/.cache/benchmarks/data.json.gz" \ + --commits-path "$PWD/.cache/benchmarks/commits.json" \ + --output "$PWD/.cache/benchmarks/bootstrap.sql" +``` + +Then load it in DuckDB: + +```bash +duckdb benchmark-explore.duckdb < .cache/benchmarks/bootstrap.sql +``` + +That creates the same tables and views the server uses, including: + +- `query_suites` +- `valid_groups` +- `engine_renames` +- `raw_commits` +- `raw_benchmarks` +- `commit_dim` +- `benchmarks_base` +- `matched_suites` +- `classified_benchmarks` +- `benchmark_points` +- `active_commits` +- `benchmark_points_active` +- `chart_defs` +- `chart_latest_idx` +- `chart_latest_values` +- `chart_series_latest_values` + +If you want a portable template instead of path-specific SQL: + +```bash +cd benchmarks-website +npm run export-sql -- --placeholders --output bootstrap.template.sql +``` + +That emits a script using `__DATA_PATH__` and `__COMMITS_PATH__` placeholders. + +## Notes + +- The website only projects the subset of the raw benchmark JSON it needs for grouping, charting, + and summaries. +- Benchmark names are part of the schema. Group, chart, and series identity are inferred from the + `name`, `storage`, and `dataset` fields during refresh. +- The server returns `503` with `Retry-After` while the initial refresh is still loading. diff --git a/benchmarks-website/SCHEMA.md b/benchmarks-website/SCHEMA.md new file mode 100644 index 00000000000..d54fd168182 --- /dev/null +++ b/benchmarks-website/SCHEMA.md @@ -0,0 +1,298 @@ +# Benchmark Website Schema + +This file is the reference for the **normalized DuckDB schema** used by the benchmark website. + +For the upstream artifact generation, refresh pipeline, and table-to-table flow, see +[ETL.md](./ETL.md). + +The implementation lives primarily in [`store/sql.js`](./store/sql.js), +[`store/metadata.js`](./store/metadata.js), and [`store/benchmark-store.js`](./store/benchmark-store.js). +For DuckDB's NDJSON reader semantics, see: +. + +## Core Fact Model + +The important point is that the website does **not** store nested charts directly. It stores a +long-form fact table, `benchmark_points_active`, keyed in practice by: + +```sql +(group_name, chart_name, series_name, commit_idx) +``` + +Everything else is metadata, lookup/config data, or a projection over that fact table. + +## Configuration Tables + +These are synthesized from [`src/config.js`](./src/config.js), not loaded from the JSON artifacts. + +```sql +create table query_suites ( + prefix varchar, + display_name varchar, + query_prefix varchar, + dataset_key varchar, + fan_out boolean, + skip boolean +); + +create table valid_groups ( + group_name varchar primary key +); + +create table engine_renames ( + src varchar primary key, + dst varchar +); +``` + +## Raw Views + +These are views over the cached artifact files. + +```sql +create view raw_commits as +select + id, + message, + timestamp, + author, + url +from read_json(...); + +create view raw_benchmarks as +select + row_number() over () as benchmark_row, + name, + unit, + value, + storage, + dataset, + commit, + commit_id +from read_json(...); +``` + +## Stage 1: Commit Ordering and Name Parsing + +```sql +create table commit_dim ( + commit_idx bigint primary key, + id varchar unique, + message varchar, + timestamp_text varchar, + commit_ts timestamptz, + author varchar, + url varchar +); + +create table benchmarks_base ( + benchmark_row bigint primary key, + name varchar, + name_lower varchar, + part1 varchar, + part1_lower varchar, + part2 varchar, + part3 varchar, + part4 varchar, + part_count bigint, + unit varchar, + raw_value double, + storage varchar, + dataset_json json, + commit_json json, + commit_id varchar +); + +create table matched_suites ( + benchmark_row bigint primary key, + prefix varchar, + display_name varchar, + query_prefix varchar, + dataset_key varchar, + fan_out boolean +); +``` + +Notes: + +- `commit_dim` orders commits by parsed timestamp, then `id`. +- `benchmarks_base` is the normalized string-parsing layer over benchmark `name`. +- `matched_suites` assigns a benchmark row to the longest matching configured suite prefix. + +## Stage 2: Classification + +```sql +create table classified_benchmarks ( + benchmark_row bigint primary key, + name varchar, + resolved_commit_id varchar, + group_name varchar, + chart_name varchar, + series_name varchar, + sort_position integer, + unit varchar, + value double +); +``` + +This is the semantic classification step: + +- `resolved_commit_id` is `coalesce(commit.id, commit_id)` +- `group_name` is inferred from benchmark name prefixes, query suite config, storage, and scale + factor metadata +- `chart_name` and `series_name` come from benchmark-name parsing +- `series_name` is passed through `engine_renames` +- `unit` and `value` are normalized for display: + - `ns -> ms/iter` + - `bytes -> MiB` + +In other words, the benchmark-name grammar is part of the schema. + +## Stage 3: Time-Series Fact Tables + +```sql +create table benchmark_points ( + group_name varchar, + chart_name varchar, + series_name varchar, + sort_position integer, + unit varchar, + value double, + commit_idx bigint +); + +create table active_commits ( + original_commit_idx bigint primary key, + commit_idx bigint unique, + id varchar, + message varchar, + timestamp varchar, + author varchar, + url varchar +); + +create table benchmark_points_active ( + group_name varchar, + chart_name varchar, + series_name varchar, + sort_position integer, + unit varchar, + value double, + commit_idx bigint +); +``` + +Notes: + +- `benchmark_points` is the first fact table after commit resolution and group filtering. +- `active_commits` trims leading commits with no benchmark data and re-bases the visible commit + axis to `0..N-1`. +- `benchmark_points_active` is the canonical fact table used for charting and summaries. + +## Stage 4: Chart Metadata and Latest-Value Projections + +```sql +create table chart_defs ( + group_name varchar, + chart_name varchar, + sort_position integer, + unit varchar, + primary key (group_name, chart_name) +); + +create table chart_latest_idx ( + group_name varchar, + chart_name varchar, + latest_commit_idx bigint, + primary key (group_name, chart_name) +); + +create table chart_latest_values ( + group_name varchar, + chart_name varchar, + series_name varchar, + value double +); + +create table chart_series_latest_values ( + group_name varchar, + chart_name varchar, + series_name varchar, + latest_value double, + primary key (group_name, chart_name, series_name) +); +``` + +There are two different "latest" concepts: + +- `chart_latest_values` + values at the chart's latest commit index +- `chart_series_latest_values` + latest non-null value seen for each series, via `arg_max(value, commit_idx)` + +## API-Facing Reads + +### `/api/metadata` + +Built in [`store/metadata.js`](./store/metadata.js). + +Primary sources: + +- `active_commits` +- `chart_defs` +- `chart_series_latest_values` +- `chart_latest_values` +- `benchmark_points_active` + +Logical output shape: + +```sql +metadata ( + total_commits bigint, + last_updated timestamptz, + commits json, + groups json +) +``` + +Each group contains: + +```sql +group_metadata ( + charts json, + total_charts bigint, + has_data boolean, + summary json +) +``` + +Summary sections are computed from SQL aggregates: + +- random access ranking from one anchor chart +- compression geomeans from Vortex/Parquet ratio charts +- compression-size min/geomean/max +- query-suite geomeans with a missing-query penalty + +### `/api/data/:group/:chart` + +Built in [`store/benchmark-store.js`](./store/benchmark-store.js). + +The query: + +1. selects the requested commit range from `active_commits` +2. enumerates the series present for that chart +3. cross joins series against commits +4. left joins `benchmark_points_active` +5. aggregates with `list(value order by commit_idx)` + +That produces a dense matrix with explicit `null` gaps: + +```sql +series_points ( + series_name varchar, + values list +) +``` + +The server then serializes the commit slice, applies downsampling if needed, and returns +`{ commits, series, requestedRange, downsampleLevel, ... }`. diff --git a/benchmarks-website/package-lock.json b/benchmarks-website/package-lock.json index d140b73d225..b64b131bb36 100644 --- a/benchmarks-website/package-lock.json +++ b/benchmarks-website/package-lock.json @@ -8,6 +8,7 @@ "name": "vortex-benchmarks-website", "version": "2.0.0", "dependencies": { + "@duckdb/node-api": "^1.5.2-r.1", "chart.js": "^4.4.4", "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", @@ -320,6 +321,107 @@ "node": ">=6.9.0" } }, + "node_modules/@duckdb/node-api": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.2-r.1.tgz", + "integrity": "sha512-OzBBnS0JGXMoS5mzKNY/Ylr7SshcRQiLFIoxQ4AlePwJ2fNeDL/fbHu/knjxUrXwW1fJBTUgwWftmxDdnZZb3A==", + "license": "MIT", + "dependencies": { + "@duckdb/node-bindings": "1.5.2-r.1" + } + }, + "node_modules/@duckdb/node-bindings": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.2-r.1.tgz", + "integrity": "sha512-bUg3bLVj70YVku6fKyQJS8ASORl7kM7YFVFznsEB9pWbtazPj+ME2x2FUk0WiTzjJdutjzSSGXF066mB4bGGZA==", + "license": "MIT", + "optionalDependencies": { + "@duckdb/node-bindings-darwin-arm64": "1.5.2-r.1", + "@duckdb/node-bindings-darwin-x64": "1.5.2-r.1", + "@duckdb/node-bindings-linux-arm64": "1.5.2-r.1", + "@duckdb/node-bindings-linux-x64": "1.5.2-r.1", + "@duckdb/node-bindings-win32-arm64": "1.5.2-r.1", + "@duckdb/node-bindings-win32-x64": "1.5.2-r.1" + } + }, + "node_modules/@duckdb/node-bindings-darwin-arm64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.2-r.1.tgz", + "integrity": "sha512-v35FyKOb8EJCvaiPF7k0gvKiJTXR7PPQDNoWR0Gu+YSX5O9b+DIguzt1348Of3HebHy6ATSMzlUekaVA9YXu+g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@duckdb/node-bindings-darwin-x64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.2-r.1.tgz", + "integrity": "sha512-SU9dIJ1BluKkkGxi4UsP4keqkkstB2YDySF9KcYu3EZKIVM3FTv2zc7XO38dXnHOq6+F3WqhWWZvD+XU945p7A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@duckdb/node-bindings-linux-arm64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.2-r.1.tgz", + "integrity": "sha512-3Tra9xM3aM3denaER4KhJ6//6PpmPbik9ECBQ+sh9PyKaEgHw/0kAcKnLm5EzWUnXF0qYmZlewvkCrse8KmOYw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-x64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.2-r.1.tgz", + "integrity": "sha512-pcQvZRHiIfJ9cq8parkSQczQHEml/IeGfnDCMAbEgD6+jaV9Y9Y5Ph1kP9aR+bm6him1S5ZIEr3kZbihjKnWbA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-win32-arm64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.2-r.1.tgz", + "integrity": "sha512-Ji8tym+N3LkrhVt0Up3bsacD/kpg4/JXFJQqxswiYvBaNCQOk+D+aiVS0GN5pcqvmnG7V7TpsDRzkLEFaWp1vw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@duckdb/node-bindings-win32-x64": { + "version": "1.5.2-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.2-r.1.tgz", + "integrity": "sha512-5XqcqC+4R8ghBEEbnc2a0sqfz1zyPBRb9YcmIWfiuDoCYSYFbKhmHcEyNftZDHcwCoLOHXnUin45jraex4STqQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", diff --git a/benchmarks-website/package.json b/benchmarks-website/package.json index 6cda687b838..13280ae7898 100644 --- a/benchmarks-website/package.json +++ b/benchmarks-website/package.json @@ -7,12 +7,14 @@ "vite": "vite", "server": "node server.js", "build": "vite build", - "preview": "vite preview" + "preview": "vite preview", + "export-sql": "node scripts/export-bootstrap-sql.js" }, "engines": { "node": ">=18.0.0" }, "dependencies": { + "@duckdb/node-api": "^1.5.2-r.1", "chart.js": "^4.4.4", "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", diff --git a/benchmarks-website/scripts/export-bootstrap-sql.js b/benchmarks-website/scripts/export-bootstrap-sql.js new file mode 100644 index 00000000000..9bc2b5eee1d --- /dev/null +++ b/benchmarks-website/scripts/export-bootstrap-sql.js @@ -0,0 +1,89 @@ +#!/usr/bin/env node + +import fs from "fs/promises"; +import path from "path"; +import { buildBootstrapSql } from "../store/sql.js"; + +function parseArgs(argv) { + const options = { + dataPath: null, + commitsPath: null, + outputPath: null, + placeholders: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + switch (arg) { + case "--data-path": + options.dataPath = argv[++i] || null; + break; + case "--commits-path": + options.commitsPath = argv[++i] || null; + break; + case "--output": + options.outputPath = argv[++i] || null; + break; + case "--placeholders": + options.placeholders = true; + break; + case "--help": + case "-h": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + return options; +} + +function printHelp() { + process.stdout.write(`Usage: + node scripts/export-bootstrap-sql.js --data-path --commits-path [--output ] + node scripts/export-bootstrap-sql.js --placeholders [--output ] + +Options: + --data-path Path to data.json.gz + --commits-path Path to commits.json + --output Write SQL to a file instead of stdout + --placeholders Emit __DATA_PATH__ / __COMMITS_PATH__ placeholders + --help, -h Show this help +`); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + + const dataPath = options.placeholders ? "__DATA_PATH__" : options.dataPath; + const commitsPath = options.placeholders + ? "__COMMITS_PATH__" + : options.commitsPath; + + if (!dataPath || !commitsPath) { + printHelp(); + process.stderr.write( + "\nerror: either provide --data-path and --commits-path, or use --placeholders\n", + ); + process.exit(1); + } + + const sql = `${buildBootstrapSql(dataPath, commitsPath)}\n`; + + if (options.outputPath) { + const outputPath = path.resolve(options.outputPath); + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, sql, "utf8"); + process.stdout.write(`${outputPath}\n`); + return; + } + + process.stdout.write(sql); +} + +main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); +}); diff --git a/benchmarks-website/server.js b/benchmarks-website/server.js index a9af96234fe..80ce55e9e82 100644 --- a/benchmarks-website/server.js +++ b/benchmarks-website/server.js @@ -2,16 +2,11 @@ import http from "http"; import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; -import zlib from "zlib"; -import readline from "readline"; -import { Readable } from "stream"; -import { LTTB } from "downsample"; -import { QUERY_SUITES, FAN_OUT_GROUPS, ENGINE_RENAMES } from "./src/config.js"; +import { BenchmarkStore } from "./store/benchmark-store.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -// Configuration const PORT = process.env.PORT || 3000; const DATA_URL = process.env.DATA_URL || @@ -20,17 +15,9 @@ const COMMITS_URL = process.env.COMMITS_URL || "https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json"; const REFRESH_INTERVAL = process.env.REFRESH_INTERVAL || 5 * 60 * 1000; -const MAX_POINTS = 200; const USE_LOCAL_DATA = process.env.USE_LOCAL_DATA === "true"; - -// Benchmark groups: non-query groups + simple suites + fan-out suites -const GROUPS = [ - "Random Access", - "Compression", - "Compression Size", - ...QUERY_SUITES.filter((s) => !s.skip && !s.fanOut).map((s) => s.displayName), - ...FAN_OUT_GROUPS, -]; +const CACHE_DIR = process.env.CACHE_DIR || undefined; +const LOADING_RETRY_AFTER_SECONDS = 1; const MIME = { ".html": "text/html", @@ -46,688 +33,103 @@ const MIME = { ".webmanifest": "application/manifest+json", }; -let store = { - commits: [], - groups: {}, - metadata: null, - downsampled: {}, - lastUpdated: null, -}; - -// Utilities -const rename = (s) => ENGINE_RENAMES[s.toLowerCase()] || ENGINE_RENAMES[s] || s; -const geoMean = (arr) => - arr.length - ? Math.pow( - arr.reduce((a, v) => a * v, 1), - 1 / arr.length, - ) - : null; - -// Categorize benchmarks based on name patterns and metadata -function getGroup(benchmark) { - const name = benchmark.name; - const lower = name.toLowerCase(); - - // Random Access: "random-access/..." or "random access/..." - if ( - lower.startsWith("random-access/") || - lower.startsWith("random access/") - ) { - return "Random Access"; - } - - // Compression Size: size measurements - if ( - lower.startsWith("vortex size/") || - lower.startsWith("vortex-file-compressed size/") || - lower.startsWith("parquet size/") || - lower.startsWith("lance size/") || - lower.includes(":raw size/") || - lower.includes(":parquet-zstd size/") || - lower.includes(":lance size/") - ) { - return "Compression Size"; - } - - // Compression: compress/decompress time and ratio measurements - if ( - lower.startsWith("compress time/") || - lower.startsWith("decompress time/") || - lower.startsWith("parquet_rs-zstd compress") || - lower.startsWith("parquet_rs-zstd decompress") || - lower.startsWith("lance compress") || - lower.startsWith("lance decompress") || - lower.startsWith("vortex:lance ratio") || - lower.startsWith("vortex:parquet-zstd ratio") || - lower.startsWith("vortex:raw ratio") - ) { - return "Compression"; - } - - // SQL query suites: match "{prefix}_q..." or "{prefix}/..." - for (const suite of QUERY_SUITES) { - if ( - !lower.startsWith(suite.prefix + "_q") && - !lower.startsWith(suite.prefix + "/") - ) - continue; - if (suite.skip) return null; - if (!suite.fanOut) return suite.displayName; - // Fan-out suites: expand by storage and scale factor - const storage = benchmark.storage?.toUpperCase() === "S3" ? "S3" : "NVMe"; - const rawSf = benchmark.dataset?.[suite.datasetKey]?.scale_factor; - const sf = rawSf ? Math.round(parseFloat(rawSf)) : 1; - return `${suite.displayName} (${storage}) (SF=${sf})`; - } - - return null; -} - -// Format query name for display: "{prefix}_q00" -> "{QUERY_PREFIX} Q0" -function formatQuery(q) { - const lower = q.toLowerCase(); - for (const suite of QUERY_SUITES) { - if (suite.skip) continue; - const m = lower.match(new RegExp(`^${suite.prefix}[_ ]?q(\\d+)`, "i")); - if (m) return `${suite.queryPrefix} Q${parseInt(m[1], 10)}`; - } - return q.toUpperCase().replace(/[_-]/g, " "); -} - -function normalizeChartName(group, chartName) { - if (group === "Compression Size" && chartName === "VORTEX FILE COMPRESSED SIZE") { - return "VORTEX SIZE"; - } - return chartName; -} +const benchmarks = new BenchmarkStore({ + dataUrl: DATA_URL, + commitsUrl: COMMITS_URL, + useLocalData: USE_LOCAL_DATA, + cacheDir: CACHE_DIR, +}); +let refreshIntervalId = null; -// LTTB downsampling -function lttbIndices(seriesMap, target) { - const keys = [...seriesMap.keys()]; - if (!keys.length) return []; - const len = seriesMap.get(keys[0])?.length || 0; - if (len <= target) return [...Array(len).keys()]; +const json = (res, code, data, headers = {}) => { + res.writeHead(code, { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + Pragma: "no-cache", + ...headers, + }); + res.end(JSON.stringify(data)); +}; - const avg = Array(len); - for (let i = 0; i < len; i++) { - let sum = 0, - n = 0; - for (const arr of seriesMap.values()) { - const v = arr[i]?.value ?? arr[i]; - if (v != null && !isNaN(v)) { - sum += v; - n++; - } - } - avg[i] = [i, n ? sum / n : 0]; +function loadingHeaders(status) { + if (status.state !== "loading") { + return {}; } - const idx = LTTB(avg, target).map((p) => Math.round(p[0])); - if (!idx.includes(0)) idx.unshift(0); - if (!idx.includes(len - 1)) idx.push(len - 1); - return idx.sort((a, b) => a - b); -} - -function downsample(data, factor) { - const target = Math.ceil(data.commits.length / factor); - if (target >= data.commits.length) return data; - - const idx = lttbIndices(data.series, target); - const series = new Map(); - for (const [k, v] of data.series) - series.set( - k, - idx.map((i) => v[i]), - ); - return { - ...data, - commits: idx.map((i) => data.commits[i]), - series, - originalLength: data.commits.length, + "Retry-After": String(LOADING_RETRY_AFTER_SECONDS), }; } -// Data fetching — streams response body directly instead of buffering -async function fetchJsonl(url) { - const res = await fetch(url); - if (!res.ok) throw new Error(`Fetch failed: ${url} ${res.status}`); - return new Promise((resolve, reject) => { - const results = []; - const rl = readline.createInterface({ - input: Readable.fromWeb(res.body), - crlfDelay: Infinity, - }); - rl.on("line", (l) => { - if (l.trim()) - try { - results.push(JSON.parse(l)); - } catch {} - }); - rl.on("close", () => resolve(results)); - rl.on("error", reject); - }); -} - -function readLocalJsonl(fp) { - return new Promise((resolve, reject) => { - const results = []; - const rl = readline.createInterface({ - input: fs.createReadStream(fp), - crlfDelay: Infinity, - }); - rl.on("line", (l) => { - if (l.trim()) - try { - results.push(JSON.parse(l)); - } catch {} - }); - rl.on("close", () => resolve(results)); - rl.on("error", reject); - }); -} - -// Stream benchmark data record-by-record without buffering the entire dataset -async function forEachBenchmark(callback) { - let stream; - if (USE_LOCAL_DATA) { - stream = fs.createReadStream(path.join(__dirname, "sample/data.json")); - } else { - const res = await fetch(DATA_URL); - if (!res.ok) throw new Error(`Fetch failed: ${DATA_URL} ${res.status}`); - stream = Readable.fromWeb(res.body).pipe(zlib.createGunzip()); - } - return new Promise((resolve, reject) => { - const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); - rl.on("line", (l) => { - if (l.trim()) - try { - callback(JSON.parse(l)); - } catch {} - }); - rl.on("close", resolve); - rl.on("error", reject); - }); -} - -// Main data processing -async function refresh() { - console.log("Refreshing data..."); - const t0 = Date.now(); - - try { - // Load commits first (small dataset, must be fully in memory for indexing) - const commitsArr = USE_LOCAL_DATA - ? await readLocalJsonl(path.join(__dirname, "sample/commits.json")) - : await fetchJsonl(COMMITS_URL); - - // Build commit index (O(1) lookup) - const commitMap = new Map(commitsArr.map((c) => [c.id, c])); - const commits = commitsArr.sort( - (a, b) => new Date(a.timestamp) - new Date(b.timestamp), - ); - const commitIdx = new Map(commits.map((c, i) => [c.id, i])); - - const groups = Object.fromEntries(GROUPS.map((g) => [g, new Map()])); - let missing = 0; - let benchmarkCount = 0; - const uncategorized = new Set(); - - // Stream benchmarks one record at a time to avoid loading all into memory - await forEachBenchmark((b) => { - benchmarkCount++; - const commit = b.commit || commitMap.get(b.commit_id); - if (!commit) { - missing++; - return; - } - - const group = getGroup(b); - if (!group) { - uncategorized.add(b.name.split("/")[0]); - return; - } - if (!groups[group]) return; - - // Random access names have the form: random-access/{dataset}/{pattern}/{format} - // Historical random access names: random-access/{format} - // Other benchmarks use: {query}/{series} - let seriesName, chartName; - const parts = b.name.split("/"); - if (group === "Random Access" && parts.length === 4) { - chartName = `${parts[1]}/${parts[2]}`.toUpperCase().replace(/[_-]/g, " "); - seriesName = rename(parts[3] || "default"); - } else if (group === "Random Access" && parts.length === 2) { - chartName = "RANDOM ACCESS"; - seriesName = rename(parts[1] || "default"); - } else { - seriesName = rename(parts[1] || "default"); - chartName = formatQuery(parts[0]); - } - chartName = normalizeChartName(group, chartName); - if (chartName.includes("PARQUET-UNC")) return; - - // Skip throughput metrics (keep only time/size) - if (b.name.includes(" throughput")) return; - - let unit = b.unit; - if (!unit) { - if (b.name.toLowerCase().includes(" size/")) unit = "bytes"; - else if (b.name.toLowerCase().includes(" ratio ")) unit = "ratio"; - else unit = "ns"; - } - - const sortPos = parts[0].match(/q(\d+)$/i)?.[1] - ? parseInt(RegExp.$1, 10) - : 0; - const idx = commitIdx.get(commit.id); - if (idx === undefined) return; - - let chart = groups[group].get(chartName); - if (!chart) { - let displayUnit = unit; - if (unit === "ns") displayUnit = "ms/iter"; - else if (unit === "bytes") displayUnit = "MiB"; - chart = { - sort_position: sortPos, - commits, - unit: displayUnit, - series: new Map(), - }; - groups[group].set(chartName, chart); - } - - if (!chart.series.has(seriesName)) { - chart.series.set(seriesName, Array(commits.length).fill(null)); - } - - // Convert values: ns -> ms, bytes -> MiB - let val = b.value; - if (unit === "ns" && typeof val === "number") { - val = val / 1e6; // ns to ms - } else if (unit === "bytes" && typeof val === "number") { - val = val / (1024 * 1024); // bytes to MiB - } - - chart.series.get(seriesName)[idx] = { value: val }; - }); - - console.log( - `Processed ${benchmarkCount} benchmarks, ${commitsArr.length} commits`, - ); - - // Log uncategorized benchmarks for debugging - if (uncategorized.size > 0) { - console.log( - `Uncategorized benchmark prefixes (${uncategorized.size}):`, - [...uncategorized].slice(0, 20).join(", "), - ); - } - - // Trim leading empty commits - let firstIdx = commits.length; - for (const gc of Object.values(groups)) { - for (const cd of gc.values()) { - for (const sd of cd.series.values()) { - const i = sd.findIndex((d) => d !== null); - if (i !== -1 && i < firstIdx) firstIdx = i; - } - } - } - - if (firstIdx > 0 && firstIdx < commits.length) { - console.log(`Trimming ${firstIdx} empty commits`); - commits.splice(0, firstIdx); - for (const gc of Object.values(groups)) { - for (const cd of gc.values()) { - cd.commits = commits; - for (const [k, v] of cd.series) cd.series.set(k, v.slice(firstIdx)); - } - } - } - - // Sort charts within groups - for (const gc of Object.values(groups)) { - const sorted = [...gc.entries()].sort( - (a, b) => - a[1].sort_position - b[1].sort_position || a[0].localeCompare(b[0]), - ); - gc.clear(); - for (const [k, v] of sorted) gc.set(k, v); - } - - // Precompute downsampled versions - const downsampled = {}; - for (const [gn, gc] of Object.entries(groups)) { - downsampled[gn] = {}; - for (const [cn, cd] of gc) { - downsampled[gn][cn] = { - "1x": cd, - "2x": downsample(cd, 2), - "4x": downsample(cd, 4), - "8x": downsample(cd, 8), - }; - } - } - - // Count charts per group for logging - const groupCounts = Object.entries(groups) - .map(([n, g]) => `${n}: ${g.size}`) - .filter((s) => !s.endsWith(": 0")); - console.log("Charts per group:", groupCounts.join(", ")); - - store = { - commits, - groups, - metadata: buildMeta(groups, commits), - downsampled, - lastUpdated: new Date().toISOString(), - }; - console.log( - `Refresh done in ${Date.now() - t0}ms (${missing} missing commits)`, - ); - } catch (e) { - console.error("Refresh error:", e); - } -} - -// Summary calculations -function latestIdx(chart) { - for (let i = chart.commits.length - 1; i >= 0; i--) { - for (const s of chart.series.values()) if (s[i]?.value != null) return i; - } - return -1; -} - -function calcSummary(name, charts) { - if (name === "Random Access") { - for (const q of charts.values()) { - const i = latestIdx(q); - if (i === -1) continue; - const vals = new Map(); - for (const [n, d] of q.series) - if (d[i]?.value != null) vals.set(n, d[i].value); - if (!vals.size) continue; - const min = Math.min(...vals.values()); - return { - type: "randomAccess", - title: "Random Access Performance", - rankings: [...vals] - .map(([n, t]) => ({ name: n, time: t, ratio: t / min })) - .sort((a, b) => a.time - b.time), - explanation: "Random access time | Ratio to fastest (lower is better)", - }; - } - return null; - } - - if (name === "Compression") { - const cc = charts.get("VORTEX:PARQUET ZSTD RATIO COMPRESS TIME"); - const dc = charts.get("VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME"); - if (!cc && !dc) return null; - const i = latestIdx(cc || dc); - if (i === -1) return null; - const collect = (c) => - c - ? [...c.series] - .filter(([n]) => !n.toLowerCase().includes("wide table")) - .map(([, d]) => d[i]?.value) - .filter((v) => v > 0) - .map((v) => 1 / v) - : []; - return { - type: "compression", - title: "Compression Throughput vs Parquet", - compressRatio: geoMean(collect(cc)), - decompressRatio: geoMean(collect(dc)), - datasetCount: collect(cc).length, - explanation: - "Inverse geomean of Vortex/Parquet ratios (higher is better)", - }; - } - - if (name === "Compression Size") { - const c = charts.get("VORTEX:PARQUET ZSTD SIZE"); - if (!c) return null; - const i = latestIdx(c); - if (i === -1) return null; - const ratios = [...c.series] - .filter(([n]) => !n.toLowerCase().includes("wide table")) - .map(([, d]) => d[i]?.value) - .filter((v) => v > 0); - return ratios.length - ? { - type: "compressionSize", - title: "Compression Size Summary", - minRatio: Math.min(...ratios), - meanRatio: geoMean(ratios), - maxRatio: Math.max(...ratios), - datasetCount: ratios.length, - explanation: - "Geomean of Vortex/Parquet size ratios (lower is better)", - } - : null; - } - - if ( - QUERY_SUITES.some( - (s) => - !s.skip && - (name === s.displayName || name.startsWith(s.displayName + " ")), - ) - ) { - const all = new Map(); - for (const q of charts.values()) - for (const n of q.series.keys()) if (!all.has(n)) all.set(n, new Map()); - for (const [qn, qd] of charts) { - for (const [sn, sd] of qd.series) { - for (let i = sd.length - 1; i >= 0; i--) { - if (sd[i]?.value != null) { - all.get(sn).set(qn, sd[i].value); - break; - } - } - } - } - if (!all.size) return null; - - const scores = new Map(); - for (const [sn, qr] of all) { - let total = 0, - max = 0; - for (const v of qr.values()) { - total += v; - max = Math.max(max, v); - } - const penalty = Math.max(300000, max) * 2; - const ratios = []; - for (const qn of charts.keys()) { - let base = Infinity; - for (const m of all.values()) - if (m.has(qn)) base = Math.min(base, m.get(qn)); - if (base < Infinity) - ratios.push((10 + (qr.get(qn) ?? penalty)) / (10 + base)); - } - if (ratios.length) - scores.set(sn, { score: geoMean(ratios), totalRuntime: total }); - } - - return scores.size - ? { - type: "queryBenchmark", - title: "Performance Summary", - rankings: [...scores] - .map(([n, d]) => ({ name: n, ...d })) - .sort((a, b) => a.score - b.score), - explanation: - "Geomean of query time ratio to fastest (lower is better)", - } - : null; - } - return null; -} - -function buildMeta(groups, commits) { - const meta = {}; - for (const [gn, gc] of Object.entries(groups)) { - const charts = [...gc].map(([cn, cd]) => { - const latest = {}; - for (const [sn, sd] of cd.series) { - for (let i = sd.length - 1; i >= 0; i--) - if (sd[i]?.value != null) { - latest[sn] = sd[i].value; - break; - } - } - return { - name: cn, - unit: cd.unit, - series: [...cd.series.keys()], - sortPosition: cd.sort_position, - totalPoints: cd.commits.length, - latestValues: latest, - }; - }); - meta[gn] = { - charts, - totalCharts: charts.length, - hasData: charts.length > 0, - summary: calcSummary(gn, gc), - }; - } +function getLoadingPayload() { + const status = benchmarks.status; return { - groups: meta, - totalCommits: commits.length, - commits: commits.map((c) => ({ - id: c.id, - message: c.message?.split("\n")[0] || "", - timestamp: c.timestamp, - author: c.author?.name || "Unknown", - })), - lastUpdated: new Date().toISOString(), + error: status.state === "error" ? "Initial refresh failed" : "Loading", + status: status.state, + ready: status.ready, + refreshing: status.refreshing, + hasData: status.hasData, + lastUpdated: status.lastUpdated, + lastRefreshStartedAt: status.lastRefreshStartedAt, + lastRefreshCompletedAt: status.lastRefreshCompletedAt, + lastRefreshError: status.lastRefreshError, }; } -// HTTP handlers -const json = (res, code, data) => { - res.writeHead(code, { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - }); - res.end(JSON.stringify(data)); -}; +function triggerRefresh() { + console.log("Refreshing data..."); + benchmarks.refresh().catch(() => {}); +} -function serveFile(res, fp) { - fs.readFile(fp, (err, data) => { +function serveFile(res, filePath) { + fs.readFile(filePath, (err, data) => { if (err) { res.writeHead(err.code === "ENOENT" ? 404 : 500); - return res.end(err.code === "ENOENT" ? "Not Found" : "Error"); + res.end(err.code === "ENOENT" ? "Not Found" : "Error"); + return; } - const ext = path.extname(fp).toLowerCase(); - const hdrs = { "Content-Type": MIME[ext] || "application/octet-stream" }; + + const ext = path.extname(filePath).toLowerCase(); + const headers = { "Content-Type": MIME[ext] || "application/octet-stream" }; if (ext === ".js") { - hdrs["Cache-Control"] = "no-cache"; - hdrs["Pragma"] = "no-cache"; + headers["Cache-Control"] = "no-cache"; + headers.Pragma = "no-cache"; } - res.writeHead(200, hdrs); + + res.writeHead(200, headers); res.end(data); }); } -function handleData(res, group, chart, start, end, last, startIdx, endIdx) { - if (!store.downsampled) return json(res, 503, { error: "Loading" }); - const gd = store.downsampled[group]; - if (!gd) return json(res, 404, { error: "Group not found" }); - const cv = gd[chart]; - if (!cv) return json(res, 404, { error: "Chart not found" }); - - const full = cv["1x"]; - const ts = (c) => - typeof c?.timestamp === "number" - ? c.timestamp - : new Date(c?.timestamp).getTime(); - - let si = 0, - ei = full.commits.length - 1; - - // Support "last=N" parameter to get the last N commits - if (last && !start && !end && startIdx === null && endIdx === null) { - const n = parseInt(last, 10); - if (n > 0 && n < full.commits.length) { - si = full.commits.length - n; - } - } else if (startIdx !== null || endIdx !== null) { - // Support index-based range (startIdx, endIdx) - if (startIdx !== null) si = Math.max(0, parseInt(startIdx, 10)); - if (endIdx !== null) - ei = Math.min(full.commits.length - 1, parseInt(endIdx, 10)); - } else { - // Timestamp-based range - if (start) { - const t = +start, - i = full.commits.findIndex((c) => ts(c) >= t); - if (i !== -1) si = i; +async function handleData(res, groupName, chartName, params) { + try { + const payload = await benchmarks.getChartData(groupName, chartName, { + start: params.get("start"), + end: params.get("end"), + last: params.get("last"), + startIdx: params.has("startIdx") ? params.get("startIdx") : null, + endIdx: params.has("endIdx") ? params.get("endIdx") : null, + }); + json(res, 200, payload); + } catch (error) { + if (error.message === "Loading") { + const status = benchmarks.status; + json(res, 503, getLoadingPayload(), loadingHeaders(status)); + return; } - if (end) { - const t = +end; - for (let i = ei; i >= 0; i--) - if (ts(full.commits[i]) <= t) { - ei = i; - break; - } + if (error.statusCode === 404) { + json(res, 404, { error: error.message }); + return; } + console.error("Data request error:", error); + json(res, 500, { error: "Internal server error" }); } - - const len = ei - si + 1; - const ver = - len <= MAX_POINTS - ? "1x" - : len <= MAX_POINTS * 2 - ? "2x" - : len <= MAX_POINTS * 4 - ? "4x" - : "8x"; - const cd = cv[ver]; - const val = (d) => d?.value ?? (typeof d === "number" ? d : null); - - let commits, series; - if (ver === "1x") { - commits = full.commits.slice(si, ei + 1); - series = Object.fromEntries( - [...full.series].map(([n, d]) => [n, d.slice(si, ei + 1).map(val)]), - ); - } else { - const s = +ver[0], - dsi = Math.floor(si / s), - dei = Math.min(Math.ceil(ei / s), cd.commits.length - 1); - commits = cd.commits.slice(dsi, dei + 1); - series = Object.fromEntries( - [...cd.series].map(([n, d]) => [n, d.slice(dsi, dei + 1).map(val)]), - ); - } - - json(res, 200, { - group, - chart, - unit: cd.unit, - downsampleLevel: ver, - originalLength: full.commits.length, - requestedRange: { startIndex: si, endIndex: ei, length: len }, - commits: commits.map((c) => ({ - id: c.id, - message: c.message?.split("\n")[0] || "", - timestamp: c.timestamp, - author: c.author?.name || "Unknown", - url: c.url, - })), - series, - }); } const server = http.createServer((req, res) => { - const [path_, qs] = req.url.split("?"); - const params = new URLSearchParams(qs || ""); + const [pathName, rawQuery] = req.url.split("?"); + const params = new URLSearchParams(rawQuery || ""); if (req.method === "OPTIONS") { res.writeHead(204, { @@ -735,41 +137,79 @@ const server = http.createServer((req, res) => { "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "Content-Type", }); - return res.end(); + res.end(); + return; } - if (path_ === "/api/metadata") - return store.metadata - ? json(res, 200, store.metadata) - : json(res, 503, { error: "Loading" }); - - if (path_.startsWith("/api/data/")) { - const p = path_.slice(10).split("/"); - return handleData( + if (pathName === "/api/metadata") { + const metadata = benchmarks.metadata; + const status = benchmarks.status; + json( res, - decodeURIComponent(p[0] || ""), - decodeURIComponent(p.slice(1).join("/") || ""), - params.get("start"), - params.get("end"), - params.get("last"), - params.has("startIdx") ? params.get("startIdx") : null, - params.has("endIdx") ? params.get("endIdx") : null, + metadata ? 200 : 503, + metadata || getLoadingPayload(), + metadata ? {} : loadingHeaders(status), ); + return; + } + + if (pathName === "/api/health") { + const status = benchmarks.status; + json(res, status.ready ? 200 : 503, status, loadingHeaders(status)); + return; + } + + if (pathName.startsWith("/api/data/")) { + const segments = pathName.slice(10).split("/"); + handleData( + res, + decodeURIComponent(segments[0] || ""), + decodeURIComponent(segments.slice(1).join("/") || ""), + params, + ).catch((error) => { + console.error("Unhandled data handler error:", error); + json(res, 500, { error: "Internal server error" }); + }); + return; } - const fp = path.join(__dirname, "dist", path_ === "/" ? "index.html" : path_); - if (!fp.startsWith(__dirname) || fp.includes("/sample/")) { + const filePath = path.join( + __dirname, + "dist", + pathName === "/" ? "index.html" : pathName, + ); + if (!filePath.startsWith(__dirname) || filePath.includes("/sample/")) { res.writeHead(403); - return res.end("Forbidden"); + res.end("Forbidden"); + return; } - serveFile(res, fp); + serveFile(res, filePath); }); -async function start() { +function start() { console.log("Starting server..."); - await refresh(); - setInterval(refresh, REFRESH_INTERVAL); - server.listen(PORT, () => console.log(`Server at http://localhost:${PORT}`)); + server.listen(PORT, () => { + console.log(`Server at http://localhost:${PORT}`); + }); + triggerRefresh(); + refreshIntervalId = setInterval(triggerRefresh, Number(REFRESH_INTERVAL)); +} + +async function shutdown() { + if (refreshIntervalId) { + clearInterval(refreshIntervalId); + refreshIntervalId = null; + } + await benchmarks.close(); + server.close(); } -start().catch(console.error); +process.on("SIGINT", () => { + shutdown().finally(() => process.exit(0)); +}); + +process.on("SIGTERM", () => { + shutdown().finally(() => process.exit(0)); +}); + +start(); diff --git a/benchmarks-website/src/App.jsx b/benchmarks-website/src/App.jsx index 0df05bebf01..f5edf0e92e2 100644 --- a/benchmarks-website/src/App.jsx +++ b/benchmarks-website/src/App.jsx @@ -1,15 +1,13 @@ -import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; +import React, { useState, useEffect, useCallback, useMemo } from 'react'; import Header from './components/Header'; import Sidebar from './components/Sidebar'; import BenchmarkSection from './components/BenchmarkSection'; import Modal from './components/Modal'; -import { fetchMetadata } from './api'; import { BENCHMARK_CONFIGS, CATEGORY_TAGS } from './config'; +import useBenchmarkMetadata from './hooks/useBenchmarkMetadata'; export default function App() { - const [metadata, setMetadata] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); + const { metadata, loading, error } = useBenchmarkMetadata(); const [expandedGroups, setExpandedGroups] = useState(new Set()); const [sidebarOpen, setSidebarOpen] = useState(false); const [categoryFilter, setCategoryFilter] = useState('all'); @@ -17,28 +15,15 @@ export default function App() { const [viewMode, setViewMode] = useState('grid'); const [modalChart, setModalChart] = useState(null); const [showBackToTop, setShowBackToTop] = useState(false); - const metadataFetched = useRef(false); useEffect(() => { - if (metadataFetched.current) return; - metadataFetched.current = true; + if (!metadata?.groups) return; - async function loadMetadata() { - try { - const data = await fetchMetadata(); - setMetadata(data); - const params = new URLSearchParams(window.location.search); - if (params.get('expanded') === 'true' && data?.groups) { - setExpandedGroups(new Set(Object.keys(data.groups))); - } - } catch (err) { - setError(err.message); - } finally { - setLoading(false); - } + const params = new URLSearchParams(window.location.search); + if (params.get('expanded') === 'true') { + setExpandedGroups(new Set(Object.keys(metadata.groups))); } - loadMetadata(); - }, []); + }, [metadata]); useEffect(() => { const handleScroll = () => { @@ -55,7 +40,6 @@ export default function App() { const hash = window.location.hash; if (hash && hash.startsWith('#group-')) { const groupId = hash.slice(1); // Remove the '#' - const groupName = groupId.replace('group-', '').replace(/-/g, ' '); // Find the matching group (case-insensitive, handle hyphenated names) const matchingGroup = Object.keys(metadata.groups).find(name => diff --git a/benchmarks-website/src/api.js b/benchmarks-website/src/api.js index 042ea7a6f0b..7fbcf781ce3 100644 --- a/benchmarks-website/src/api.js +++ b/benchmarks-website/src/api.js @@ -1,9 +1,53 @@ const API_BASE = ''; -export async function fetchMetadata() { - const response = await fetch(`${API_BASE}/api/metadata`); - if (!response.ok) throw new Error(`Failed to fetch metadata: ${response.status}`); - return response.json(); +function parseRetryAfterMs(value) { + if (!value) return null; + + const seconds = Number.parseInt(value, 10); + if (Number.isFinite(seconds) && seconds >= 0) { + return seconds * 1000; + } + + const retryAt = Date.parse(value); + if (Number.isNaN(retryAt)) { + return null; + } + + return Math.max(0, retryAt - Date.now()); +} + +async function readResponse(response) { + const contentType = response.headers.get('content-type') || ''; + + if (contentType.includes('application/json')) { + return response.json(); + } + + const text = await response.text(); + return text ? { error: text } : null; +} + +export async function fetchMetadata(options = {}) { + const response = await fetch(`${API_BASE}/api/metadata`, { + cache: 'no-store', + signal: options.signal, + }); + const payload = await readResponse(response); + + if (!response.ok) { + const message = payload?.lastRefreshError + ? `Failed to fetch metadata: ${payload.lastRefreshError}` + : payload?.error + ? `Failed to fetch metadata: ${payload.error}` + : `Failed to fetch metadata: ${response.status}`; + const error = new Error(message); + error.status = response.status; + error.payload = payload; + error.retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after')); + throw error; + } + + return payload; } export async function fetchChartData(groupName, chartName, options = {}) { @@ -35,7 +79,19 @@ export async function fetchChartData(groupName, chartName, options = {}) { if (params.toString()) url += '?' + params.toString(); - const response = await fetch(url); - if (!response.ok) throw new Error(`Failed to fetch chart data: ${response.status}`); - return response.json(); + const response = await fetch(url, { cache: 'no-store' }); + const payload = await readResponse(response); + + if (!response.ok) { + const message = payload?.error + ? `Failed to fetch chart data: ${payload.error}` + : `Failed to fetch chart data: ${response.status}`; + const error = new Error(message); + error.status = response.status; + error.payload = payload; + error.retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after')); + throw error; + } + + return payload; } diff --git a/benchmarks-website/src/components/ChartContainer.jsx b/benchmarks-website/src/components/ChartContainer.jsx index 1dfb193e836..681192a09a4 100644 --- a/benchmarks-website/src/components/ChartContainer.jsx +++ b/benchmarks-website/src/components/ChartContainer.jsx @@ -277,7 +277,7 @@ export default function ChartContainer({ const { series, commits } = chartData; const datasets = []; - const labels = commits.map(c => formatDate(c.timestamp)); + const labels = commits.map(c => formatDate(c?.timestamp)); Object.entries(series).forEach(([seriesName, points]) => { // Apply removed datasets filter diff --git a/benchmarks-website/src/components/Header.jsx b/benchmarks-website/src/components/Header.jsx index 95edda98560..952cfd80087 100644 --- a/benchmarks-website/src/components/Header.jsx +++ b/benchmarks-website/src/components/Header.jsx @@ -1,4 +1,3 @@ -import React from 'react'; export default function Header({ sidebarOpen, @@ -25,8 +24,8 @@ export default function Header({ - - + + Vortex diff --git a/benchmarks-website/src/components/Modal.jsx b/benchmarks-website/src/components/Modal.jsx index b9bb7d419b1..f118e18f795 100644 --- a/benchmarks-website/src/components/Modal.jsx +++ b/benchmarks-website/src/components/Modal.jsx @@ -95,7 +95,7 @@ export default function Modal({ chartData, onClose }) { const { series, commits } = data; const datasets = []; - const labels = commits.map(c => formatDate(c.timestamp)); + const labels = commits.map(c => formatDate(c?.timestamp)); Object.entries(series).forEach(([seriesName, points]) => { if (config?.removedDatasets?.has(seriesName)) return; diff --git a/benchmarks-website/src/hooks/useBenchmarkMetadata.js b/benchmarks-website/src/hooks/useBenchmarkMetadata.js new file mode 100644 index 00000000000..f287560e0c7 --- /dev/null +++ b/benchmarks-website/src/hooks/useBenchmarkMetadata.js @@ -0,0 +1,75 @@ +import { useEffect, useState } from 'react'; +import { fetchMetadata } from '../api'; + +const METADATA_RETRY_DELAY_MS = 1000; + +const initialState = { + metadata: null, + loading: true, + error: null, +}; + +export default function useBenchmarkMetadata() { + const [state, setState] = useState(initialState); + + useEffect(() => { + let active = true; + let retryTimer = null; + let activeController = null; + + const clearRetry = () => { + if (retryTimer !== null) { + window.clearTimeout(retryTimer); + retryTimer = null; + } + }; + + const loadMetadata = async () => { + clearRetry(); + activeController?.abort(); + activeController = new AbortController(); + + try { + const metadata = await fetchMetadata({ signal: activeController.signal }); + if (!active) return; + + setState({ + metadata, + loading: false, + error: null, + }); + } catch (error) { + if (!active || error.name === 'AbortError') return; + + if (error.status === 503 && error.payload?.status !== 'error') { + setState((current) => ({ + metadata: current.metadata, + loading: true, + error: null, + })); + retryTimer = window.setTimeout( + loadMetadata, + error.retryAfterMs ?? METADATA_RETRY_DELAY_MS, + ); + return; + } + + setState({ + metadata: null, + loading: false, + error: error.message, + }); + } + }; + + loadMetadata(); + + return () => { + active = false; + clearRetry(); + activeController?.abort(); + }; + }, []); + + return state; +} diff --git a/benchmarks-website/store/benchmark-store.js b/benchmarks-website/store/benchmark-store.js new file mode 100644 index 00000000000..8f112d94f4d --- /dev/null +++ b/benchmarks-website/store/benchmark-store.js @@ -0,0 +1,354 @@ +import { DuckDBInstance } from "@duckdb/node-api"; +import { prepareInputFiles } from "./cache.js"; +import { queryRows, withConnection } from "./db.js"; +import { downsample, downsampleLevel } from "./downsample.js"; +import { buildMetadata, collectDiagnostics } from "./metadata.js"; +import { buildBootstrapSql } from "./sql.js"; +import { firstLine } from "./utils.js"; + +const CHART_RANGE_QUERY = ` + with requested_commits as ( + select commit_idx + from active_commits + where commit_idx between $start_idx and $end_idx + ), + requested_series as ( + select distinct series_name + from benchmark_points_active + where group_name = $group_name + and chart_name = $chart_name + ), + dense_points as ( + select + rs.series_name, + rc.commit_idx, + bpa.value + from requested_series rs + cross join requested_commits rc + left join benchmark_points_active bpa + on bpa.group_name = $group_name + and bpa.chart_name = $chart_name + and bpa.series_name = rs.series_name + and bpa.commit_idx = rc.commit_idx + ) + select + series_name, + list(value order by commit_idx) as values + from dense_points + group by 1 + order by 1 +`; + +async function buildStoreState({ instance, inputs }) { + await withConnection(instance, async (connection) => { + await connection.run("begin transaction"); + try { + await connection.run(buildBootstrapSql(inputs.dataPath, inputs.commitsPath)); + await connection.run("commit"); + } catch (error) { + try { + await connection.run("rollback"); + } catch { + // Best effort: keep the previous committed state if the rebuild failed. + } + throw error; + } + }); + + const lastUpdated = new Date().toISOString(); + const [{ commits, metadata, chartIndex }, diagnostics] = await Promise.all([ + buildMetadata(instance, lastUpdated), + withConnection(instance, collectDiagnostics), + ]); + + return { + commits, + metadata, + chartIndex, + lastUpdated, + diagnostics, + }; +} + +function commitTimestamp(commit) { + return typeof commit.timestamp === "number" + ? commit.timestamp + : new Date(commit.timestamp).getTime(); +} + +function resolveRequestedRange(commits, options) { + const totalCommits = commits.length; + let startIdx = 0; + let endIdx = totalCommits - 1; + + if ( + options.last && + !options.start && + !options.end && + options.startIdx == null && + options.endIdx == null + ) { + const count = Number.parseInt(options.last, 10); + if (count > 0 && count < totalCommits) { + startIdx = totalCommits - count; + } + } else if (options.startIdx != null || options.endIdx != null) { + if (options.startIdx != null) { + startIdx = Math.max(0, Number.parseInt(options.startIdx, 10)); + } + if (options.endIdx != null) { + endIdx = Math.min(totalCommits - 1, Number.parseInt(options.endIdx, 10)); + } + } else { + if (options.start) { + const startTs = Number(options.start); + const idx = commits.findIndex((commit) => commitTimestamp(commit) >= startTs); + if (idx !== -1) startIdx = idx; + } + + if (options.end) { + const endTs = Number(options.end); + for (let i = endIdx; i >= 0; i--) { + if (commitTimestamp(commits[i]) <= endTs) { + endIdx = i; + break; + } + } + } + } + + return { + startIdx: Math.max(0, Math.min(startIdx, totalCommits - 1)), + endIdx: Math.max(startIdx, Math.min(endIdx, totalCommits - 1)), + }; +} + +function serializeRequestedCommits(commits, startIdx, endIdx) { + return commits + .slice(startIdx, endIdx + 1) + .map(({ id, message, timestamp, author, url }) => ({ + id, + message: firstLine(message), + timestamp, + author, + url, + })); +} + +function buildSeriesMap(rows) { + return new Map( + rows.map((row) => [ + row.series_name, + (row.values || []).map((value) => (value == null ? null : value)), + ]), + ); +} + +export class BenchmarkStore { + constructor(options) { + this.options = options; + this.state = null; + this.instance = null; + this.instancePromise = null; + this.refreshPromise = null; + this.remoteCheckTimer = null; + this.lastRefreshStartedAt = null; + this.lastRefreshCompletedAt = null; + this.lastRefreshError = null; + } + + get metadata() { + return this.state?.metadata || null; + } + + get status() { + let state = "idle"; + + if (this.state) { + if (this.refreshPromise) { + state = "refreshing"; + } else if (this.lastRefreshError) { + state = "stale"; + } else { + state = "ready"; + } + } else if (this.lastRefreshError) { + state = "error"; + } else if (this.refreshPromise) { + state = "loading"; + } + + return { + state, + ready: Boolean(this.state), + refreshing: Boolean(this.refreshPromise), + hasData: Boolean(this.state?.metadata), + lastUpdated: this.state?.lastUpdated || null, + lastRefreshStartedAt: this.lastRefreshStartedAt, + lastRefreshCompletedAt: this.lastRefreshCompletedAt, + lastRefreshError: this.lastRefreshError, + }; + } + + async getInstance() { + if (this.instance) return this.instance; + + if (!this.instancePromise) { + this.instancePromise = DuckDBInstance.create(":memory:") + .then((instance) => { + this.instance = instance; + return instance; + }) + .finally(() => { + this.instancePromise = null; + }); + } + + return this.instancePromise; + } + + scheduleRemoteCheck() { + if (this.remoteCheckTimer) return; + + this.remoteCheckTimer = setTimeout(() => { + this.remoteCheckTimer = null; + this.refresh({ forceRemoteCheck: true }).catch(() => {}); + }, 0); + } + + async refresh({ forceRemoteCheck = false } = {}) { + if (this.refreshPromise) return this.refreshPromise; + + this.lastRefreshStartedAt = new Date().toISOString(); + this.lastRefreshError = null; + this.refreshPromise = (async () => { + const startedAt = Date.now(); + const hasState = Boolean(this.state); + const inputs = await prepareInputFiles({ + ...this.options, + preferCached: !hasState && !forceRemoteCheck, + forceRemoteCheck, + }); + + if (hasState && !inputs.changed) { + this.lastRefreshCompletedAt = new Date().toISOString(); + console.log( + `Refresh skipped in ${Date.now() - startedAt}ms (${inputs.source})`, + ); + return; + } + + const instance = await this.getInstance(); + const nextState = await buildStoreState({ instance, inputs }); + + console.log( + `Processed ${nextState.diagnostics.benchmarkCount} benchmarks, ${nextState.commits.length} commits`, + ); + + if (nextState.diagnostics.uncategorized.length > 0) { + console.log( + `Uncategorized benchmark prefixes (${nextState.diagnostics.uncategorized.length}):`, + nextState.diagnostics.uncategorized.join(", "), + ); + } + + const chartCounts = nextState.diagnostics.groupCounts + .map((row) => `${row.group_name}: ${row.chart_count}`) + .filter((entry) => !entry.endsWith(": 0")); + console.log("Charts per group:", chartCounts.join(", ")); + + this.state = nextState; + this.lastRefreshCompletedAt = nextState.lastUpdated; + + console.log( + `Refresh done in ${Date.now() - startedAt}ms (${nextState.diagnostics.missingCommits} missing commits, source: ${inputs.source})`, + ); + + if (inputs.deferRemoteCheck) { + console.log( + "Serving cached benchmark files for startup; scheduling remote revalidation", + ); + this.scheduleRemoteCheck(); + } + })() + .catch((error) => { + this.lastRefreshError = error?.message || String(error); + console.error("Refresh error:", error); + throw error; + }) + .finally(() => { + this.refreshPromise = null; + }); + + return this.refreshPromise; + } + + async close() { + if (this.remoteCheckTimer) { + clearTimeout(this.remoteCheckTimer); + this.remoteCheckTimer = null; + } + + this.state = null; + + if (this.instance) { + this.instance.closeSync(); + this.instance = null; + } + } + + async getChartData(groupName, chartName, options = {}) { + if (!this.state) { + throw new Error("Loading"); + } + + const chart = this.state.chartIndex.get(`${groupName}\u0000${chartName}`); + if (!chart) { + const error = new Error("Chart not found"); + error.statusCode = 404; + throw error; + } + + const { commits, metadata } = this.state; + const instance = this.instance; + if (!instance) { + throw new Error("Loading"); + } + + const { startIdx, endIdx } = resolveRequestedRange(commits, options); + const rows = await withConnection(instance, async (connection) => + queryRows(connection, CHART_RANGE_QUERY, { + group_name: groupName, + chart_name: chartName, + start_idx: startIdx, + end_idx: endIdx, + }), + ); + + const requestedCommits = serializeRequestedCommits(commits, startIdx, endIdx); + const selected = { + commits: requestedCommits, + series: buildSeriesMap(rows), + }; + const level = downsampleLevel(requestedCommits.length); + const sampled = + level === "1x" + ? selected + : downsample(selected, Number.parseInt(level, 10)); + + return { + group: groupName, + chart: chartName, + unit: chart.unit, + downsampleLevel: level, + originalLength: metadata.totalCommits, + requestedRange: { + startIndex: startIdx, + endIndex: endIdx, + length: endIdx - startIdx + 1, + }, + commits: sampled.commits, + series: Object.fromEntries(sampled.series), + }; + } +} diff --git a/benchmarks-website/store/cache.js b/benchmarks-website/store/cache.js new file mode 100644 index 00000000000..7141b353b24 --- /dev/null +++ b/benchmarks-website/store/cache.js @@ -0,0 +1,174 @@ +import fs from "fs"; +import fsp from "fs/promises"; +import os from "os"; +import path from "path"; +import { Readable } from "stream"; +import { pipeline } from "stream/promises"; +import { fileURLToPath } from "url"; +import { DEFAULT_CACHE_DIR_NAME } from "./constants.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const WEBSITE_DIR = path.dirname(__dirname); + +function cachePaths(cacheDir) { + return { + dataPath: path.join(cacheDir, "data.json.gz"), + commitsPath: path.join(cacheDir, "commits.json"), + manifestPath: path.join(cacheDir, "manifest.json"), + }; +} + +async function pathExists(filePath) { + try { + await fsp.access(filePath); + return true; + } catch { + return false; + } +} + +async function readJsonFile(filePath) { + try { + const raw = await fsp.readFile(filePath, "utf8"); + return JSON.parse(raw); + } catch (error) { + if (error.code === "ENOENT") return null; + throw error; + } +} + +async function writeJsonFileAtomic(filePath, value) { + const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; + await fsp.writeFile(tempPath, JSON.stringify(value, null, 2)); + await fsp.rename(tempPath, filePath); +} + +async function downloadToFile(url, destination, metadata = {}) { + const headers = {}; + if (metadata.etag) headers["If-None-Match"] = metadata.etag; + if (metadata.lastModified) headers["If-Modified-Since"] = metadata.lastModified; + + const response = await fetch(url, { headers }); + if (response.status === 304) { + return { + changed: false, + metadata: { + ...metadata, + checkedAt: new Date().toISOString(), + }, + }; + } + + if (!response.ok) { + throw new Error(`Fetch failed: ${url} ${response.status}`); + } + + if (!response.body) { + throw new Error(`Fetch failed: ${url} empty body`); + } + + const tempPath = `${destination}.tmp-${process.pid}-${Date.now()}`; + try { + await pipeline(Readable.fromWeb(response.body), fs.createWriteStream(tempPath)); + await fsp.rename(tempPath, destination); + } finally { + if (await pathExists(tempPath)) { + await fsp.unlink(tempPath); + } + } + + return { + changed: true, + metadata: { + url, + etag: response.headers.get("etag"), + lastModified: response.headers.get("last-modified"), + fetchedAt: new Date().toISOString(), + checkedAt: new Date().toISOString(), + }, + }; +} + +export async function prepareInputFiles({ + dataUrl, + commitsUrl, + useLocalData, + cacheDir, + preferCached = false, + forceRemoteCheck = false, +}) { + if (useLocalData) { + return { + dataPath: path.join(WEBSITE_DIR, "sample/data.json"), + commitsPath: path.join(WEBSITE_DIR, "sample/commits.json"), + changed: true, + source: "local", + deferRemoteCheck: false, + }; + } + + const resolvedCacheDir = + cacheDir || path.join(os.tmpdir(), DEFAULT_CACHE_DIR_NAME); + await fsp.mkdir(resolvedCacheDir, { recursive: true }); + + const { dataPath, commitsPath, manifestPath } = cachePaths(resolvedCacheDir); + const manifest = (await readJsonFile(manifestPath)) || {}; + const [hasDataFile, hasCommitsFile] = await Promise.all([ + pathExists(dataPath), + pathExists(commitsPath), + ]); + const hasCachedFiles = hasDataFile && hasCommitsFile; + + if (preferCached && hasCachedFiles && !forceRemoteCheck) { + return { + dataPath, + commitsPath, + changed: true, + source: "cache", + deferRemoteCheck: true, + }; + } + + try { + const [dataResult, commitsResult] = await Promise.all([ + downloadToFile(dataUrl, dataPath, manifest.data || {}), + downloadToFile(commitsUrl, commitsPath, manifest.commits || {}), + ]); + + await writeJsonFileAtomic(manifestPath, { + version: 1, + dataUrl, + commitsUrl, + data: dataResult.metadata, + commits: commitsResult.metadata, + updatedAt: new Date().toISOString(), + }); + + return { + dataPath, + commitsPath, + changed: !hasCachedFiles || dataResult.changed || commitsResult.changed, + source: + !hasCachedFiles || dataResult.changed || commitsResult.changed + ? "remote" + : "cache", + deferRemoteCheck: false, + }; + } catch (error) { + if (hasCachedFiles) { + console.warn( + `Falling back to cached benchmark files after refresh failed: ${error.message}`, + ); + return { + dataPath, + commitsPath, + changed: false, + source: "stale-cache", + deferRemoteCheck: false, + }; + } + + throw error; + } +} diff --git a/benchmarks-website/store/constants.js b/benchmarks-website/store/constants.js new file mode 100644 index 00000000000..fa5799bc3e5 --- /dev/null +++ b/benchmarks-website/store/constants.js @@ -0,0 +1,20 @@ +import { FAN_OUT_GROUPS, QUERY_SUITES } from "../src/config.js"; + +export const MAX_POINTS = 200; +export const DEFAULT_CACHE_DIR_NAME = "vortex-benchmarks-website-cache"; + +export const GROUPS = [ + "Random Access", + "Compression", + "Compression Size", + ...QUERY_SUITES.filter((suite) => !suite.skip && !suite.fanOut).map( + (suite) => suite.displayName, + ), + ...FAN_OUT_GROUPS, +]; + +export const QUERY_GROUP_EXCLUSIONS = new Set([ + "Random Access", + "Compression", + "Compression Size", +]); diff --git a/benchmarks-website/store/db.js b/benchmarks-website/store/db.js new file mode 100644 index 00000000000..543c33e2ab7 --- /dev/null +++ b/benchmarks-website/store/db.js @@ -0,0 +1,33 @@ +function normalizeValue(value) { + if (typeof value === "bigint") { + const asNumber = Number(value); + return Number.isSafeInteger(asNumber) ? asNumber : value.toString(); + } + + if (Array.isArray(value)) { + return value.map(normalizeValue); + } + + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, normalizeValue(child)]), + ); + } + + return value; +} + +export async function queryRows(connection, sql, values) { + const result = await connection.run(sql, values); + const rows = await result.getRowObjectsJS(); + return rows.map((row) => normalizeValue(row)); +} + +export async function withConnection(instance, fn) { + const connection = await instance.connect(); + try { + return await fn(connection); + } finally { + connection.closeSync(); + } +} diff --git a/benchmarks-website/store/downsample.js b/benchmarks-website/store/downsample.js new file mode 100644 index 00000000000..47a774184cc --- /dev/null +++ b/benchmarks-website/store/downsample.js @@ -0,0 +1,68 @@ +import { LTTB } from "downsample"; +import { MAX_POINTS } from "./constants.js"; + +function lttbIndices(seriesMap, target) { + const keys = [...seriesMap.keys()]; + if (!keys.length) return []; + + const length = seriesMap.get(keys[0])?.length || 0; + if (length <= target) return [...Array(length).keys()]; + + const averages = Array(length); + for (let i = 0; i < length; i++) { + let sum = 0; + let count = 0; + + for (const series of seriesMap.values()) { + const value = series[i]?.value ?? series[i]; + if (value != null && !Number.isNaN(value)) { + sum += value; + count++; + } + } + + averages[i] = [i, count ? sum / count : 0]; + } + + const indices = LTTB(averages, target).map((point) => Math.round(point[0])); + if (!indices.includes(0)) indices.unshift(0); + if (!indices.includes(length - 1)) indices.push(length - 1); + return indices.sort((a, b) => a - b); +} + +export function downsample(data, factor) { + const target = Math.ceil(data.commits.length / factor); + if (target >= data.commits.length) return data; + + const indices = [...new Set( + lttbIndices(data.series, target).filter( + (index) => + Number.isInteger(index) && + index >= 0 && + index < data.commits.length, + ), + )].sort((a, b) => a - b); + + if (!indices.length) return data; + + const series = new Map(); + for (const [seriesName, values] of data.series) { + series.set( + seriesName, + indices.map((index) => values[index]), + ); + } + + return { + ...data, + commits: indices.map((index) => data.commits[index]), + series, + }; +} + +export function downsampleLevel(length) { + if (length <= MAX_POINTS) return "1x"; + if (length <= MAX_POINTS * 2) return "2x"; + if (length <= MAX_POINTS * 4) return "4x"; + return "8x"; +} diff --git a/benchmarks-website/store/metadata.js b/benchmarks-website/store/metadata.js new file mode 100644 index 00000000000..296e7df2c6a --- /dev/null +++ b/benchmarks-website/store/metadata.js @@ -0,0 +1,401 @@ +import { GROUPS, QUERY_GROUP_EXCLUSIONS } from "./constants.js"; +import { queryRows, withConnection } from "./db.js"; +import { firstLine } from "./utils.js"; + +const COMMITS_QUERY = ` + select + commit_idx, + id, + message, + timestamp, + author, + url + from active_commits + order by commit_idx +`; + +const CHART_ROWS_QUERY = ` + select + cd.group_name, + cd.chart_name, + cd.unit, + cd.sort_position, + list(cslv.series_name order by cslv.series_name) as series_names, + json_group_object(cslv.series_name, cslv.latest_value) as latest_values + from chart_defs cd + join chart_series_latest_values cslv + on cslv.group_name = cd.group_name + and cslv.chart_name = cd.chart_name + group by 1, 2, 3, 4 + order by 1, 4, 2 +`; + +const RANDOM_ACCESS_SUMMARY_QUERY = ` + with selected_chart as ( + select chart_name + from chart_defs + where group_name = 'Random Access' + order by sort_position, chart_name + limit 1 + ) + select + series_name as name, + value as time, + value / min(value) over () as ratio + from chart_latest_values + where group_name = 'Random Access' + and chart_name = (select chart_name from selected_chart) + order by value, series_name +`; + +const COMPRESSION_SUMMARY_QUERY = ` + with anchor_chart as ( + select chart_name + from chart_defs + where group_name = 'Compression' + and chart_name in ( + 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME', + 'VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME' + ) + order by case + when chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' then 0 + else 1 + end + limit 1 + ), + anchor_idx as ( + select latest_commit_idx as commit_idx + from chart_latest_idx + where group_name = 'Compression' + and chart_name = (select chart_name from anchor_chart) + ) + select + ( + select geomean(1.0 / value) + from benchmark_points_active + where group_name = 'Compression' + and chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' + and commit_idx = (select commit_idx from anchor_idx) + and lower(series_name) not like '%wide table%' + and value > 0 + ) as compress_ratio, + ( + select geomean(1.0 / value) + from benchmark_points_active + where group_name = 'Compression' + and chart_name = 'VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME' + and commit_idx = (select commit_idx from anchor_idx) + and lower(series_name) not like '%wide table%' + and value > 0 + ) as decompress_ratio, + ( + select count(*) + from benchmark_points_active + where group_name = 'Compression' + and chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' + and commit_idx = (select commit_idx from anchor_idx) + and lower(series_name) not like '%wide table%' + and value > 0 + ) as dataset_count +`; + +const COMPRESSION_SIZE_SUMMARY_QUERY = ` + with anchor_idx as ( + select latest_commit_idx as commit_idx + from chart_latest_idx + where group_name = 'Compression Size' + and chart_name = 'VORTEX:PARQUET ZSTD SIZE' + ) + select + min(value) as min_ratio, + geomean(value) as mean_ratio, + max(value) as max_ratio, + count(*) as dataset_count + from benchmark_points_active + where group_name = 'Compression Size' + and chart_name = 'VORTEX:PARQUET ZSTD SIZE' + and commit_idx = (select commit_idx from anchor_idx) + and lower(series_name) not like '%wide table%' + and value > 0 +`; + +const QUERY_SUMMARY_QUERY = ` + with per_series as ( + select * + from chart_series_latest_values + where group_name not in ( + 'Random Access', + 'Compression', + 'Compression Size' + ) + ), + chart_bases as ( + select + group_name, + chart_name, + min(latest_value) as base_value + from per_series + group by 1, 2 + ), + series_totals as ( + select + group_name, + series_name, + sum(latest_value) as total_runtime, + max(latest_value) as max_runtime + from per_series + group by 1, 2 + ), + group_series as ( + select distinct group_name, series_name + from per_series + ), + ratios as ( + select + gs.group_name, + gs.series_name, + ((10.0 + coalesce(ps.latest_value, greatest(300000.0, st.max_runtime) * 2.0)) + / (10.0 + cb.base_value)) as ratio + from group_series gs + join series_totals st + on st.group_name = gs.group_name + and st.series_name = gs.series_name + join chart_defs cd + on cd.group_name = gs.group_name + join chart_bases cb + on cb.group_name = cd.group_name + and cb.chart_name = cd.chart_name + left join per_series ps + on ps.group_name = cd.group_name + and ps.chart_name = cd.chart_name + and ps.series_name = gs.series_name + ) + select + gs.group_name, + gs.series_name as name, + geomean(r.ratio) as score, + st.total_runtime + from group_series gs + join ratios r + on r.group_name = gs.group_name + and r.series_name = gs.series_name + join series_totals st + on st.group_name = gs.group_name + and st.series_name = gs.series_name + group by 1, 2, 4 + order by 1, 3, 4, 2 +`; + +const DIAGNOSTICS_COUNTS_QUERY = ` + select + (select count(*) from raw_benchmarks) as benchmark_count, + ( + select count(*) + from classified_benchmarks cb + left join commit_dim cd + on cd.id = cb.resolved_commit_id + where cb.resolved_commit_id is null + or cd.id is null + ) as missing_commits +`; + +const UNCATEGORIZED_QUERY = ` + select distinct split_part(name, '/', 1) as prefix + from classified_benchmarks + where group_name is null + order by prefix + limit 20 +`; + +const GROUP_COUNTS_QUERY = ` + select + group_name, + count(*) as chart_count + from chart_defs + group by 1 + order by 1 +`; + +function buildEmptyGroups() { + return Object.fromEntries( + GROUPS.map((groupName) => [ + groupName, + { + charts: [], + totalCharts: 0, + hasData: false, + summary: null, + }, + ]), + ); +} + +function buildChartIndex(groupMap) { + const chartIndex = new Map(); + for (const [groupName, groupData] of Object.entries(groupMap)) { + for (const chart of groupData.charts) { + chartIndex.set(`${groupName}\u0000${chart.name}`, chart); + } + } + return chartIndex; +} + +function appendChartRows(groups, chartRows, totalCommits) { + for (const row of chartRows) { + const group = groups[row.group_name]; + if (!group) continue; + + group.charts.push({ + name: row.chart_name, + unit: row.unit, + series: row.series_names || [], + sortPosition: row.sort_position, + totalPoints: totalCommits, + latestValues: row.latest_values || {}, + }); + } +} + +function applyRandomAccessSummary(groups, rows) { + if (!rows.length) return; + + groups["Random Access"].summary = { + type: "randomAccess", + title: "Random Access Performance", + rankings: rows.map((row) => ({ + name: row.name, + time: row.time, + ratio: row.ratio, + })), + explanation: "Random access time | Ratio to fastest (lower is better)", + }; +} + +function applyCompressionSummary(groups, row) { + if (!row?.compress_ratio && !row?.decompress_ratio) return; + + groups.Compression.summary = { + type: "compression", + title: "Compression Throughput vs Parquet", + compressRatio: row.compress_ratio, + decompressRatio: row.decompress_ratio, + datasetCount: row.dataset_count, + explanation: "Inverse geomean of Vortex/Parquet ratios (higher is better)", + }; +} + +function applyCompressionSizeSummary(groups, row) { + if (!row?.mean_ratio) return; + + groups["Compression Size"].summary = { + type: "compressionSize", + title: "Compression Size Summary", + minRatio: row.min_ratio, + meanRatio: row.mean_ratio, + maxRatio: row.max_ratio, + datasetCount: row.dataset_count, + explanation: "Geomean of Vortex/Parquet size ratios (lower is better)", + }; +} + +function applyQuerySummary(groups, rows) { + for (const row of rows) { + if (QUERY_GROUP_EXCLUSIONS.has(row.group_name)) continue; + + const group = groups[row.group_name]; + if (!group) continue; + + if (!group.summary) { + group.summary = { + type: "queryBenchmark", + title: "Performance Summary", + rankings: [], + explanation: "Geomean of query time ratio to fastest (lower is better)", + }; + } + + group.summary.rankings.push({ + name: row.name, + score: row.score, + totalRuntime: row.total_runtime, + }); + } +} + +function finalizeGroups(groups) { + for (const group of Object.values(groups)) { + group.totalCharts = group.charts.length; + group.hasData = group.charts.length > 0; + } +} + +async function fetchCoreMetadata(connection) { + const commits = await queryRows(connection, COMMITS_QUERY); + const chartRows = await queryRows(connection, CHART_ROWS_QUERY); + + return { commits, chartRows }; +} + +async function fetchSummaryMetadata(connection) { + const randomAccessRows = await queryRows(connection, RANDOM_ACCESS_SUMMARY_QUERY); + const compressionRows = await queryRows(connection, COMPRESSION_SUMMARY_QUERY); + const compressionSizeRows = await queryRows( + connection, + COMPRESSION_SIZE_SUMMARY_QUERY, + ); + + return { + randomAccessRows, + compressionRows, + compressionSizeRows, + }; +} + +export async function buildMetadata(instance, lastUpdated) { + const [ + { commits, chartRows }, + { randomAccessRows, compressionRows, compressionSizeRows }, + querySummaryRows, + ] = await Promise.all([ + withConnection(instance, fetchCoreMetadata), + withConnection(instance, fetchSummaryMetadata), + withConnection(instance, (connection) => queryRows(connection, QUERY_SUMMARY_QUERY)), + ]); + + const groups = buildEmptyGroups(); + appendChartRows(groups, chartRows, commits.length); + applyRandomAccessSummary(groups, randomAccessRows); + applyCompressionSummary(groups, compressionRows[0]); + applyCompressionSizeSummary(groups, compressionSizeRows[0]); + applyQuerySummary(groups, querySummaryRows); + finalizeGroups(groups); + + return { + commits, + metadata: { + groups, + totalCommits: commits.length, + commits: commits.map(({ id, message, timestamp, author }) => ({ + id, + message: firstLine(message), + timestamp, + author, + })), + lastUpdated, + }, + chartIndex: buildChartIndex(groups), + }; +} + +export async function collectDiagnostics(connection) { + const [counts] = await queryRows(connection, DIAGNOSTICS_COUNTS_QUERY); + const uncategorized = await queryRows(connection, UNCATEGORIZED_QUERY); + const groupCounts = await queryRows(connection, GROUP_COUNTS_QUERY); + + return { + benchmarkCount: counts?.benchmark_count ?? 0, + missingCommits: counts?.missing_commits ?? 0, + uncategorized: uncategorized.map((row) => row.prefix), + groupCounts, + }; +} diff --git a/benchmarks-website/store/sql.js b/benchmarks-website/store/sql.js new file mode 100644 index 00000000000..907a5831267 --- /dev/null +++ b/benchmarks-website/store/sql.js @@ -0,0 +1,479 @@ +import { ENGINE_RENAMES, QUERY_SUITES } from "../src/config.js"; +import { GROUPS } from "./constants.js"; + +function sqlStringLiteral(value) { + return `'${String(value).replace(/'/g, "''")}'`; +} + +function sqlValue(value) { + if (value === null || value === undefined) return "NULL"; + if (typeof value === "boolean") return value ? "TRUE" : "FALSE"; + if (typeof value === "number") return Number.isFinite(value) ? String(value) : "NULL"; + return sqlStringLiteral(value); +} + +function sqlValues(rows) { + return rows + .map((row) => `(${row.map((value) => sqlValue(value)).join(", ")})`) + .join(",\n"); +} + +function joinStatements(statements) { + return statements + .map((statement) => statement.trim().replace(/;?$/, ";")) + .join("\n\n"); +} + +function randomAccessCondition(expression = "name_lower") { + return `(starts_with(${expression}, 'random-access/') or starts_with(${expression}, 'random access/'))`; +} + +function renameSeriesInSql(sourceExpression) { + return `coalesce( + (select dst from engine_renames er where er.src = lower(${sourceExpression}) limit 1), + ${sourceExpression} + )`; +} + +function buildQuerySuitesStatement() { + const rows = QUERY_SUITES.map((suite) => [ + suite.prefix, + suite.displayName || null, + suite.queryPrefix || null, + suite.datasetKey || null, + Boolean(suite.fanOut), + Boolean(suite.skip), + ]); + + return ` +create or replace table query_suites as +select * from ( + values + ${sqlValues(rows)} +) as suites(prefix, display_name, query_prefix, dataset_key, fan_out, skip) +`; +} + +function buildValidGroupsStatement() { + const rows = GROUPS.map((groupName) => [groupName]); + + return ` +create or replace table valid_groups as +select * from ( + values + ${sqlValues(rows)} +) as groups(group_name) +`; +} + +function buildEngineRenamesStatement() { + const rows = Object.entries(ENGINE_RENAMES); + + return ` +create or replace table engine_renames as +select * from ( + values + ${sqlValues(rows)} +) as renames(src, dst) +`; +} + +function buildRawCommitsViewStatement(commitsPath) { + return ` +create or replace view raw_commits as +select * +from read_json( + ${sqlStringLiteral(commitsPath)}, + format = 'newline_delimited', + compression = 'auto_detect', + columns = { + id: 'VARCHAR', + message: 'VARCHAR', + timestamp: 'VARCHAR', + author: 'JSON', + url: 'VARCHAR' + } +) +`; +} + +function buildCommitDimStatement() { + return ` +create or replace table commit_dim as +select + row_number() over (order by commit_ts nulls last, id) - 1 as commit_idx, + id, + message, + timestamp as timestamp_text, + commit_ts, + author, + url +from ( + select + id, + message, + timestamp, + try_cast(timestamp as timestamptz) as commit_ts, + coalesce(json_extract_string(author, '$.name'), 'Unknown') as author, + url + from raw_commits +) +`; +} + +function buildRawBenchmarksViewStatement(dataPath) { + return ` +create or replace view raw_benchmarks as +select + row_number() over () as benchmark_row, + * +from read_json( + ${sqlStringLiteral(dataPath)}, + format = 'newline_delimited', + compression = 'auto_detect', + columns = { + name: 'VARCHAR', + unit: 'VARCHAR', + value: 'DOUBLE', + storage: 'VARCHAR', + dataset: 'JSON', + commit: 'JSON', + commit_id: 'VARCHAR' + } +) +`; +} + +function buildBenchmarksBaseStatement() { + return ` +create or replace table benchmarks_base as +select + benchmark_row, + name, + lower(name) as name_lower, + split_part(name, '/', 1) as part1, + lower(split_part(name, '/', 1)) as part1_lower, + coalesce(nullif(split_part(name, '/', 2), ''), 'default') as part2, + coalesce(nullif(split_part(name, '/', 3), ''), 'default') as part3, + coalesce(nullif(split_part(name, '/', 4), ''), 'default') as part4, + array_length(string_split(name, '/')) as part_count, + unit, + value as raw_value, + storage, + dataset as dataset_json, + commit as commit_json, + commit_id +from raw_benchmarks +`; +} + +function buildMatchedSuitesStatement() { + return ` +create or replace table matched_suites as +select + b.benchmark_row, + s.prefix, + s.display_name, + s.query_prefix, + s.dataset_key, + s.fan_out +from benchmarks_base b +join query_suites s + on (b.name_lower like s.prefix || '_q%' or b.name_lower like s.prefix || '/%') +where not s.skip +qualify row_number() over (partition by b.benchmark_row order by length(s.prefix) desc) = 1 +`; +} + +function buildGroupNameExpressionSql() { + return ` +case + when ${randomAccessCondition()} then 'Random Access' + when starts_with(name_lower, 'vortex size/') + or starts_with(name_lower, 'vortex-file-compressed size/') + or starts_with(name_lower, 'parquet size/') + or starts_with(name_lower, 'lance size/') + or contains(name_lower, ':raw size/') + or contains(name_lower, ':parquet-zstd size/') + or contains(name_lower, ':lance size/') + then 'Compression Size' + when starts_with(name_lower, 'compress time/') + or starts_with(name_lower, 'decompress time/') + or starts_with(name_lower, 'parquet_rs-zstd compress') + or starts_with(name_lower, 'parquet_rs-zstd decompress') + or starts_with(name_lower, 'lance compress') + or starts_with(name_lower, 'lance decompress') + or starts_with(name_lower, 'vortex:lance ratio') + or starts_with(name_lower, 'vortex:parquet-zstd ratio') + or starts_with(name_lower, 'vortex:raw ratio') + then 'Compression' + when suite_prefix is not null and not suite_fan_out then suite_display_name + when suite_prefix is not null and suite_fan_out then + suite_display_name + || ' (' + || case when upper(coalesce(storage, '')) = 'S3' then 'S3' else 'NVMe' end + || ') (SF=' + || cast(cast(round(coalesce(try_cast(suite_scale_factor as double), 1.0)) as bigint) as varchar) + || ')' + else null +end +`; +} + +function buildRawChartNameExpressionSql() { + return ` +case + when ${randomAccessCondition()} and part_count = 4 + then upper(replace(replace(part2 || '/' || part3, '_', ' '), '-', ' ')) + when ${randomAccessCondition()} and part_count = 2 + then 'RANDOM ACCESS' + when suite_prefix is not null + and regexp_extract(part1_lower, '^' || suite_prefix || '[_ ]?q([0-9]+)', 1) <> '' + then suite_query_prefix || ' Q' || cast(cast(regexp_extract(part1_lower, '^' || suite_prefix || '[_ ]?q([0-9]+)', 1) as integer) as varchar) + else upper(replace(replace(part1, '_', ' '), '-', ' ')) +end +`; +} + +function buildRawSeriesNameExpressionSql() { + return ` +case + when ${randomAccessCondition()} and part_count = 4 then part4 + else part2 +end +`; +} + +function buildRawUnitExpressionSql() { + return ` +case + when unit is not null then unit + when contains(name_lower, ' size/') then 'bytes' + when contains(name_lower, ' ratio ') then 'ratio' + else 'ns' +end +`; +} + +function buildSortPositionExpressionSql() { + return ` +coalesce( + try_cast(nullif(regexp_extract(part1_lower, 'q([0-9]+)$', 1), '') as integer), + 0 +) +`; +} + +function buildChartNameExpressionSql() { + return ` +case + when group_name = 'Compression Size' and raw_chart_name = 'VORTEX FILE COMPRESSED SIZE' then 'VORTEX SIZE' + else raw_chart_name +end +`; +} + +function buildDisplayUnitExpressionSql() { + return ` +case + when raw_unit = 'ns' then 'ms/iter' + when raw_unit = 'bytes' then 'MiB' + else raw_unit +end +`; +} + +function buildDisplayValueExpressionSql() { + return ` +case + when raw_unit = 'ns' then raw_value / 1000000.0 + when raw_unit = 'bytes' then raw_value / 1048576.0 + else raw_value +end +`; +} + +function buildClassifiedBenchmarksStatement() { + return ` +create or replace table classified_benchmarks as +with base as ( + select + b.*, + ms.prefix as suite_prefix, + ms.display_name as suite_display_name, + ms.query_prefix as suite_query_prefix, + ms.dataset_key as suite_dataset_key, + coalesce(ms.fan_out, false) as suite_fan_out, + coalesce(json_extract_string(b.commit_json, '$.id'), b.commit_id) as resolved_commit_id, + case + when ms.dataset_key = 'tpch' then json_extract_string(b.dataset_json, '$.tpch.scale_factor') + when ms.dataset_key = 'tpcds' then json_extract_string(b.dataset_json, '$.tpcds.scale_factor') + else null + end as suite_scale_factor + from benchmarks_base b + left join matched_suites ms using (benchmark_row) +), +named as ( + select + *, + ${buildGroupNameExpressionSql()} as group_name, + ${buildRawChartNameExpressionSql()} as raw_chart_name, + ${buildRawSeriesNameExpressionSql()} as raw_series_name, + ${buildRawUnitExpressionSql()} as raw_unit, + ${buildSortPositionExpressionSql()} as sort_position + from base +), +renamed as ( + select + n.*, + ${renameSeriesInSql("n.raw_series_name")} as series_name + from named n +) +select + benchmark_row, + name, + resolved_commit_id, + group_name, + ${buildChartNameExpressionSql()} as chart_name, + series_name, + sort_position, + ${buildDisplayUnitExpressionSql()} as unit, + ${buildDisplayValueExpressionSql()} as value +from renamed +`; +} + +function buildBenchmarkPointsStatement() { + return ` +create or replace table benchmark_points as +select + cb.group_name, + cb.chart_name, + cb.series_name, + cb.sort_position, + cb.unit, + cb.value, + cd.commit_idx +from classified_benchmarks cb +join valid_groups vg on vg.group_name = cb.group_name +join commit_dim cd on cd.id = cb.resolved_commit_id +where cb.group_name is not null + and cb.chart_name not like '%PARQUET-UNC%' + and cb.name not like '% throughput%' + and cb.value is not null +`; +} + +function buildActiveCommitsStatement() { + return ` +create or replace table active_commits as +with first_active as ( + select coalesce(min(commit_idx), 0) as min_commit_idx + from benchmark_points +) +select + cd.commit_idx as original_commit_idx, + cd.commit_idx - fa.min_commit_idx as commit_idx, + cd.id, + cd.message, + cd.timestamp_text as timestamp, + cd.author, + cd.url +from commit_dim cd +cross join first_active fa +where cd.commit_idx >= fa.min_commit_idx +order by cd.commit_idx +`; +} + +function buildBenchmarkPointsActiveStatement() { + return ` +create or replace table benchmark_points_active as +select + bp.group_name, + bp.chart_name, + bp.series_name, + bp.sort_position, + bp.unit, + bp.value, + ac.commit_idx +from benchmark_points bp +join active_commits ac + on ac.original_commit_idx = bp.commit_idx +`; +} + +function buildChartDefinitionsStatement() { + return ` +create or replace table chart_defs as +select + group_name, + chart_name, + min(sort_position) as sort_position, + min(unit) as unit +from benchmark_points_active +group by 1, 2 +`; +} + +function buildChartLatestIndexStatement() { + return ` +create or replace table chart_latest_idx as +select + group_name, + chart_name, + max(commit_idx) as latest_commit_idx +from benchmark_points_active +group by 1, 2 +`; +} + +function buildChartLatestValuesStatement() { + return ` +create or replace table chart_latest_values as +select + bpa.group_name, + bpa.chart_name, + bpa.series_name, + bpa.value +from benchmark_points_active bpa +join chart_latest_idx cli + on cli.group_name = bpa.group_name + and cli.chart_name = bpa.chart_name + and cli.latest_commit_idx = bpa.commit_idx +`; +} + +function buildChartSeriesLatestValuesStatement() { + return ` +create or replace table chart_series_latest_values as +select + group_name, + chart_name, + series_name, + arg_max(value, commit_idx) as latest_value +from benchmark_points_active +group by 1, 2, 3 +`; +} + +export function buildBootstrapSql(dataPath, commitsPath) { + return joinStatements([ + buildQuerySuitesStatement(), + buildValidGroupsStatement(), + buildEngineRenamesStatement(), + buildRawCommitsViewStatement(commitsPath), + buildCommitDimStatement(), + buildRawBenchmarksViewStatement(dataPath), + buildBenchmarksBaseStatement(), + buildMatchedSuitesStatement(), + buildClassifiedBenchmarksStatement(), + buildBenchmarkPointsStatement(), + buildActiveCommitsStatement(), + buildBenchmarkPointsActiveStatement(), + buildChartDefinitionsStatement(), + buildChartLatestIndexStatement(), + buildChartLatestValuesStatement(), + buildChartSeriesLatestValuesStatement(), + ]); +} diff --git a/benchmarks-website/store/utils.js b/benchmarks-website/store/utils.js new file mode 100644 index 00000000000..462c974fbce --- /dev/null +++ b/benchmarks-website/store/utils.js @@ -0,0 +1,3 @@ +export function firstLine(message) { + return String(message || "").split("\n")[0] || ""; +} From cc06c602286ac58fd926c6e12c431cca1bd30577 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 24 Apr 2026 15:11:54 +0100 Subject: [PATCH 186/250] Revert "DuckDB backend for benchmarks website" (#7623) Reverts vortex-data/vortex#7491, seems like it still needs some more work --- benchmarks-website/Dockerfile | 5 +- benchmarks-website/ETL.md | 302 ------- benchmarks-website/README.md | 206 ----- benchmarks-website/SCHEMA.md | 298 ------- benchmarks-website/package-lock.json | 102 --- benchmarks-website/package.json | 4 +- .../scripts/export-bootstrap-sql.js | 89 -- benchmarks-website/server.js | 830 +++++++++++++++--- benchmarks-website/src/App.jsx | 32 +- benchmarks-website/src/api.js | 70 +- .../src/components/ChartContainer.jsx | 2 +- benchmarks-website/src/components/Header.jsx | 5 +- benchmarks-website/src/components/Modal.jsx | 2 +- .../src/hooks/useBenchmarkMetadata.js | 75 -- benchmarks-website/store/benchmark-store.js | 354 -------- benchmarks-website/store/cache.js | 174 ---- benchmarks-website/store/constants.js | 20 - benchmarks-website/store/db.js | 33 - benchmarks-website/store/downsample.js | 68 -- benchmarks-website/store/metadata.js | 401 --------- benchmarks-website/store/sql.js | 479 ---------- benchmarks-website/store/utils.js | 3 - 22 files changed, 734 insertions(+), 2820 deletions(-) delete mode 100644 benchmarks-website/ETL.md delete mode 100644 benchmarks-website/README.md delete mode 100644 benchmarks-website/SCHEMA.md delete mode 100644 benchmarks-website/scripts/export-bootstrap-sql.js delete mode 100644 benchmarks-website/src/hooks/useBenchmarkMetadata.js delete mode 100644 benchmarks-website/store/benchmark-store.js delete mode 100644 benchmarks-website/store/cache.js delete mode 100644 benchmarks-website/store/constants.js delete mode 100644 benchmarks-website/store/db.js delete mode 100644 benchmarks-website/store/downsample.js delete mode 100644 benchmarks-website/store/metadata.js delete mode 100644 benchmarks-website/store/sql.js delete mode 100644 benchmarks-website/store/utils.js diff --git a/benchmarks-website/Dockerfile b/benchmarks-website/Dockerfile index 094d9a9a822..1f87a7148b5 100644 --- a/benchmarks-website/Dockerfile +++ b/benchmarks-website/Dockerfile @@ -1,17 +1,16 @@ -FROM node:24-bookworm-slim AS build +FROM node:24-alpine AS build WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci COPY . . RUN npm run build -FROM node:24-bookworm-slim +FROM node:24-alpine WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev COPY --from=build /app/dist ./dist COPY server.js . -COPY store ./store COPY src/config.js ./src/config.js EXPOSE 3000 CMD ["node", "server.js"] diff --git a/benchmarks-website/ETL.md b/benchmarks-website/ETL.md deleted file mode 100644 index 3087e683d60..00000000000 --- a/benchmarks-website/ETL.md +++ /dev/null @@ -1,302 +0,0 @@ -# Benchmark Website ETL - -For the exact normalized table definitions, see [SCHEMA.md](./SCHEMA.md). - -## Source Artifacts - -The website consumes two append-only JSON artifacts from S3: - -- `commits.json` -- `data.json.gz` - -Those artifacts are produced by GitHub Actions on every push to `develop`: - -- [`../.github/workflows/bench.yml`](../.github/workflows/bench.yml) - runs the random-access and compression benchmarks -- [`../.github/workflows/sql-benchmarks.yml`](../.github/workflows/sql-benchmarks.yml) - runs ClickBench, TPC-H, TPC-DS, StatPopGen, PolarSignals, and other SQL workloads - -Both workflows append into the S3 objects with [`../scripts/cat-s3.sh`](../scripts/cat-s3.sh), -which uses S3 `ETag` compare-and-swap to preserve append-only semantics under concurrent writers. - -## Artifact Generation - -### `commits.json` - -`commits.json` is generated by [`../scripts/commit-json.sh`](../scripts/commit-json.sh). - -Important detail: `timestamp` comes from the Git **committer date**: - -```bash -git log -1 --format=%cd --date=iso-strict "$GITHUB_SHA" -``` - -So the website orders commits by committer timestamp, not author timestamp. - -Logical emitted schema: - -```sql -create table commits_json_artifact ( - id varchar primary key, - message varchar, - timestamp varchar, -- ISO 8601 with timezone, from git %cd - author json, - committer json, - tree_id varchar, - url varchar -); -``` - -Representative record: - -```json -{ - "author": { "email": "alice@example.com", "name": "Alice" }, - "committer": { "email": "alice@example.com", "name": "Alice" }, - "id": "abc123...", - "message": "Improve Vortex scan planning", - "timestamp": "2026-04-17T09:41:52+00:00", - "tree_id": "def456...", - "url": "https://github.com/vortex-data/vortex/commit/abc123..." -} -``` - -### `data.json.gz` - -`data.json.gz` is gzipped newline-delimited JSON. Each line is one benchmark sample emitted by -the benchmark runners in `vortex-bench`. - -The website intentionally projects only the subset it needs: - -```sql -create table benchmark_json_projection ( - name varchar, - unit varchar, - value double, - storage varchar, - dataset json, - commit json, - commit_id varchar -); -``` - -Fields present upstream but ignored by the website include `target`, `time`, `bytes`, -`all_runtimes`, and `env_triple`. See -[`../vortex-bench/src/measurements.rs`](../vortex-bench/src/measurements.rs). - -Two raw fields carry most of the semantic structure: - -- `name` - encodes benchmark family, query number, chart identity, and series identity -- `dataset` - is used for suite fan-out such as TPC-H / TPC-DS scale factors - -Representative naming patterns: - -```text -random-access/// -clickbench_q01/: -tpch_q07/: -compress time/ -vortex size/ -``` - -## Refresh Pipeline - -At refresh time, the server: - -1. downloads or reuses cached copies of `commits.json` and `data.json.gz` -2. exposes them to DuckDB as NDJSON views -3. materializes the normalized relational model in `:memory:` DuckDB -4. computes metadata and latest-value projections -5. serves chart slices and summaries from those tables - -The refresh uses conditional HTTP requests against the cached `etag` and `last-modified` values. -If both files are unchanged and the server already has an in-memory state, the rebuild is skipped. - -## Unicode Relation Diagrams - -```text - Refresh / Materialization Graph - -┌──────────────────────────────┐ ┌──────────────────────────────┐ -│ raw_commits │ │ raw_benchmarks │ -│ view over commits.json │ │ view over data.json.gz │ -│ ──────────────────────────── │ │ ──────────────────────────── │ -│ id │ │ benchmark_row │ -│ message │ │ name │ -│ timestamp │ │ unit │ -│ author │ │ value │ -│ url │ │ storage │ -└──────────────┬───────────────┘ │ dataset │ - │ │ commit │ - │ order by timestamp │ commit_id │ - ▼ └──────────────┬───────────────┘ -┌──────────────────────────────┐ │ -│ commit_dim │ │ split name / keep raw JSON -│ materialized table │ ▼ -│ ──────────────────────────── │ ┌──────────────────────────────┐ -│ PK commit_idx │ │ benchmarks_base │ -│ UK id │ │ materialized table │ -│ message │ │ ──────────────────────────── │ -│ timestamp_text │ │ PK benchmark_row │ -│ commit_ts │ │ name / name_lower │ -│ author │ │ part1..part4 / part_count │ -│ url │ │ unit / raw_value │ -└──────────────┬───────────────┘ │ storage │ - │ │ dataset_json │ - │ join on id = │ commit_json │ - │ resolved_commit_id│ commit_id │ - │ └──────────────┬───────────────┘ - │ │ - │ ┌─────────────────────┼─────────────────────┐ - │ │ │ │ - │ ▼ ▼ ▼ - │ ┌───────────────────┐ ┌───────────────────┐ ┌──────────────────────┐ - │ │ query_suites │ │ engine_renames │ │ valid_groups │ - │ │ config table │ │ config table │ │ config table │ - │ │ prefix │ │ src │ │ PK group_name │ - │ │ display_name │ │ dst │ └──────────┬───────────┘ - │ │ query_prefix │ └─────────┬─────────┘ │ allowlist - │ │ dataset_key │ │ rename series │ - │ │ fan_out / skip │ │ │ - │ └─────────┬─────────┘ │ │ - │ │ match on prefix │ │ - │ ▼ │ │ - │ ┌──────────────────────────────┐ │ │ - │ │ matched_suites │ │ │ - │ │ materialized table │ │ │ - │ │ PK benchmark_row │ │ │ - │ │ prefix / display_name │ │ │ - │ │ query_prefix │ │ │ - │ │ dataset_key / fan_out │ │ │ - │ └──────────────┬───────────────┘ │ │ - │ │ │ │ - │ └────────┬────────┴──────────────┐ │ - │ ▼ │ │ - │ ┌──────────────────────────────────────────────┐ │ │ - │ │ classified_benchmarks │◄┘ │ - │ │ materialized table │ │ - │ │ ──────────────────────────────────────────── │ │ - │ │ PK benchmark_row │ │ - │ │ name │ │ - │ │ resolved_commit_id │ │ - │ │ group_name │ │ - │ │ chart_name │ │ - │ │ series_name │ │ - │ │ sort_position │ │ - │ │ unit / value │ │ - │ └──────────────────────┬─────────────────────────┘ │ - │ │ │ - │ │ join commit + apply valid_groups │ - │ ▼ │ - │ ┌──────────────────────────────────────────────┐◄───────────┘ - │ │ benchmark_points │ - │ │ first fact table │ - │ │ ──────────────────────────────────────────── │ - │ │ group_name │ - │ │ chart_name │ - │ │ series_name │ - │ │ sort_position │ - │ │ unit / value │ - │ │ FK commit_idx -> commit_dim.commit_idx │ - │ └──────────────────────┬───────────────────────┘ - │ │ trim leading empty commits - │ ▼ - │ ┌──────────────────────────────────────────────┐ - │ │ active_commits │ - │ │ rebased commit axis │ - │ │ ──────────────────────────────────────────── │ - │ │ PK original_commit_idx │ - │ │ UK commit_idx │ - │ │ id / message / timestamp / author / url │ - │ └──────────────────────┬───────────────────────┘ - │ │ join on original_commit_idx - └──────────────────────────────┴─────────────────────────────────────┐ - ▼ - ┌──────────────────────────────────────────────┐ - │ benchmark_points_active │ - │ canonical chart fact table │ - │ ──────────────────────────────────────────── │ - │ group_name │ - │ chart_name │ - │ series_name │ - │ sort_position │ - │ unit / value │ - │ commit_idx │ - │ practical key: │ - │ (group_name, chart_name, series_name, │ - │ commit_idx) │ - └──────────────────────────────────────────────┘ -``` - -```text - Serving / Derived Projection Graph - - ┌──────────────────────────────────────────────┐ - │ benchmark_points_active │ - │ group_name, chart_name, series_name, │ - │ sort_position, unit, value, commit_idx │ - └───────┬──────────────────────┬───────────────┘ - │ │ - group by chart │ │ arg_max(value, commit_idx) - │ │ - ▼ ▼ - ┌──────────────────────────────┐ ┌──────────────────────────────┐ - │ chart_defs │ │ chart_series_latest_values │ - │ PK (group_name, chart_name) │ │ PK (group_name, chart_name, │ - │ sort_position │ │ series_name) │ - │ unit │ │ latest_value │ - └──────────────┬──────────────┘ └──────────────┬───────────────┘ - │ │ - │ latest commit per chart │ - ▼ │ - ┌──────────────────────────────┐ │ - │ chart_latest_idx │ │ - │ PK (group_name, chart_name) │ │ - │ latest_commit_idx │ │ - └──────────────┬──────────────┘ │ - │ join on latest_commit_idx │ - ▼ │ - ┌──────────────────────────────┐ │ - │ chart_latest_values │──────────────────────┘ - │ group_name │ - │ chart_name │ - │ series_name │ - │ value │ - └──────────────┬──────────────┘ - │ - │ + active_commits - ▼ - ┌──────────────────────────────┐ - │ /api/metadata │ - │ groups, charts, summaries, │ - │ commits, lastUpdated │ - └──────────────────────────────┘ - - ┌──────────────────────────────┐ - │ active_commits │ - │ commit_idx, id, message, │ - │ timestamp, author, url │ - └──────────────┬──────────────┘ - │ range slice - ▼ - ┌──────────────────────────────┐ - │ /api/data/:group/:chart │ - │ dense series arrays over │ - │ requested commit range │ - └──────────────────────────────┘ -``` - -## Transformation Notes - -- `commit_dim` is the first place timestamps become typed and ordered. -- `benchmarks_base` is the string-parsing layer over benchmark `name`. -- `classified_benchmarks` is the semantic step where the row becomes: - - a group - - a chart - - a series - - a normalized display unit/value -- `benchmark_points_active` is the canonical long-form chart fact table. -- The API never stores dense charts on disk; dense series arrays are produced on demand from the - long-form fact table. diff --git a/benchmarks-website/README.md b/benchmarks-website/README.md deleted file mode 100644 index 2b67382e7e3..00000000000 --- a/benchmarks-website/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# Benchmarks Website - -This directory contains the benchmark website frontend, the Node HTTP server, and the DuckDB-based -refresh pipeline that turns the raw benchmark artifacts into chartable time series. - -For the data model and table relationships, start with [SCHEMA.md](./SCHEMA.md). -For the upstream artifact generation and refresh/materialization flow, see [ETL.md](./ETL.md). - -## Prerequisites - -- Node.js `>=18` -- npm -- Optional: DuckDB CLI, if you want to query the cached artifacts directly - -Install dependencies: - -```bash -cd benchmarks-website -npm install -``` - -## Development Server - -Run the frontend and backend together: - -```bash -cd benchmarks-website -npm run dev -``` - -That starts: - -- Vite on `http://localhost:5173` -- the API/static server on `http://localhost:3000` - -Useful endpoints: - -- `http://localhost:3000/api/metadata` -- `http://localhost:3000/api/health` - -The backend refreshes from these artifact URLs by default: - -- `https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz` -- `https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json` - -Relevant environment variables: - -```bash -PORT=3000 -REFRESH_INTERVAL=300000 -DATA_URL=https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz -COMMITS_URL=https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json -CACHE_DIR=/path/to/local/cache -``` - -`CACHE_DIR` is the most useful one during development. If it is unset, the server uses a temp -directory under `os.tmpdir()`. - -## Pull The Data Locally - -If you want a predictable local copy for exploration, populate a cache directory yourself and point -the server at it. - -```bash -cd benchmarks-website -mkdir -p .cache/benchmarks - -curl -L \ - https://vortex-ci-benchmark-results.s3.amazonaws.com/data.json.gz \ - -o .cache/benchmarks/data.json.gz - -curl -L \ - https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json \ - -o .cache/benchmarks/commits.json -``` - -Then start the dev server against that cache: - -```bash -cd benchmarks-website -CACHE_DIR="$PWD/.cache/benchmarks" npm run dev -``` - -On first startup, the server will use the cached files immediately and then asynchronously -revalidate them against S3. - -## Explore The Cached Data Directly - -Once `data.json.gz` and `commits.json` exist locally, you can query them with DuckDB without -running the website. - -Example with the DuckDB CLI: - -```sql -create view raw_commits as -select * -from read_json( - '.cache/benchmarks/commits.json', - format = 'newline_delimited', - compression = 'auto_detect', - columns = { - id: 'VARCHAR', - message: 'VARCHAR', - timestamp: 'VARCHAR', - author: 'JSON', - url: 'VARCHAR' - } -); - -create view raw_benchmarks as -select * -from read_json( - '.cache/benchmarks/data.json.gz', - format = 'newline_delimited', - compression = 'auto_detect', - columns = { - name: 'VARCHAR', - unit: 'VARCHAR', - value: 'DOUBLE', - storage: 'VARCHAR', - dataset: 'JSON', - commit: 'JSON', - commit_id: 'VARCHAR' - } -); -``` - -Useful starter queries: - -```sql -select count(*) as commit_count from raw_commits; - -select count(*) as benchmark_count from raw_benchmarks; - -select split_part(name, '/', 1) as prefix, count(*) as rows -from raw_benchmarks -group by 1 -order by 2 desc -limit 20; - -select - coalesce(json_extract_string(commit, '$.id'), commit_id) as resolved_commit_id, - count(*) as rows -from raw_benchmarks -group by 1 -order by 2 desc -limit 20; -``` - -If you want the normalized relational model rather than the raw JSON views, follow the pipeline in -[SCHEMA.md](./SCHEMA.md) and [`store/sql.js`](./store/sql.js). - -## Export The Full Bootstrap SQL - -If you want the exact SQL that the server uses to create all config tables, raw views, normalized -tables, and derived projections, export it from the shared SQL builder: - -```bash -cd benchmarks-website -npm run export-sql -- \ - --data-path "$PWD/.cache/benchmarks/data.json.gz" \ - --commits-path "$PWD/.cache/benchmarks/commits.json" \ - --output "$PWD/.cache/benchmarks/bootstrap.sql" -``` - -Then load it in DuckDB: - -```bash -duckdb benchmark-explore.duckdb < .cache/benchmarks/bootstrap.sql -``` - -That creates the same tables and views the server uses, including: - -- `query_suites` -- `valid_groups` -- `engine_renames` -- `raw_commits` -- `raw_benchmarks` -- `commit_dim` -- `benchmarks_base` -- `matched_suites` -- `classified_benchmarks` -- `benchmark_points` -- `active_commits` -- `benchmark_points_active` -- `chart_defs` -- `chart_latest_idx` -- `chart_latest_values` -- `chart_series_latest_values` - -If you want a portable template instead of path-specific SQL: - -```bash -cd benchmarks-website -npm run export-sql -- --placeholders --output bootstrap.template.sql -``` - -That emits a script using `__DATA_PATH__` and `__COMMITS_PATH__` placeholders. - -## Notes - -- The website only projects the subset of the raw benchmark JSON it needs for grouping, charting, - and summaries. -- Benchmark names are part of the schema. Group, chart, and series identity are inferred from the - `name`, `storage`, and `dataset` fields during refresh. -- The server returns `503` with `Retry-After` while the initial refresh is still loading. diff --git a/benchmarks-website/SCHEMA.md b/benchmarks-website/SCHEMA.md deleted file mode 100644 index d54fd168182..00000000000 --- a/benchmarks-website/SCHEMA.md +++ /dev/null @@ -1,298 +0,0 @@ -# Benchmark Website Schema - -This file is the reference for the **normalized DuckDB schema** used by the benchmark website. - -For the upstream artifact generation, refresh pipeline, and table-to-table flow, see -[ETL.md](./ETL.md). - -The implementation lives primarily in [`store/sql.js`](./store/sql.js), -[`store/metadata.js`](./store/metadata.js), and [`store/benchmark-store.js`](./store/benchmark-store.js). -For DuckDB's NDJSON reader semantics, see: -. - -## Core Fact Model - -The important point is that the website does **not** store nested charts directly. It stores a -long-form fact table, `benchmark_points_active`, keyed in practice by: - -```sql -(group_name, chart_name, series_name, commit_idx) -``` - -Everything else is metadata, lookup/config data, or a projection over that fact table. - -## Configuration Tables - -These are synthesized from [`src/config.js`](./src/config.js), not loaded from the JSON artifacts. - -```sql -create table query_suites ( - prefix varchar, - display_name varchar, - query_prefix varchar, - dataset_key varchar, - fan_out boolean, - skip boolean -); - -create table valid_groups ( - group_name varchar primary key -); - -create table engine_renames ( - src varchar primary key, - dst varchar -); -``` - -## Raw Views - -These are views over the cached artifact files. - -```sql -create view raw_commits as -select - id, - message, - timestamp, - author, - url -from read_json(...); - -create view raw_benchmarks as -select - row_number() over () as benchmark_row, - name, - unit, - value, - storage, - dataset, - commit, - commit_id -from read_json(...); -``` - -## Stage 1: Commit Ordering and Name Parsing - -```sql -create table commit_dim ( - commit_idx bigint primary key, - id varchar unique, - message varchar, - timestamp_text varchar, - commit_ts timestamptz, - author varchar, - url varchar -); - -create table benchmarks_base ( - benchmark_row bigint primary key, - name varchar, - name_lower varchar, - part1 varchar, - part1_lower varchar, - part2 varchar, - part3 varchar, - part4 varchar, - part_count bigint, - unit varchar, - raw_value double, - storage varchar, - dataset_json json, - commit_json json, - commit_id varchar -); - -create table matched_suites ( - benchmark_row bigint primary key, - prefix varchar, - display_name varchar, - query_prefix varchar, - dataset_key varchar, - fan_out boolean -); -``` - -Notes: - -- `commit_dim` orders commits by parsed timestamp, then `id`. -- `benchmarks_base` is the normalized string-parsing layer over benchmark `name`. -- `matched_suites` assigns a benchmark row to the longest matching configured suite prefix. - -## Stage 2: Classification - -```sql -create table classified_benchmarks ( - benchmark_row bigint primary key, - name varchar, - resolved_commit_id varchar, - group_name varchar, - chart_name varchar, - series_name varchar, - sort_position integer, - unit varchar, - value double -); -``` - -This is the semantic classification step: - -- `resolved_commit_id` is `coalesce(commit.id, commit_id)` -- `group_name` is inferred from benchmark name prefixes, query suite config, storage, and scale - factor metadata -- `chart_name` and `series_name` come from benchmark-name parsing -- `series_name` is passed through `engine_renames` -- `unit` and `value` are normalized for display: - - `ns -> ms/iter` - - `bytes -> MiB` - -In other words, the benchmark-name grammar is part of the schema. - -## Stage 3: Time-Series Fact Tables - -```sql -create table benchmark_points ( - group_name varchar, - chart_name varchar, - series_name varchar, - sort_position integer, - unit varchar, - value double, - commit_idx bigint -); - -create table active_commits ( - original_commit_idx bigint primary key, - commit_idx bigint unique, - id varchar, - message varchar, - timestamp varchar, - author varchar, - url varchar -); - -create table benchmark_points_active ( - group_name varchar, - chart_name varchar, - series_name varchar, - sort_position integer, - unit varchar, - value double, - commit_idx bigint -); -``` - -Notes: - -- `benchmark_points` is the first fact table after commit resolution and group filtering. -- `active_commits` trims leading commits with no benchmark data and re-bases the visible commit - axis to `0..N-1`. -- `benchmark_points_active` is the canonical fact table used for charting and summaries. - -## Stage 4: Chart Metadata and Latest-Value Projections - -```sql -create table chart_defs ( - group_name varchar, - chart_name varchar, - sort_position integer, - unit varchar, - primary key (group_name, chart_name) -); - -create table chart_latest_idx ( - group_name varchar, - chart_name varchar, - latest_commit_idx bigint, - primary key (group_name, chart_name) -); - -create table chart_latest_values ( - group_name varchar, - chart_name varchar, - series_name varchar, - value double -); - -create table chart_series_latest_values ( - group_name varchar, - chart_name varchar, - series_name varchar, - latest_value double, - primary key (group_name, chart_name, series_name) -); -``` - -There are two different "latest" concepts: - -- `chart_latest_values` - values at the chart's latest commit index -- `chart_series_latest_values` - latest non-null value seen for each series, via `arg_max(value, commit_idx)` - -## API-Facing Reads - -### `/api/metadata` - -Built in [`store/metadata.js`](./store/metadata.js). - -Primary sources: - -- `active_commits` -- `chart_defs` -- `chart_series_latest_values` -- `chart_latest_values` -- `benchmark_points_active` - -Logical output shape: - -```sql -metadata ( - total_commits bigint, - last_updated timestamptz, - commits json, - groups json -) -``` - -Each group contains: - -```sql -group_metadata ( - charts json, - total_charts bigint, - has_data boolean, - summary json -) -``` - -Summary sections are computed from SQL aggregates: - -- random access ranking from one anchor chart -- compression geomeans from Vortex/Parquet ratio charts -- compression-size min/geomean/max -- query-suite geomeans with a missing-query penalty - -### `/api/data/:group/:chart` - -Built in [`store/benchmark-store.js`](./store/benchmark-store.js). - -The query: - -1. selects the requested commit range from `active_commits` -2. enumerates the series present for that chart -3. cross joins series against commits -4. left joins `benchmark_points_active` -5. aggregates with `list(value order by commit_idx)` - -That produces a dense matrix with explicit `null` gaps: - -```sql -series_points ( - series_name varchar, - values list -) -``` - -The server then serializes the commit slice, applies downsampling if needed, and returns -`{ commits, series, requestedRange, downsampleLevel, ... }`. diff --git a/benchmarks-website/package-lock.json b/benchmarks-website/package-lock.json index b64b131bb36..d140b73d225 100644 --- a/benchmarks-website/package-lock.json +++ b/benchmarks-website/package-lock.json @@ -8,7 +8,6 @@ "name": "vortex-benchmarks-website", "version": "2.0.0", "dependencies": { - "@duckdb/node-api": "^1.5.2-r.1", "chart.js": "^4.4.4", "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", @@ -321,107 +320,6 @@ "node": ">=6.9.0" } }, - "node_modules/@duckdb/node-api": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.2-r.1.tgz", - "integrity": "sha512-OzBBnS0JGXMoS5mzKNY/Ylr7SshcRQiLFIoxQ4AlePwJ2fNeDL/fbHu/knjxUrXwW1fJBTUgwWftmxDdnZZb3A==", - "license": "MIT", - "dependencies": { - "@duckdb/node-bindings": "1.5.2-r.1" - } - }, - "node_modules/@duckdb/node-bindings": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.2-r.1.tgz", - "integrity": "sha512-bUg3bLVj70YVku6fKyQJS8ASORl7kM7YFVFznsEB9pWbtazPj+ME2x2FUk0WiTzjJdutjzSSGXF066mB4bGGZA==", - "license": "MIT", - "optionalDependencies": { - "@duckdb/node-bindings-darwin-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-darwin-x64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-x64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-x64": "1.5.2-r.1" - } - }, - "node_modules/@duckdb/node-bindings-darwin-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-v35FyKOb8EJCvaiPF7k0gvKiJTXR7PPQDNoWR0Gu+YSX5O9b+DIguzt1348Of3HebHy6ATSMzlUekaVA9YXu+g==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@duckdb/node-bindings-darwin-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.2-r.1.tgz", - "integrity": "sha512-SU9dIJ1BluKkkGxi4UsP4keqkkstB2YDySF9KcYu3EZKIVM3FTv2zc7XO38dXnHOq6+F3WqhWWZvD+XU945p7A==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@duckdb/node-bindings-linux-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-3Tra9xM3aM3denaER4KhJ6//6PpmPbik9ECBQ+sh9PyKaEgHw/0kAcKnLm5EzWUnXF0qYmZlewvkCrse8KmOYw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@duckdb/node-bindings-linux-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.2-r.1.tgz", - "integrity": "sha512-pcQvZRHiIfJ9cq8parkSQczQHEml/IeGfnDCMAbEgD6+jaV9Y9Y5Ph1kP9aR+bm6him1S5ZIEr3kZbihjKnWbA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@duckdb/node-bindings-win32-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-Ji8tym+N3LkrhVt0Up3bsacD/kpg4/JXFJQqxswiYvBaNCQOk+D+aiVS0GN5pcqvmnG7V7TpsDRzkLEFaWp1vw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@duckdb/node-bindings-win32-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.2-r.1.tgz", - "integrity": "sha512-5XqcqC+4R8ghBEEbnc2a0sqfz1zyPBRb9YcmIWfiuDoCYSYFbKhmHcEyNftZDHcwCoLOHXnUin45jraex4STqQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", diff --git a/benchmarks-website/package.json b/benchmarks-website/package.json index 13280ae7898..6cda687b838 100644 --- a/benchmarks-website/package.json +++ b/benchmarks-website/package.json @@ -7,14 +7,12 @@ "vite": "vite", "server": "node server.js", "build": "vite build", - "preview": "vite preview", - "export-sql": "node scripts/export-bootstrap-sql.js" + "preview": "vite preview" }, "engines": { "node": ">=18.0.0" }, "dependencies": { - "@duckdb/node-api": "^1.5.2-r.1", "chart.js": "^4.4.4", "chartjs-plugin-zoom": "^2.0.1", "downsample": "^1.4.0", diff --git a/benchmarks-website/scripts/export-bootstrap-sql.js b/benchmarks-website/scripts/export-bootstrap-sql.js deleted file mode 100644 index 9bc2b5eee1d..00000000000 --- a/benchmarks-website/scripts/export-bootstrap-sql.js +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env node - -import fs from "fs/promises"; -import path from "path"; -import { buildBootstrapSql } from "../store/sql.js"; - -function parseArgs(argv) { - const options = { - dataPath: null, - commitsPath: null, - outputPath: null, - placeholders: false, - }; - - for (let i = 0; i < argv.length; i++) { - const arg = argv[i]; - switch (arg) { - case "--data-path": - options.dataPath = argv[++i] || null; - break; - case "--commits-path": - options.commitsPath = argv[++i] || null; - break; - case "--output": - options.outputPath = argv[++i] || null; - break; - case "--placeholders": - options.placeholders = true; - break; - case "--help": - case "-h": - printHelp(); - process.exit(0); - break; - default: - throw new Error(`Unknown argument: ${arg}`); - } - } - - return options; -} - -function printHelp() { - process.stdout.write(`Usage: - node scripts/export-bootstrap-sql.js --data-path --commits-path [--output ] - node scripts/export-bootstrap-sql.js --placeholders [--output ] - -Options: - --data-path Path to data.json.gz - --commits-path Path to commits.json - --output Write SQL to a file instead of stdout - --placeholders Emit __DATA_PATH__ / __COMMITS_PATH__ placeholders - --help, -h Show this help -`); -} - -async function main() { - const options = parseArgs(process.argv.slice(2)); - - const dataPath = options.placeholders ? "__DATA_PATH__" : options.dataPath; - const commitsPath = options.placeholders - ? "__COMMITS_PATH__" - : options.commitsPath; - - if (!dataPath || !commitsPath) { - printHelp(); - process.stderr.write( - "\nerror: either provide --data-path and --commits-path, or use --placeholders\n", - ); - process.exit(1); - } - - const sql = `${buildBootstrapSql(dataPath, commitsPath)}\n`; - - if (options.outputPath) { - const outputPath = path.resolve(options.outputPath); - await fs.mkdir(path.dirname(outputPath), { recursive: true }); - await fs.writeFile(outputPath, sql, "utf8"); - process.stdout.write(`${outputPath}\n`); - return; - } - - process.stdout.write(sql); -} - -main().catch((error) => { - process.stderr.write(`${error.message}\n`); - process.exit(1); -}); diff --git a/benchmarks-website/server.js b/benchmarks-website/server.js index 80ce55e9e82..a9af96234fe 100644 --- a/benchmarks-website/server.js +++ b/benchmarks-website/server.js @@ -2,11 +2,16 @@ import http from "http"; import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; -import { BenchmarkStore } from "./store/benchmark-store.js"; +import zlib from "zlib"; +import readline from "readline"; +import { Readable } from "stream"; +import { LTTB } from "downsample"; +import { QUERY_SUITES, FAN_OUT_GROUPS, ENGINE_RENAMES } from "./src/config.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +// Configuration const PORT = process.env.PORT || 3000; const DATA_URL = process.env.DATA_URL || @@ -15,9 +20,17 @@ const COMMITS_URL = process.env.COMMITS_URL || "https://vortex-ci-benchmark-results.s3.amazonaws.com/commits.json"; const REFRESH_INTERVAL = process.env.REFRESH_INTERVAL || 5 * 60 * 1000; +const MAX_POINTS = 200; const USE_LOCAL_DATA = process.env.USE_LOCAL_DATA === "true"; -const CACHE_DIR = process.env.CACHE_DIR || undefined; -const LOADING_RETRY_AFTER_SECONDS = 1; + +// Benchmark groups: non-query groups + simple suites + fan-out suites +const GROUPS = [ + "Random Access", + "Compression", + "Compression Size", + ...QUERY_SUITES.filter((s) => !s.skip && !s.fanOut).map((s) => s.displayName), + ...FAN_OUT_GROUPS, +]; const MIME = { ".html": "text/html", @@ -33,103 +46,688 @@ const MIME = { ".webmanifest": "application/manifest+json", }; -const benchmarks = new BenchmarkStore({ - dataUrl: DATA_URL, - commitsUrl: COMMITS_URL, - useLocalData: USE_LOCAL_DATA, - cacheDir: CACHE_DIR, -}); -let refreshIntervalId = null; - -const json = (res, code, data, headers = {}) => { - res.writeHead(code, { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - "Cache-Control": "no-store", - Pragma: "no-cache", - ...headers, - }); - res.end(JSON.stringify(data)); +let store = { + commits: [], + groups: {}, + metadata: null, + downsampled: {}, + lastUpdated: null, }; -function loadingHeaders(status) { - if (status.state !== "loading") { - return {}; +// Utilities +const rename = (s) => ENGINE_RENAMES[s.toLowerCase()] || ENGINE_RENAMES[s] || s; +const geoMean = (arr) => + arr.length + ? Math.pow( + arr.reduce((a, v) => a * v, 1), + 1 / arr.length, + ) + : null; + +// Categorize benchmarks based on name patterns and metadata +function getGroup(benchmark) { + const name = benchmark.name; + const lower = name.toLowerCase(); + + // Random Access: "random-access/..." or "random access/..." + if ( + lower.startsWith("random-access/") || + lower.startsWith("random access/") + ) { + return "Random Access"; } - return { - "Retry-After": String(LOADING_RETRY_AFTER_SECONDS), - }; + // Compression Size: size measurements + if ( + lower.startsWith("vortex size/") || + lower.startsWith("vortex-file-compressed size/") || + lower.startsWith("parquet size/") || + lower.startsWith("lance size/") || + lower.includes(":raw size/") || + lower.includes(":parquet-zstd size/") || + lower.includes(":lance size/") + ) { + return "Compression Size"; + } + + // Compression: compress/decompress time and ratio measurements + if ( + lower.startsWith("compress time/") || + lower.startsWith("decompress time/") || + lower.startsWith("parquet_rs-zstd compress") || + lower.startsWith("parquet_rs-zstd decompress") || + lower.startsWith("lance compress") || + lower.startsWith("lance decompress") || + lower.startsWith("vortex:lance ratio") || + lower.startsWith("vortex:parquet-zstd ratio") || + lower.startsWith("vortex:raw ratio") + ) { + return "Compression"; + } + + // SQL query suites: match "{prefix}_q..." or "{prefix}/..." + for (const suite of QUERY_SUITES) { + if ( + !lower.startsWith(suite.prefix + "_q") && + !lower.startsWith(suite.prefix + "/") + ) + continue; + if (suite.skip) return null; + if (!suite.fanOut) return suite.displayName; + // Fan-out suites: expand by storage and scale factor + const storage = benchmark.storage?.toUpperCase() === "S3" ? "S3" : "NVMe"; + const rawSf = benchmark.dataset?.[suite.datasetKey]?.scale_factor; + const sf = rawSf ? Math.round(parseFloat(rawSf)) : 1; + return `${suite.displayName} (${storage}) (SF=${sf})`; + } + + return null; } -function getLoadingPayload() { - const status = benchmarks.status; +// Format query name for display: "{prefix}_q00" -> "{QUERY_PREFIX} Q0" +function formatQuery(q) { + const lower = q.toLowerCase(); + for (const suite of QUERY_SUITES) { + if (suite.skip) continue; + const m = lower.match(new RegExp(`^${suite.prefix}[_ ]?q(\\d+)`, "i")); + if (m) return `${suite.queryPrefix} Q${parseInt(m[1], 10)}`; + } + return q.toUpperCase().replace(/[_-]/g, " "); +} + +function normalizeChartName(group, chartName) { + if (group === "Compression Size" && chartName === "VORTEX FILE COMPRESSED SIZE") { + return "VORTEX SIZE"; + } + return chartName; +} + +// LTTB downsampling +function lttbIndices(seriesMap, target) { + const keys = [...seriesMap.keys()]; + if (!keys.length) return []; + const len = seriesMap.get(keys[0])?.length || 0; + if (len <= target) return [...Array(len).keys()]; + + const avg = Array(len); + for (let i = 0; i < len; i++) { + let sum = 0, + n = 0; + for (const arr of seriesMap.values()) { + const v = arr[i]?.value ?? arr[i]; + if (v != null && !isNaN(v)) { + sum += v; + n++; + } + } + avg[i] = [i, n ? sum / n : 0]; + } + + const idx = LTTB(avg, target).map((p) => Math.round(p[0])); + if (!idx.includes(0)) idx.unshift(0); + if (!idx.includes(len - 1)) idx.push(len - 1); + return idx.sort((a, b) => a - b); +} + +function downsample(data, factor) { + const target = Math.ceil(data.commits.length / factor); + if (target >= data.commits.length) return data; + + const idx = lttbIndices(data.series, target); + const series = new Map(); + for (const [k, v] of data.series) + series.set( + k, + idx.map((i) => v[i]), + ); + return { - error: status.state === "error" ? "Initial refresh failed" : "Loading", - status: status.state, - ready: status.ready, - refreshing: status.refreshing, - hasData: status.hasData, - lastUpdated: status.lastUpdated, - lastRefreshStartedAt: status.lastRefreshStartedAt, - lastRefreshCompletedAt: status.lastRefreshCompletedAt, - lastRefreshError: status.lastRefreshError, + ...data, + commits: idx.map((i) => data.commits[i]), + series, + originalLength: data.commits.length, }; } -function triggerRefresh() { +// Data fetching — streams response body directly instead of buffering +async function fetchJsonl(url) { + const res = await fetch(url); + if (!res.ok) throw new Error(`Fetch failed: ${url} ${res.status}`); + return new Promise((resolve, reject) => { + const results = []; + const rl = readline.createInterface({ + input: Readable.fromWeb(res.body), + crlfDelay: Infinity, + }); + rl.on("line", (l) => { + if (l.trim()) + try { + results.push(JSON.parse(l)); + } catch {} + }); + rl.on("close", () => resolve(results)); + rl.on("error", reject); + }); +} + +function readLocalJsonl(fp) { + return new Promise((resolve, reject) => { + const results = []; + const rl = readline.createInterface({ + input: fs.createReadStream(fp), + crlfDelay: Infinity, + }); + rl.on("line", (l) => { + if (l.trim()) + try { + results.push(JSON.parse(l)); + } catch {} + }); + rl.on("close", () => resolve(results)); + rl.on("error", reject); + }); +} + +// Stream benchmark data record-by-record without buffering the entire dataset +async function forEachBenchmark(callback) { + let stream; + if (USE_LOCAL_DATA) { + stream = fs.createReadStream(path.join(__dirname, "sample/data.json")); + } else { + const res = await fetch(DATA_URL); + if (!res.ok) throw new Error(`Fetch failed: ${DATA_URL} ${res.status}`); + stream = Readable.fromWeb(res.body).pipe(zlib.createGunzip()); + } + return new Promise((resolve, reject) => { + const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); + rl.on("line", (l) => { + if (l.trim()) + try { + callback(JSON.parse(l)); + } catch {} + }); + rl.on("close", resolve); + rl.on("error", reject); + }); +} + +// Main data processing +async function refresh() { console.log("Refreshing data..."); - benchmarks.refresh().catch(() => {}); + const t0 = Date.now(); + + try { + // Load commits first (small dataset, must be fully in memory for indexing) + const commitsArr = USE_LOCAL_DATA + ? await readLocalJsonl(path.join(__dirname, "sample/commits.json")) + : await fetchJsonl(COMMITS_URL); + + // Build commit index (O(1) lookup) + const commitMap = new Map(commitsArr.map((c) => [c.id, c])); + const commits = commitsArr.sort( + (a, b) => new Date(a.timestamp) - new Date(b.timestamp), + ); + const commitIdx = new Map(commits.map((c, i) => [c.id, i])); + + const groups = Object.fromEntries(GROUPS.map((g) => [g, new Map()])); + let missing = 0; + let benchmarkCount = 0; + const uncategorized = new Set(); + + // Stream benchmarks one record at a time to avoid loading all into memory + await forEachBenchmark((b) => { + benchmarkCount++; + const commit = b.commit || commitMap.get(b.commit_id); + if (!commit) { + missing++; + return; + } + + const group = getGroup(b); + if (!group) { + uncategorized.add(b.name.split("/")[0]); + return; + } + if (!groups[group]) return; + + // Random access names have the form: random-access/{dataset}/{pattern}/{format} + // Historical random access names: random-access/{format} + // Other benchmarks use: {query}/{series} + let seriesName, chartName; + const parts = b.name.split("/"); + if (group === "Random Access" && parts.length === 4) { + chartName = `${parts[1]}/${parts[2]}`.toUpperCase().replace(/[_-]/g, " "); + seriesName = rename(parts[3] || "default"); + } else if (group === "Random Access" && parts.length === 2) { + chartName = "RANDOM ACCESS"; + seriesName = rename(parts[1] || "default"); + } else { + seriesName = rename(parts[1] || "default"); + chartName = formatQuery(parts[0]); + } + chartName = normalizeChartName(group, chartName); + if (chartName.includes("PARQUET-UNC")) return; + + // Skip throughput metrics (keep only time/size) + if (b.name.includes(" throughput")) return; + + let unit = b.unit; + if (!unit) { + if (b.name.toLowerCase().includes(" size/")) unit = "bytes"; + else if (b.name.toLowerCase().includes(" ratio ")) unit = "ratio"; + else unit = "ns"; + } + + const sortPos = parts[0].match(/q(\d+)$/i)?.[1] + ? parseInt(RegExp.$1, 10) + : 0; + const idx = commitIdx.get(commit.id); + if (idx === undefined) return; + + let chart = groups[group].get(chartName); + if (!chart) { + let displayUnit = unit; + if (unit === "ns") displayUnit = "ms/iter"; + else if (unit === "bytes") displayUnit = "MiB"; + chart = { + sort_position: sortPos, + commits, + unit: displayUnit, + series: new Map(), + }; + groups[group].set(chartName, chart); + } + + if (!chart.series.has(seriesName)) { + chart.series.set(seriesName, Array(commits.length).fill(null)); + } + + // Convert values: ns -> ms, bytes -> MiB + let val = b.value; + if (unit === "ns" && typeof val === "number") { + val = val / 1e6; // ns to ms + } else if (unit === "bytes" && typeof val === "number") { + val = val / (1024 * 1024); // bytes to MiB + } + + chart.series.get(seriesName)[idx] = { value: val }; + }); + + console.log( + `Processed ${benchmarkCount} benchmarks, ${commitsArr.length} commits`, + ); + + // Log uncategorized benchmarks for debugging + if (uncategorized.size > 0) { + console.log( + `Uncategorized benchmark prefixes (${uncategorized.size}):`, + [...uncategorized].slice(0, 20).join(", "), + ); + } + + // Trim leading empty commits + let firstIdx = commits.length; + for (const gc of Object.values(groups)) { + for (const cd of gc.values()) { + for (const sd of cd.series.values()) { + const i = sd.findIndex((d) => d !== null); + if (i !== -1 && i < firstIdx) firstIdx = i; + } + } + } + + if (firstIdx > 0 && firstIdx < commits.length) { + console.log(`Trimming ${firstIdx} empty commits`); + commits.splice(0, firstIdx); + for (const gc of Object.values(groups)) { + for (const cd of gc.values()) { + cd.commits = commits; + for (const [k, v] of cd.series) cd.series.set(k, v.slice(firstIdx)); + } + } + } + + // Sort charts within groups + for (const gc of Object.values(groups)) { + const sorted = [...gc.entries()].sort( + (a, b) => + a[1].sort_position - b[1].sort_position || a[0].localeCompare(b[0]), + ); + gc.clear(); + for (const [k, v] of sorted) gc.set(k, v); + } + + // Precompute downsampled versions + const downsampled = {}; + for (const [gn, gc] of Object.entries(groups)) { + downsampled[gn] = {}; + for (const [cn, cd] of gc) { + downsampled[gn][cn] = { + "1x": cd, + "2x": downsample(cd, 2), + "4x": downsample(cd, 4), + "8x": downsample(cd, 8), + }; + } + } + + // Count charts per group for logging + const groupCounts = Object.entries(groups) + .map(([n, g]) => `${n}: ${g.size}`) + .filter((s) => !s.endsWith(": 0")); + console.log("Charts per group:", groupCounts.join(", ")); + + store = { + commits, + groups, + metadata: buildMeta(groups, commits), + downsampled, + lastUpdated: new Date().toISOString(), + }; + console.log( + `Refresh done in ${Date.now() - t0}ms (${missing} missing commits)`, + ); + } catch (e) { + console.error("Refresh error:", e); + } } -function serveFile(res, filePath) { - fs.readFile(filePath, (err, data) => { +// Summary calculations +function latestIdx(chart) { + for (let i = chart.commits.length - 1; i >= 0; i--) { + for (const s of chart.series.values()) if (s[i]?.value != null) return i; + } + return -1; +} + +function calcSummary(name, charts) { + if (name === "Random Access") { + for (const q of charts.values()) { + const i = latestIdx(q); + if (i === -1) continue; + const vals = new Map(); + for (const [n, d] of q.series) + if (d[i]?.value != null) vals.set(n, d[i].value); + if (!vals.size) continue; + const min = Math.min(...vals.values()); + return { + type: "randomAccess", + title: "Random Access Performance", + rankings: [...vals] + .map(([n, t]) => ({ name: n, time: t, ratio: t / min })) + .sort((a, b) => a.time - b.time), + explanation: "Random access time | Ratio to fastest (lower is better)", + }; + } + return null; + } + + if (name === "Compression") { + const cc = charts.get("VORTEX:PARQUET ZSTD RATIO COMPRESS TIME"); + const dc = charts.get("VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME"); + if (!cc && !dc) return null; + const i = latestIdx(cc || dc); + if (i === -1) return null; + const collect = (c) => + c + ? [...c.series] + .filter(([n]) => !n.toLowerCase().includes("wide table")) + .map(([, d]) => d[i]?.value) + .filter((v) => v > 0) + .map((v) => 1 / v) + : []; + return { + type: "compression", + title: "Compression Throughput vs Parquet", + compressRatio: geoMean(collect(cc)), + decompressRatio: geoMean(collect(dc)), + datasetCount: collect(cc).length, + explanation: + "Inverse geomean of Vortex/Parquet ratios (higher is better)", + }; + } + + if (name === "Compression Size") { + const c = charts.get("VORTEX:PARQUET ZSTD SIZE"); + if (!c) return null; + const i = latestIdx(c); + if (i === -1) return null; + const ratios = [...c.series] + .filter(([n]) => !n.toLowerCase().includes("wide table")) + .map(([, d]) => d[i]?.value) + .filter((v) => v > 0); + return ratios.length + ? { + type: "compressionSize", + title: "Compression Size Summary", + minRatio: Math.min(...ratios), + meanRatio: geoMean(ratios), + maxRatio: Math.max(...ratios), + datasetCount: ratios.length, + explanation: + "Geomean of Vortex/Parquet size ratios (lower is better)", + } + : null; + } + + if ( + QUERY_SUITES.some( + (s) => + !s.skip && + (name === s.displayName || name.startsWith(s.displayName + " ")), + ) + ) { + const all = new Map(); + for (const q of charts.values()) + for (const n of q.series.keys()) if (!all.has(n)) all.set(n, new Map()); + for (const [qn, qd] of charts) { + for (const [sn, sd] of qd.series) { + for (let i = sd.length - 1; i >= 0; i--) { + if (sd[i]?.value != null) { + all.get(sn).set(qn, sd[i].value); + break; + } + } + } + } + if (!all.size) return null; + + const scores = new Map(); + for (const [sn, qr] of all) { + let total = 0, + max = 0; + for (const v of qr.values()) { + total += v; + max = Math.max(max, v); + } + const penalty = Math.max(300000, max) * 2; + const ratios = []; + for (const qn of charts.keys()) { + let base = Infinity; + for (const m of all.values()) + if (m.has(qn)) base = Math.min(base, m.get(qn)); + if (base < Infinity) + ratios.push((10 + (qr.get(qn) ?? penalty)) / (10 + base)); + } + if (ratios.length) + scores.set(sn, { score: geoMean(ratios), totalRuntime: total }); + } + + return scores.size + ? { + type: "queryBenchmark", + title: "Performance Summary", + rankings: [...scores] + .map(([n, d]) => ({ name: n, ...d })) + .sort((a, b) => a.score - b.score), + explanation: + "Geomean of query time ratio to fastest (lower is better)", + } + : null; + } + return null; +} + +function buildMeta(groups, commits) { + const meta = {}; + for (const [gn, gc] of Object.entries(groups)) { + const charts = [...gc].map(([cn, cd]) => { + const latest = {}; + for (const [sn, sd] of cd.series) { + for (let i = sd.length - 1; i >= 0; i--) + if (sd[i]?.value != null) { + latest[sn] = sd[i].value; + break; + } + } + return { + name: cn, + unit: cd.unit, + series: [...cd.series.keys()], + sortPosition: cd.sort_position, + totalPoints: cd.commits.length, + latestValues: latest, + }; + }); + meta[gn] = { + charts, + totalCharts: charts.length, + hasData: charts.length > 0, + summary: calcSummary(gn, gc), + }; + } + return { + groups: meta, + totalCommits: commits.length, + commits: commits.map((c) => ({ + id: c.id, + message: c.message?.split("\n")[0] || "", + timestamp: c.timestamp, + author: c.author?.name || "Unknown", + })), + lastUpdated: new Date().toISOString(), + }; +} + +// HTTP handlers +const json = (res, code, data) => { + res.writeHead(code, { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }); + res.end(JSON.stringify(data)); +}; + +function serveFile(res, fp) { + fs.readFile(fp, (err, data) => { if (err) { res.writeHead(err.code === "ENOENT" ? 404 : 500); - res.end(err.code === "ENOENT" ? "Not Found" : "Error"); - return; + return res.end(err.code === "ENOENT" ? "Not Found" : "Error"); } - - const ext = path.extname(filePath).toLowerCase(); - const headers = { "Content-Type": MIME[ext] || "application/octet-stream" }; + const ext = path.extname(fp).toLowerCase(); + const hdrs = { "Content-Type": MIME[ext] || "application/octet-stream" }; if (ext === ".js") { - headers["Cache-Control"] = "no-cache"; - headers.Pragma = "no-cache"; + hdrs["Cache-Control"] = "no-cache"; + hdrs["Pragma"] = "no-cache"; } - - res.writeHead(200, headers); + res.writeHead(200, hdrs); res.end(data); }); } -async function handleData(res, groupName, chartName, params) { - try { - const payload = await benchmarks.getChartData(groupName, chartName, { - start: params.get("start"), - end: params.get("end"), - last: params.get("last"), - startIdx: params.has("startIdx") ? params.get("startIdx") : null, - endIdx: params.has("endIdx") ? params.get("endIdx") : null, - }); - json(res, 200, payload); - } catch (error) { - if (error.message === "Loading") { - const status = benchmarks.status; - json(res, 503, getLoadingPayload(), loadingHeaders(status)); - return; +function handleData(res, group, chart, start, end, last, startIdx, endIdx) { + if (!store.downsampled) return json(res, 503, { error: "Loading" }); + const gd = store.downsampled[group]; + if (!gd) return json(res, 404, { error: "Group not found" }); + const cv = gd[chart]; + if (!cv) return json(res, 404, { error: "Chart not found" }); + + const full = cv["1x"]; + const ts = (c) => + typeof c?.timestamp === "number" + ? c.timestamp + : new Date(c?.timestamp).getTime(); + + let si = 0, + ei = full.commits.length - 1; + + // Support "last=N" parameter to get the last N commits + if (last && !start && !end && startIdx === null && endIdx === null) { + const n = parseInt(last, 10); + if (n > 0 && n < full.commits.length) { + si = full.commits.length - n; } - if (error.statusCode === 404) { - json(res, 404, { error: error.message }); - return; + } else if (startIdx !== null || endIdx !== null) { + // Support index-based range (startIdx, endIdx) + if (startIdx !== null) si = Math.max(0, parseInt(startIdx, 10)); + if (endIdx !== null) + ei = Math.min(full.commits.length - 1, parseInt(endIdx, 10)); + } else { + // Timestamp-based range + if (start) { + const t = +start, + i = full.commits.findIndex((c) => ts(c) >= t); + if (i !== -1) si = i; } - console.error("Data request error:", error); - json(res, 500, { error: "Internal server error" }); + if (end) { + const t = +end; + for (let i = ei; i >= 0; i--) + if (ts(full.commits[i]) <= t) { + ei = i; + break; + } + } + } + + const len = ei - si + 1; + const ver = + len <= MAX_POINTS + ? "1x" + : len <= MAX_POINTS * 2 + ? "2x" + : len <= MAX_POINTS * 4 + ? "4x" + : "8x"; + const cd = cv[ver]; + const val = (d) => d?.value ?? (typeof d === "number" ? d : null); + + let commits, series; + if (ver === "1x") { + commits = full.commits.slice(si, ei + 1); + series = Object.fromEntries( + [...full.series].map(([n, d]) => [n, d.slice(si, ei + 1).map(val)]), + ); + } else { + const s = +ver[0], + dsi = Math.floor(si / s), + dei = Math.min(Math.ceil(ei / s), cd.commits.length - 1); + commits = cd.commits.slice(dsi, dei + 1); + series = Object.fromEntries( + [...cd.series].map(([n, d]) => [n, d.slice(dsi, dei + 1).map(val)]), + ); } + + json(res, 200, { + group, + chart, + unit: cd.unit, + downsampleLevel: ver, + originalLength: full.commits.length, + requestedRange: { startIndex: si, endIndex: ei, length: len }, + commits: commits.map((c) => ({ + id: c.id, + message: c.message?.split("\n")[0] || "", + timestamp: c.timestamp, + author: c.author?.name || "Unknown", + url: c.url, + })), + series, + }); } const server = http.createServer((req, res) => { - const [pathName, rawQuery] = req.url.split("?"); - const params = new URLSearchParams(rawQuery || ""); + const [path_, qs] = req.url.split("?"); + const params = new URLSearchParams(qs || ""); if (req.method === "OPTIONS") { res.writeHead(204, { @@ -137,79 +735,41 @@ const server = http.createServer((req, res) => { "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "Content-Type", }); - res.end(); - return; + return res.end(); } - if (pathName === "/api/metadata") { - const metadata = benchmarks.metadata; - const status = benchmarks.status; - json( - res, - metadata ? 200 : 503, - metadata || getLoadingPayload(), - metadata ? {} : loadingHeaders(status), - ); - return; - } + if (path_ === "/api/metadata") + return store.metadata + ? json(res, 200, store.metadata) + : json(res, 503, { error: "Loading" }); - if (pathName === "/api/health") { - const status = benchmarks.status; - json(res, status.ready ? 200 : 503, status, loadingHeaders(status)); - return; - } - - if (pathName.startsWith("/api/data/")) { - const segments = pathName.slice(10).split("/"); - handleData( + if (path_.startsWith("/api/data/")) { + const p = path_.slice(10).split("/"); + return handleData( res, - decodeURIComponent(segments[0] || ""), - decodeURIComponent(segments.slice(1).join("/") || ""), - params, - ).catch((error) => { - console.error("Unhandled data handler error:", error); - json(res, 500, { error: "Internal server error" }); - }); - return; + decodeURIComponent(p[0] || ""), + decodeURIComponent(p.slice(1).join("/") || ""), + params.get("start"), + params.get("end"), + params.get("last"), + params.has("startIdx") ? params.get("startIdx") : null, + params.has("endIdx") ? params.get("endIdx") : null, + ); } - const filePath = path.join( - __dirname, - "dist", - pathName === "/" ? "index.html" : pathName, - ); - if (!filePath.startsWith(__dirname) || filePath.includes("/sample/")) { + const fp = path.join(__dirname, "dist", path_ === "/" ? "index.html" : path_); + if (!fp.startsWith(__dirname) || fp.includes("/sample/")) { res.writeHead(403); - res.end("Forbidden"); - return; + return res.end("Forbidden"); } - serveFile(res, filePath); + serveFile(res, fp); }); -function start() { +async function start() { console.log("Starting server..."); - server.listen(PORT, () => { - console.log(`Server at http://localhost:${PORT}`); - }); - triggerRefresh(); - refreshIntervalId = setInterval(triggerRefresh, Number(REFRESH_INTERVAL)); -} - -async function shutdown() { - if (refreshIntervalId) { - clearInterval(refreshIntervalId); - refreshIntervalId = null; - } - await benchmarks.close(); - server.close(); + await refresh(); + setInterval(refresh, REFRESH_INTERVAL); + server.listen(PORT, () => console.log(`Server at http://localhost:${PORT}`)); } -process.on("SIGINT", () => { - shutdown().finally(() => process.exit(0)); -}); - -process.on("SIGTERM", () => { - shutdown().finally(() => process.exit(0)); -}); - -start(); +start().catch(console.error); diff --git a/benchmarks-website/src/App.jsx b/benchmarks-website/src/App.jsx index f5edf0e92e2..0df05bebf01 100644 --- a/benchmarks-website/src/App.jsx +++ b/benchmarks-website/src/App.jsx @@ -1,13 +1,15 @@ -import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import Header from './components/Header'; import Sidebar from './components/Sidebar'; import BenchmarkSection from './components/BenchmarkSection'; import Modal from './components/Modal'; +import { fetchMetadata } from './api'; import { BENCHMARK_CONFIGS, CATEGORY_TAGS } from './config'; -import useBenchmarkMetadata from './hooks/useBenchmarkMetadata'; export default function App() { - const { metadata, loading, error } = useBenchmarkMetadata(); + const [metadata, setMetadata] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); const [expandedGroups, setExpandedGroups] = useState(new Set()); const [sidebarOpen, setSidebarOpen] = useState(false); const [categoryFilter, setCategoryFilter] = useState('all'); @@ -15,15 +17,28 @@ export default function App() { const [viewMode, setViewMode] = useState('grid'); const [modalChart, setModalChart] = useState(null); const [showBackToTop, setShowBackToTop] = useState(false); + const metadataFetched = useRef(false); useEffect(() => { - if (!metadata?.groups) return; + if (metadataFetched.current) return; + metadataFetched.current = true; - const params = new URLSearchParams(window.location.search); - if (params.get('expanded') === 'true') { - setExpandedGroups(new Set(Object.keys(metadata.groups))); + async function loadMetadata() { + try { + const data = await fetchMetadata(); + setMetadata(data); + const params = new URLSearchParams(window.location.search); + if (params.get('expanded') === 'true' && data?.groups) { + setExpandedGroups(new Set(Object.keys(data.groups))); + } + } catch (err) { + setError(err.message); + } finally { + setLoading(false); + } } - }, [metadata]); + loadMetadata(); + }, []); useEffect(() => { const handleScroll = () => { @@ -40,6 +55,7 @@ export default function App() { const hash = window.location.hash; if (hash && hash.startsWith('#group-')) { const groupId = hash.slice(1); // Remove the '#' + const groupName = groupId.replace('group-', '').replace(/-/g, ' '); // Find the matching group (case-insensitive, handle hyphenated names) const matchingGroup = Object.keys(metadata.groups).find(name => diff --git a/benchmarks-website/src/api.js b/benchmarks-website/src/api.js index 7fbcf781ce3..042ea7a6f0b 100644 --- a/benchmarks-website/src/api.js +++ b/benchmarks-website/src/api.js @@ -1,53 +1,9 @@ const API_BASE = ''; -function parseRetryAfterMs(value) { - if (!value) return null; - - const seconds = Number.parseInt(value, 10); - if (Number.isFinite(seconds) && seconds >= 0) { - return seconds * 1000; - } - - const retryAt = Date.parse(value); - if (Number.isNaN(retryAt)) { - return null; - } - - return Math.max(0, retryAt - Date.now()); -} - -async function readResponse(response) { - const contentType = response.headers.get('content-type') || ''; - - if (contentType.includes('application/json')) { - return response.json(); - } - - const text = await response.text(); - return text ? { error: text } : null; -} - -export async function fetchMetadata(options = {}) { - const response = await fetch(`${API_BASE}/api/metadata`, { - cache: 'no-store', - signal: options.signal, - }); - const payload = await readResponse(response); - - if (!response.ok) { - const message = payload?.lastRefreshError - ? `Failed to fetch metadata: ${payload.lastRefreshError}` - : payload?.error - ? `Failed to fetch metadata: ${payload.error}` - : `Failed to fetch metadata: ${response.status}`; - const error = new Error(message); - error.status = response.status; - error.payload = payload; - error.retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after')); - throw error; - } - - return payload; +export async function fetchMetadata() { + const response = await fetch(`${API_BASE}/api/metadata`); + if (!response.ok) throw new Error(`Failed to fetch metadata: ${response.status}`); + return response.json(); } export async function fetchChartData(groupName, chartName, options = {}) { @@ -79,19 +35,7 @@ export async function fetchChartData(groupName, chartName, options = {}) { if (params.toString()) url += '?' + params.toString(); - const response = await fetch(url, { cache: 'no-store' }); - const payload = await readResponse(response); - - if (!response.ok) { - const message = payload?.error - ? `Failed to fetch chart data: ${payload.error}` - : `Failed to fetch chart data: ${response.status}`; - const error = new Error(message); - error.status = response.status; - error.payload = payload; - error.retryAfterMs = parseRetryAfterMs(response.headers.get('retry-after')); - throw error; - } - - return payload; + const response = await fetch(url); + if (!response.ok) throw new Error(`Failed to fetch chart data: ${response.status}`); + return response.json(); } diff --git a/benchmarks-website/src/components/ChartContainer.jsx b/benchmarks-website/src/components/ChartContainer.jsx index 681192a09a4..1dfb193e836 100644 --- a/benchmarks-website/src/components/ChartContainer.jsx +++ b/benchmarks-website/src/components/ChartContainer.jsx @@ -277,7 +277,7 @@ export default function ChartContainer({ const { series, commits } = chartData; const datasets = []; - const labels = commits.map(c => formatDate(c?.timestamp)); + const labels = commits.map(c => formatDate(c.timestamp)); Object.entries(series).forEach(([seriesName, points]) => { // Apply removed datasets filter diff --git a/benchmarks-website/src/components/Header.jsx b/benchmarks-website/src/components/Header.jsx index 952cfd80087..95edda98560 100644 --- a/benchmarks-website/src/components/Header.jsx +++ b/benchmarks-website/src/components/Header.jsx @@ -1,3 +1,4 @@ +import React from 'react'; export default function Header({ sidebarOpen, @@ -24,8 +25,8 @@ export default function Header({ - - + + Vortex diff --git a/benchmarks-website/src/components/Modal.jsx b/benchmarks-website/src/components/Modal.jsx index f118e18f795..b9bb7d419b1 100644 --- a/benchmarks-website/src/components/Modal.jsx +++ b/benchmarks-website/src/components/Modal.jsx @@ -95,7 +95,7 @@ export default function Modal({ chartData, onClose }) { const { series, commits } = data; const datasets = []; - const labels = commits.map(c => formatDate(c?.timestamp)); + const labels = commits.map(c => formatDate(c.timestamp)); Object.entries(series).forEach(([seriesName, points]) => { if (config?.removedDatasets?.has(seriesName)) return; diff --git a/benchmarks-website/src/hooks/useBenchmarkMetadata.js b/benchmarks-website/src/hooks/useBenchmarkMetadata.js deleted file mode 100644 index f287560e0c7..00000000000 --- a/benchmarks-website/src/hooks/useBenchmarkMetadata.js +++ /dev/null @@ -1,75 +0,0 @@ -import { useEffect, useState } from 'react'; -import { fetchMetadata } from '../api'; - -const METADATA_RETRY_DELAY_MS = 1000; - -const initialState = { - metadata: null, - loading: true, - error: null, -}; - -export default function useBenchmarkMetadata() { - const [state, setState] = useState(initialState); - - useEffect(() => { - let active = true; - let retryTimer = null; - let activeController = null; - - const clearRetry = () => { - if (retryTimer !== null) { - window.clearTimeout(retryTimer); - retryTimer = null; - } - }; - - const loadMetadata = async () => { - clearRetry(); - activeController?.abort(); - activeController = new AbortController(); - - try { - const metadata = await fetchMetadata({ signal: activeController.signal }); - if (!active) return; - - setState({ - metadata, - loading: false, - error: null, - }); - } catch (error) { - if (!active || error.name === 'AbortError') return; - - if (error.status === 503 && error.payload?.status !== 'error') { - setState((current) => ({ - metadata: current.metadata, - loading: true, - error: null, - })); - retryTimer = window.setTimeout( - loadMetadata, - error.retryAfterMs ?? METADATA_RETRY_DELAY_MS, - ); - return; - } - - setState({ - metadata: null, - loading: false, - error: error.message, - }); - } - }; - - loadMetadata(); - - return () => { - active = false; - clearRetry(); - activeController?.abort(); - }; - }, []); - - return state; -} diff --git a/benchmarks-website/store/benchmark-store.js b/benchmarks-website/store/benchmark-store.js deleted file mode 100644 index 8f112d94f4d..00000000000 --- a/benchmarks-website/store/benchmark-store.js +++ /dev/null @@ -1,354 +0,0 @@ -import { DuckDBInstance } from "@duckdb/node-api"; -import { prepareInputFiles } from "./cache.js"; -import { queryRows, withConnection } from "./db.js"; -import { downsample, downsampleLevel } from "./downsample.js"; -import { buildMetadata, collectDiagnostics } from "./metadata.js"; -import { buildBootstrapSql } from "./sql.js"; -import { firstLine } from "./utils.js"; - -const CHART_RANGE_QUERY = ` - with requested_commits as ( - select commit_idx - from active_commits - where commit_idx between $start_idx and $end_idx - ), - requested_series as ( - select distinct series_name - from benchmark_points_active - where group_name = $group_name - and chart_name = $chart_name - ), - dense_points as ( - select - rs.series_name, - rc.commit_idx, - bpa.value - from requested_series rs - cross join requested_commits rc - left join benchmark_points_active bpa - on bpa.group_name = $group_name - and bpa.chart_name = $chart_name - and bpa.series_name = rs.series_name - and bpa.commit_idx = rc.commit_idx - ) - select - series_name, - list(value order by commit_idx) as values - from dense_points - group by 1 - order by 1 -`; - -async function buildStoreState({ instance, inputs }) { - await withConnection(instance, async (connection) => { - await connection.run("begin transaction"); - try { - await connection.run(buildBootstrapSql(inputs.dataPath, inputs.commitsPath)); - await connection.run("commit"); - } catch (error) { - try { - await connection.run("rollback"); - } catch { - // Best effort: keep the previous committed state if the rebuild failed. - } - throw error; - } - }); - - const lastUpdated = new Date().toISOString(); - const [{ commits, metadata, chartIndex }, diagnostics] = await Promise.all([ - buildMetadata(instance, lastUpdated), - withConnection(instance, collectDiagnostics), - ]); - - return { - commits, - metadata, - chartIndex, - lastUpdated, - diagnostics, - }; -} - -function commitTimestamp(commit) { - return typeof commit.timestamp === "number" - ? commit.timestamp - : new Date(commit.timestamp).getTime(); -} - -function resolveRequestedRange(commits, options) { - const totalCommits = commits.length; - let startIdx = 0; - let endIdx = totalCommits - 1; - - if ( - options.last && - !options.start && - !options.end && - options.startIdx == null && - options.endIdx == null - ) { - const count = Number.parseInt(options.last, 10); - if (count > 0 && count < totalCommits) { - startIdx = totalCommits - count; - } - } else if (options.startIdx != null || options.endIdx != null) { - if (options.startIdx != null) { - startIdx = Math.max(0, Number.parseInt(options.startIdx, 10)); - } - if (options.endIdx != null) { - endIdx = Math.min(totalCommits - 1, Number.parseInt(options.endIdx, 10)); - } - } else { - if (options.start) { - const startTs = Number(options.start); - const idx = commits.findIndex((commit) => commitTimestamp(commit) >= startTs); - if (idx !== -1) startIdx = idx; - } - - if (options.end) { - const endTs = Number(options.end); - for (let i = endIdx; i >= 0; i--) { - if (commitTimestamp(commits[i]) <= endTs) { - endIdx = i; - break; - } - } - } - } - - return { - startIdx: Math.max(0, Math.min(startIdx, totalCommits - 1)), - endIdx: Math.max(startIdx, Math.min(endIdx, totalCommits - 1)), - }; -} - -function serializeRequestedCommits(commits, startIdx, endIdx) { - return commits - .slice(startIdx, endIdx + 1) - .map(({ id, message, timestamp, author, url }) => ({ - id, - message: firstLine(message), - timestamp, - author, - url, - })); -} - -function buildSeriesMap(rows) { - return new Map( - rows.map((row) => [ - row.series_name, - (row.values || []).map((value) => (value == null ? null : value)), - ]), - ); -} - -export class BenchmarkStore { - constructor(options) { - this.options = options; - this.state = null; - this.instance = null; - this.instancePromise = null; - this.refreshPromise = null; - this.remoteCheckTimer = null; - this.lastRefreshStartedAt = null; - this.lastRefreshCompletedAt = null; - this.lastRefreshError = null; - } - - get metadata() { - return this.state?.metadata || null; - } - - get status() { - let state = "idle"; - - if (this.state) { - if (this.refreshPromise) { - state = "refreshing"; - } else if (this.lastRefreshError) { - state = "stale"; - } else { - state = "ready"; - } - } else if (this.lastRefreshError) { - state = "error"; - } else if (this.refreshPromise) { - state = "loading"; - } - - return { - state, - ready: Boolean(this.state), - refreshing: Boolean(this.refreshPromise), - hasData: Boolean(this.state?.metadata), - lastUpdated: this.state?.lastUpdated || null, - lastRefreshStartedAt: this.lastRefreshStartedAt, - lastRefreshCompletedAt: this.lastRefreshCompletedAt, - lastRefreshError: this.lastRefreshError, - }; - } - - async getInstance() { - if (this.instance) return this.instance; - - if (!this.instancePromise) { - this.instancePromise = DuckDBInstance.create(":memory:") - .then((instance) => { - this.instance = instance; - return instance; - }) - .finally(() => { - this.instancePromise = null; - }); - } - - return this.instancePromise; - } - - scheduleRemoteCheck() { - if (this.remoteCheckTimer) return; - - this.remoteCheckTimer = setTimeout(() => { - this.remoteCheckTimer = null; - this.refresh({ forceRemoteCheck: true }).catch(() => {}); - }, 0); - } - - async refresh({ forceRemoteCheck = false } = {}) { - if (this.refreshPromise) return this.refreshPromise; - - this.lastRefreshStartedAt = new Date().toISOString(); - this.lastRefreshError = null; - this.refreshPromise = (async () => { - const startedAt = Date.now(); - const hasState = Boolean(this.state); - const inputs = await prepareInputFiles({ - ...this.options, - preferCached: !hasState && !forceRemoteCheck, - forceRemoteCheck, - }); - - if (hasState && !inputs.changed) { - this.lastRefreshCompletedAt = new Date().toISOString(); - console.log( - `Refresh skipped in ${Date.now() - startedAt}ms (${inputs.source})`, - ); - return; - } - - const instance = await this.getInstance(); - const nextState = await buildStoreState({ instance, inputs }); - - console.log( - `Processed ${nextState.diagnostics.benchmarkCount} benchmarks, ${nextState.commits.length} commits`, - ); - - if (nextState.diagnostics.uncategorized.length > 0) { - console.log( - `Uncategorized benchmark prefixes (${nextState.diagnostics.uncategorized.length}):`, - nextState.diagnostics.uncategorized.join(", "), - ); - } - - const chartCounts = nextState.diagnostics.groupCounts - .map((row) => `${row.group_name}: ${row.chart_count}`) - .filter((entry) => !entry.endsWith(": 0")); - console.log("Charts per group:", chartCounts.join(", ")); - - this.state = nextState; - this.lastRefreshCompletedAt = nextState.lastUpdated; - - console.log( - `Refresh done in ${Date.now() - startedAt}ms (${nextState.diagnostics.missingCommits} missing commits, source: ${inputs.source})`, - ); - - if (inputs.deferRemoteCheck) { - console.log( - "Serving cached benchmark files for startup; scheduling remote revalidation", - ); - this.scheduleRemoteCheck(); - } - })() - .catch((error) => { - this.lastRefreshError = error?.message || String(error); - console.error("Refresh error:", error); - throw error; - }) - .finally(() => { - this.refreshPromise = null; - }); - - return this.refreshPromise; - } - - async close() { - if (this.remoteCheckTimer) { - clearTimeout(this.remoteCheckTimer); - this.remoteCheckTimer = null; - } - - this.state = null; - - if (this.instance) { - this.instance.closeSync(); - this.instance = null; - } - } - - async getChartData(groupName, chartName, options = {}) { - if (!this.state) { - throw new Error("Loading"); - } - - const chart = this.state.chartIndex.get(`${groupName}\u0000${chartName}`); - if (!chart) { - const error = new Error("Chart not found"); - error.statusCode = 404; - throw error; - } - - const { commits, metadata } = this.state; - const instance = this.instance; - if (!instance) { - throw new Error("Loading"); - } - - const { startIdx, endIdx } = resolveRequestedRange(commits, options); - const rows = await withConnection(instance, async (connection) => - queryRows(connection, CHART_RANGE_QUERY, { - group_name: groupName, - chart_name: chartName, - start_idx: startIdx, - end_idx: endIdx, - }), - ); - - const requestedCommits = serializeRequestedCommits(commits, startIdx, endIdx); - const selected = { - commits: requestedCommits, - series: buildSeriesMap(rows), - }; - const level = downsampleLevel(requestedCommits.length); - const sampled = - level === "1x" - ? selected - : downsample(selected, Number.parseInt(level, 10)); - - return { - group: groupName, - chart: chartName, - unit: chart.unit, - downsampleLevel: level, - originalLength: metadata.totalCommits, - requestedRange: { - startIndex: startIdx, - endIndex: endIdx, - length: endIdx - startIdx + 1, - }, - commits: sampled.commits, - series: Object.fromEntries(sampled.series), - }; - } -} diff --git a/benchmarks-website/store/cache.js b/benchmarks-website/store/cache.js deleted file mode 100644 index 7141b353b24..00000000000 --- a/benchmarks-website/store/cache.js +++ /dev/null @@ -1,174 +0,0 @@ -import fs from "fs"; -import fsp from "fs/promises"; -import os from "os"; -import path from "path"; -import { Readable } from "stream"; -import { pipeline } from "stream/promises"; -import { fileURLToPath } from "url"; -import { DEFAULT_CACHE_DIR_NAME } from "./constants.js"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const WEBSITE_DIR = path.dirname(__dirname); - -function cachePaths(cacheDir) { - return { - dataPath: path.join(cacheDir, "data.json.gz"), - commitsPath: path.join(cacheDir, "commits.json"), - manifestPath: path.join(cacheDir, "manifest.json"), - }; -} - -async function pathExists(filePath) { - try { - await fsp.access(filePath); - return true; - } catch { - return false; - } -} - -async function readJsonFile(filePath) { - try { - const raw = await fsp.readFile(filePath, "utf8"); - return JSON.parse(raw); - } catch (error) { - if (error.code === "ENOENT") return null; - throw error; - } -} - -async function writeJsonFileAtomic(filePath, value) { - const tempPath = `${filePath}.tmp-${process.pid}-${Date.now()}`; - await fsp.writeFile(tempPath, JSON.stringify(value, null, 2)); - await fsp.rename(tempPath, filePath); -} - -async function downloadToFile(url, destination, metadata = {}) { - const headers = {}; - if (metadata.etag) headers["If-None-Match"] = metadata.etag; - if (metadata.lastModified) headers["If-Modified-Since"] = metadata.lastModified; - - const response = await fetch(url, { headers }); - if (response.status === 304) { - return { - changed: false, - metadata: { - ...metadata, - checkedAt: new Date().toISOString(), - }, - }; - } - - if (!response.ok) { - throw new Error(`Fetch failed: ${url} ${response.status}`); - } - - if (!response.body) { - throw new Error(`Fetch failed: ${url} empty body`); - } - - const tempPath = `${destination}.tmp-${process.pid}-${Date.now()}`; - try { - await pipeline(Readable.fromWeb(response.body), fs.createWriteStream(tempPath)); - await fsp.rename(tempPath, destination); - } finally { - if (await pathExists(tempPath)) { - await fsp.unlink(tempPath); - } - } - - return { - changed: true, - metadata: { - url, - etag: response.headers.get("etag"), - lastModified: response.headers.get("last-modified"), - fetchedAt: new Date().toISOString(), - checkedAt: new Date().toISOString(), - }, - }; -} - -export async function prepareInputFiles({ - dataUrl, - commitsUrl, - useLocalData, - cacheDir, - preferCached = false, - forceRemoteCheck = false, -}) { - if (useLocalData) { - return { - dataPath: path.join(WEBSITE_DIR, "sample/data.json"), - commitsPath: path.join(WEBSITE_DIR, "sample/commits.json"), - changed: true, - source: "local", - deferRemoteCheck: false, - }; - } - - const resolvedCacheDir = - cacheDir || path.join(os.tmpdir(), DEFAULT_CACHE_DIR_NAME); - await fsp.mkdir(resolvedCacheDir, { recursive: true }); - - const { dataPath, commitsPath, manifestPath } = cachePaths(resolvedCacheDir); - const manifest = (await readJsonFile(manifestPath)) || {}; - const [hasDataFile, hasCommitsFile] = await Promise.all([ - pathExists(dataPath), - pathExists(commitsPath), - ]); - const hasCachedFiles = hasDataFile && hasCommitsFile; - - if (preferCached && hasCachedFiles && !forceRemoteCheck) { - return { - dataPath, - commitsPath, - changed: true, - source: "cache", - deferRemoteCheck: true, - }; - } - - try { - const [dataResult, commitsResult] = await Promise.all([ - downloadToFile(dataUrl, dataPath, manifest.data || {}), - downloadToFile(commitsUrl, commitsPath, manifest.commits || {}), - ]); - - await writeJsonFileAtomic(manifestPath, { - version: 1, - dataUrl, - commitsUrl, - data: dataResult.metadata, - commits: commitsResult.metadata, - updatedAt: new Date().toISOString(), - }); - - return { - dataPath, - commitsPath, - changed: !hasCachedFiles || dataResult.changed || commitsResult.changed, - source: - !hasCachedFiles || dataResult.changed || commitsResult.changed - ? "remote" - : "cache", - deferRemoteCheck: false, - }; - } catch (error) { - if (hasCachedFiles) { - console.warn( - `Falling back to cached benchmark files after refresh failed: ${error.message}`, - ); - return { - dataPath, - commitsPath, - changed: false, - source: "stale-cache", - deferRemoteCheck: false, - }; - } - - throw error; - } -} diff --git a/benchmarks-website/store/constants.js b/benchmarks-website/store/constants.js deleted file mode 100644 index fa5799bc3e5..00000000000 --- a/benchmarks-website/store/constants.js +++ /dev/null @@ -1,20 +0,0 @@ -import { FAN_OUT_GROUPS, QUERY_SUITES } from "../src/config.js"; - -export const MAX_POINTS = 200; -export const DEFAULT_CACHE_DIR_NAME = "vortex-benchmarks-website-cache"; - -export const GROUPS = [ - "Random Access", - "Compression", - "Compression Size", - ...QUERY_SUITES.filter((suite) => !suite.skip && !suite.fanOut).map( - (suite) => suite.displayName, - ), - ...FAN_OUT_GROUPS, -]; - -export const QUERY_GROUP_EXCLUSIONS = new Set([ - "Random Access", - "Compression", - "Compression Size", -]); diff --git a/benchmarks-website/store/db.js b/benchmarks-website/store/db.js deleted file mode 100644 index 543c33e2ab7..00000000000 --- a/benchmarks-website/store/db.js +++ /dev/null @@ -1,33 +0,0 @@ -function normalizeValue(value) { - if (typeof value === "bigint") { - const asNumber = Number(value); - return Number.isSafeInteger(asNumber) ? asNumber : value.toString(); - } - - if (Array.isArray(value)) { - return value.map(normalizeValue); - } - - if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value).map(([key, child]) => [key, normalizeValue(child)]), - ); - } - - return value; -} - -export async function queryRows(connection, sql, values) { - const result = await connection.run(sql, values); - const rows = await result.getRowObjectsJS(); - return rows.map((row) => normalizeValue(row)); -} - -export async function withConnection(instance, fn) { - const connection = await instance.connect(); - try { - return await fn(connection); - } finally { - connection.closeSync(); - } -} diff --git a/benchmarks-website/store/downsample.js b/benchmarks-website/store/downsample.js deleted file mode 100644 index 47a774184cc..00000000000 --- a/benchmarks-website/store/downsample.js +++ /dev/null @@ -1,68 +0,0 @@ -import { LTTB } from "downsample"; -import { MAX_POINTS } from "./constants.js"; - -function lttbIndices(seriesMap, target) { - const keys = [...seriesMap.keys()]; - if (!keys.length) return []; - - const length = seriesMap.get(keys[0])?.length || 0; - if (length <= target) return [...Array(length).keys()]; - - const averages = Array(length); - for (let i = 0; i < length; i++) { - let sum = 0; - let count = 0; - - for (const series of seriesMap.values()) { - const value = series[i]?.value ?? series[i]; - if (value != null && !Number.isNaN(value)) { - sum += value; - count++; - } - } - - averages[i] = [i, count ? sum / count : 0]; - } - - const indices = LTTB(averages, target).map((point) => Math.round(point[0])); - if (!indices.includes(0)) indices.unshift(0); - if (!indices.includes(length - 1)) indices.push(length - 1); - return indices.sort((a, b) => a - b); -} - -export function downsample(data, factor) { - const target = Math.ceil(data.commits.length / factor); - if (target >= data.commits.length) return data; - - const indices = [...new Set( - lttbIndices(data.series, target).filter( - (index) => - Number.isInteger(index) && - index >= 0 && - index < data.commits.length, - ), - )].sort((a, b) => a - b); - - if (!indices.length) return data; - - const series = new Map(); - for (const [seriesName, values] of data.series) { - series.set( - seriesName, - indices.map((index) => values[index]), - ); - } - - return { - ...data, - commits: indices.map((index) => data.commits[index]), - series, - }; -} - -export function downsampleLevel(length) { - if (length <= MAX_POINTS) return "1x"; - if (length <= MAX_POINTS * 2) return "2x"; - if (length <= MAX_POINTS * 4) return "4x"; - return "8x"; -} diff --git a/benchmarks-website/store/metadata.js b/benchmarks-website/store/metadata.js deleted file mode 100644 index 296e7df2c6a..00000000000 --- a/benchmarks-website/store/metadata.js +++ /dev/null @@ -1,401 +0,0 @@ -import { GROUPS, QUERY_GROUP_EXCLUSIONS } from "./constants.js"; -import { queryRows, withConnection } from "./db.js"; -import { firstLine } from "./utils.js"; - -const COMMITS_QUERY = ` - select - commit_idx, - id, - message, - timestamp, - author, - url - from active_commits - order by commit_idx -`; - -const CHART_ROWS_QUERY = ` - select - cd.group_name, - cd.chart_name, - cd.unit, - cd.sort_position, - list(cslv.series_name order by cslv.series_name) as series_names, - json_group_object(cslv.series_name, cslv.latest_value) as latest_values - from chart_defs cd - join chart_series_latest_values cslv - on cslv.group_name = cd.group_name - and cslv.chart_name = cd.chart_name - group by 1, 2, 3, 4 - order by 1, 4, 2 -`; - -const RANDOM_ACCESS_SUMMARY_QUERY = ` - with selected_chart as ( - select chart_name - from chart_defs - where group_name = 'Random Access' - order by sort_position, chart_name - limit 1 - ) - select - series_name as name, - value as time, - value / min(value) over () as ratio - from chart_latest_values - where group_name = 'Random Access' - and chart_name = (select chart_name from selected_chart) - order by value, series_name -`; - -const COMPRESSION_SUMMARY_QUERY = ` - with anchor_chart as ( - select chart_name - from chart_defs - where group_name = 'Compression' - and chart_name in ( - 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME', - 'VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME' - ) - order by case - when chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' then 0 - else 1 - end - limit 1 - ), - anchor_idx as ( - select latest_commit_idx as commit_idx - from chart_latest_idx - where group_name = 'Compression' - and chart_name = (select chart_name from anchor_chart) - ) - select - ( - select geomean(1.0 / value) - from benchmark_points_active - where group_name = 'Compression' - and chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' - and commit_idx = (select commit_idx from anchor_idx) - and lower(series_name) not like '%wide table%' - and value > 0 - ) as compress_ratio, - ( - select geomean(1.0 / value) - from benchmark_points_active - where group_name = 'Compression' - and chart_name = 'VORTEX:PARQUET ZSTD RATIO DECOMPRESS TIME' - and commit_idx = (select commit_idx from anchor_idx) - and lower(series_name) not like '%wide table%' - and value > 0 - ) as decompress_ratio, - ( - select count(*) - from benchmark_points_active - where group_name = 'Compression' - and chart_name = 'VORTEX:PARQUET ZSTD RATIO COMPRESS TIME' - and commit_idx = (select commit_idx from anchor_idx) - and lower(series_name) not like '%wide table%' - and value > 0 - ) as dataset_count -`; - -const COMPRESSION_SIZE_SUMMARY_QUERY = ` - with anchor_idx as ( - select latest_commit_idx as commit_idx - from chart_latest_idx - where group_name = 'Compression Size' - and chart_name = 'VORTEX:PARQUET ZSTD SIZE' - ) - select - min(value) as min_ratio, - geomean(value) as mean_ratio, - max(value) as max_ratio, - count(*) as dataset_count - from benchmark_points_active - where group_name = 'Compression Size' - and chart_name = 'VORTEX:PARQUET ZSTD SIZE' - and commit_idx = (select commit_idx from anchor_idx) - and lower(series_name) not like '%wide table%' - and value > 0 -`; - -const QUERY_SUMMARY_QUERY = ` - with per_series as ( - select * - from chart_series_latest_values - where group_name not in ( - 'Random Access', - 'Compression', - 'Compression Size' - ) - ), - chart_bases as ( - select - group_name, - chart_name, - min(latest_value) as base_value - from per_series - group by 1, 2 - ), - series_totals as ( - select - group_name, - series_name, - sum(latest_value) as total_runtime, - max(latest_value) as max_runtime - from per_series - group by 1, 2 - ), - group_series as ( - select distinct group_name, series_name - from per_series - ), - ratios as ( - select - gs.group_name, - gs.series_name, - ((10.0 + coalesce(ps.latest_value, greatest(300000.0, st.max_runtime) * 2.0)) - / (10.0 + cb.base_value)) as ratio - from group_series gs - join series_totals st - on st.group_name = gs.group_name - and st.series_name = gs.series_name - join chart_defs cd - on cd.group_name = gs.group_name - join chart_bases cb - on cb.group_name = cd.group_name - and cb.chart_name = cd.chart_name - left join per_series ps - on ps.group_name = cd.group_name - and ps.chart_name = cd.chart_name - and ps.series_name = gs.series_name - ) - select - gs.group_name, - gs.series_name as name, - geomean(r.ratio) as score, - st.total_runtime - from group_series gs - join ratios r - on r.group_name = gs.group_name - and r.series_name = gs.series_name - join series_totals st - on st.group_name = gs.group_name - and st.series_name = gs.series_name - group by 1, 2, 4 - order by 1, 3, 4, 2 -`; - -const DIAGNOSTICS_COUNTS_QUERY = ` - select - (select count(*) from raw_benchmarks) as benchmark_count, - ( - select count(*) - from classified_benchmarks cb - left join commit_dim cd - on cd.id = cb.resolved_commit_id - where cb.resolved_commit_id is null - or cd.id is null - ) as missing_commits -`; - -const UNCATEGORIZED_QUERY = ` - select distinct split_part(name, '/', 1) as prefix - from classified_benchmarks - where group_name is null - order by prefix - limit 20 -`; - -const GROUP_COUNTS_QUERY = ` - select - group_name, - count(*) as chart_count - from chart_defs - group by 1 - order by 1 -`; - -function buildEmptyGroups() { - return Object.fromEntries( - GROUPS.map((groupName) => [ - groupName, - { - charts: [], - totalCharts: 0, - hasData: false, - summary: null, - }, - ]), - ); -} - -function buildChartIndex(groupMap) { - const chartIndex = new Map(); - for (const [groupName, groupData] of Object.entries(groupMap)) { - for (const chart of groupData.charts) { - chartIndex.set(`${groupName}\u0000${chart.name}`, chart); - } - } - return chartIndex; -} - -function appendChartRows(groups, chartRows, totalCommits) { - for (const row of chartRows) { - const group = groups[row.group_name]; - if (!group) continue; - - group.charts.push({ - name: row.chart_name, - unit: row.unit, - series: row.series_names || [], - sortPosition: row.sort_position, - totalPoints: totalCommits, - latestValues: row.latest_values || {}, - }); - } -} - -function applyRandomAccessSummary(groups, rows) { - if (!rows.length) return; - - groups["Random Access"].summary = { - type: "randomAccess", - title: "Random Access Performance", - rankings: rows.map((row) => ({ - name: row.name, - time: row.time, - ratio: row.ratio, - })), - explanation: "Random access time | Ratio to fastest (lower is better)", - }; -} - -function applyCompressionSummary(groups, row) { - if (!row?.compress_ratio && !row?.decompress_ratio) return; - - groups.Compression.summary = { - type: "compression", - title: "Compression Throughput vs Parquet", - compressRatio: row.compress_ratio, - decompressRatio: row.decompress_ratio, - datasetCount: row.dataset_count, - explanation: "Inverse geomean of Vortex/Parquet ratios (higher is better)", - }; -} - -function applyCompressionSizeSummary(groups, row) { - if (!row?.mean_ratio) return; - - groups["Compression Size"].summary = { - type: "compressionSize", - title: "Compression Size Summary", - minRatio: row.min_ratio, - meanRatio: row.mean_ratio, - maxRatio: row.max_ratio, - datasetCount: row.dataset_count, - explanation: "Geomean of Vortex/Parquet size ratios (lower is better)", - }; -} - -function applyQuerySummary(groups, rows) { - for (const row of rows) { - if (QUERY_GROUP_EXCLUSIONS.has(row.group_name)) continue; - - const group = groups[row.group_name]; - if (!group) continue; - - if (!group.summary) { - group.summary = { - type: "queryBenchmark", - title: "Performance Summary", - rankings: [], - explanation: "Geomean of query time ratio to fastest (lower is better)", - }; - } - - group.summary.rankings.push({ - name: row.name, - score: row.score, - totalRuntime: row.total_runtime, - }); - } -} - -function finalizeGroups(groups) { - for (const group of Object.values(groups)) { - group.totalCharts = group.charts.length; - group.hasData = group.charts.length > 0; - } -} - -async function fetchCoreMetadata(connection) { - const commits = await queryRows(connection, COMMITS_QUERY); - const chartRows = await queryRows(connection, CHART_ROWS_QUERY); - - return { commits, chartRows }; -} - -async function fetchSummaryMetadata(connection) { - const randomAccessRows = await queryRows(connection, RANDOM_ACCESS_SUMMARY_QUERY); - const compressionRows = await queryRows(connection, COMPRESSION_SUMMARY_QUERY); - const compressionSizeRows = await queryRows( - connection, - COMPRESSION_SIZE_SUMMARY_QUERY, - ); - - return { - randomAccessRows, - compressionRows, - compressionSizeRows, - }; -} - -export async function buildMetadata(instance, lastUpdated) { - const [ - { commits, chartRows }, - { randomAccessRows, compressionRows, compressionSizeRows }, - querySummaryRows, - ] = await Promise.all([ - withConnection(instance, fetchCoreMetadata), - withConnection(instance, fetchSummaryMetadata), - withConnection(instance, (connection) => queryRows(connection, QUERY_SUMMARY_QUERY)), - ]); - - const groups = buildEmptyGroups(); - appendChartRows(groups, chartRows, commits.length); - applyRandomAccessSummary(groups, randomAccessRows); - applyCompressionSummary(groups, compressionRows[0]); - applyCompressionSizeSummary(groups, compressionSizeRows[0]); - applyQuerySummary(groups, querySummaryRows); - finalizeGroups(groups); - - return { - commits, - metadata: { - groups, - totalCommits: commits.length, - commits: commits.map(({ id, message, timestamp, author }) => ({ - id, - message: firstLine(message), - timestamp, - author, - })), - lastUpdated, - }, - chartIndex: buildChartIndex(groups), - }; -} - -export async function collectDiagnostics(connection) { - const [counts] = await queryRows(connection, DIAGNOSTICS_COUNTS_QUERY); - const uncategorized = await queryRows(connection, UNCATEGORIZED_QUERY); - const groupCounts = await queryRows(connection, GROUP_COUNTS_QUERY); - - return { - benchmarkCount: counts?.benchmark_count ?? 0, - missingCommits: counts?.missing_commits ?? 0, - uncategorized: uncategorized.map((row) => row.prefix), - groupCounts, - }; -} diff --git a/benchmarks-website/store/sql.js b/benchmarks-website/store/sql.js deleted file mode 100644 index 907a5831267..00000000000 --- a/benchmarks-website/store/sql.js +++ /dev/null @@ -1,479 +0,0 @@ -import { ENGINE_RENAMES, QUERY_SUITES } from "../src/config.js"; -import { GROUPS } from "./constants.js"; - -function sqlStringLiteral(value) { - return `'${String(value).replace(/'/g, "''")}'`; -} - -function sqlValue(value) { - if (value === null || value === undefined) return "NULL"; - if (typeof value === "boolean") return value ? "TRUE" : "FALSE"; - if (typeof value === "number") return Number.isFinite(value) ? String(value) : "NULL"; - return sqlStringLiteral(value); -} - -function sqlValues(rows) { - return rows - .map((row) => `(${row.map((value) => sqlValue(value)).join(", ")})`) - .join(",\n"); -} - -function joinStatements(statements) { - return statements - .map((statement) => statement.trim().replace(/;?$/, ";")) - .join("\n\n"); -} - -function randomAccessCondition(expression = "name_lower") { - return `(starts_with(${expression}, 'random-access/') or starts_with(${expression}, 'random access/'))`; -} - -function renameSeriesInSql(sourceExpression) { - return `coalesce( - (select dst from engine_renames er where er.src = lower(${sourceExpression}) limit 1), - ${sourceExpression} - )`; -} - -function buildQuerySuitesStatement() { - const rows = QUERY_SUITES.map((suite) => [ - suite.prefix, - suite.displayName || null, - suite.queryPrefix || null, - suite.datasetKey || null, - Boolean(suite.fanOut), - Boolean(suite.skip), - ]); - - return ` -create or replace table query_suites as -select * from ( - values - ${sqlValues(rows)} -) as suites(prefix, display_name, query_prefix, dataset_key, fan_out, skip) -`; -} - -function buildValidGroupsStatement() { - const rows = GROUPS.map((groupName) => [groupName]); - - return ` -create or replace table valid_groups as -select * from ( - values - ${sqlValues(rows)} -) as groups(group_name) -`; -} - -function buildEngineRenamesStatement() { - const rows = Object.entries(ENGINE_RENAMES); - - return ` -create or replace table engine_renames as -select * from ( - values - ${sqlValues(rows)} -) as renames(src, dst) -`; -} - -function buildRawCommitsViewStatement(commitsPath) { - return ` -create or replace view raw_commits as -select * -from read_json( - ${sqlStringLiteral(commitsPath)}, - format = 'newline_delimited', - compression = 'auto_detect', - columns = { - id: 'VARCHAR', - message: 'VARCHAR', - timestamp: 'VARCHAR', - author: 'JSON', - url: 'VARCHAR' - } -) -`; -} - -function buildCommitDimStatement() { - return ` -create or replace table commit_dim as -select - row_number() over (order by commit_ts nulls last, id) - 1 as commit_idx, - id, - message, - timestamp as timestamp_text, - commit_ts, - author, - url -from ( - select - id, - message, - timestamp, - try_cast(timestamp as timestamptz) as commit_ts, - coalesce(json_extract_string(author, '$.name'), 'Unknown') as author, - url - from raw_commits -) -`; -} - -function buildRawBenchmarksViewStatement(dataPath) { - return ` -create or replace view raw_benchmarks as -select - row_number() over () as benchmark_row, - * -from read_json( - ${sqlStringLiteral(dataPath)}, - format = 'newline_delimited', - compression = 'auto_detect', - columns = { - name: 'VARCHAR', - unit: 'VARCHAR', - value: 'DOUBLE', - storage: 'VARCHAR', - dataset: 'JSON', - commit: 'JSON', - commit_id: 'VARCHAR' - } -) -`; -} - -function buildBenchmarksBaseStatement() { - return ` -create or replace table benchmarks_base as -select - benchmark_row, - name, - lower(name) as name_lower, - split_part(name, '/', 1) as part1, - lower(split_part(name, '/', 1)) as part1_lower, - coalesce(nullif(split_part(name, '/', 2), ''), 'default') as part2, - coalesce(nullif(split_part(name, '/', 3), ''), 'default') as part3, - coalesce(nullif(split_part(name, '/', 4), ''), 'default') as part4, - array_length(string_split(name, '/')) as part_count, - unit, - value as raw_value, - storage, - dataset as dataset_json, - commit as commit_json, - commit_id -from raw_benchmarks -`; -} - -function buildMatchedSuitesStatement() { - return ` -create or replace table matched_suites as -select - b.benchmark_row, - s.prefix, - s.display_name, - s.query_prefix, - s.dataset_key, - s.fan_out -from benchmarks_base b -join query_suites s - on (b.name_lower like s.prefix || '_q%' or b.name_lower like s.prefix || '/%') -where not s.skip -qualify row_number() over (partition by b.benchmark_row order by length(s.prefix) desc) = 1 -`; -} - -function buildGroupNameExpressionSql() { - return ` -case - when ${randomAccessCondition()} then 'Random Access' - when starts_with(name_lower, 'vortex size/') - or starts_with(name_lower, 'vortex-file-compressed size/') - or starts_with(name_lower, 'parquet size/') - or starts_with(name_lower, 'lance size/') - or contains(name_lower, ':raw size/') - or contains(name_lower, ':parquet-zstd size/') - or contains(name_lower, ':lance size/') - then 'Compression Size' - when starts_with(name_lower, 'compress time/') - or starts_with(name_lower, 'decompress time/') - or starts_with(name_lower, 'parquet_rs-zstd compress') - or starts_with(name_lower, 'parquet_rs-zstd decompress') - or starts_with(name_lower, 'lance compress') - or starts_with(name_lower, 'lance decompress') - or starts_with(name_lower, 'vortex:lance ratio') - or starts_with(name_lower, 'vortex:parquet-zstd ratio') - or starts_with(name_lower, 'vortex:raw ratio') - then 'Compression' - when suite_prefix is not null and not suite_fan_out then suite_display_name - when suite_prefix is not null and suite_fan_out then - suite_display_name - || ' (' - || case when upper(coalesce(storage, '')) = 'S3' then 'S3' else 'NVMe' end - || ') (SF=' - || cast(cast(round(coalesce(try_cast(suite_scale_factor as double), 1.0)) as bigint) as varchar) - || ')' - else null -end -`; -} - -function buildRawChartNameExpressionSql() { - return ` -case - when ${randomAccessCondition()} and part_count = 4 - then upper(replace(replace(part2 || '/' || part3, '_', ' '), '-', ' ')) - when ${randomAccessCondition()} and part_count = 2 - then 'RANDOM ACCESS' - when suite_prefix is not null - and regexp_extract(part1_lower, '^' || suite_prefix || '[_ ]?q([0-9]+)', 1) <> '' - then suite_query_prefix || ' Q' || cast(cast(regexp_extract(part1_lower, '^' || suite_prefix || '[_ ]?q([0-9]+)', 1) as integer) as varchar) - else upper(replace(replace(part1, '_', ' '), '-', ' ')) -end -`; -} - -function buildRawSeriesNameExpressionSql() { - return ` -case - when ${randomAccessCondition()} and part_count = 4 then part4 - else part2 -end -`; -} - -function buildRawUnitExpressionSql() { - return ` -case - when unit is not null then unit - when contains(name_lower, ' size/') then 'bytes' - when contains(name_lower, ' ratio ') then 'ratio' - else 'ns' -end -`; -} - -function buildSortPositionExpressionSql() { - return ` -coalesce( - try_cast(nullif(regexp_extract(part1_lower, 'q([0-9]+)$', 1), '') as integer), - 0 -) -`; -} - -function buildChartNameExpressionSql() { - return ` -case - when group_name = 'Compression Size' and raw_chart_name = 'VORTEX FILE COMPRESSED SIZE' then 'VORTEX SIZE' - else raw_chart_name -end -`; -} - -function buildDisplayUnitExpressionSql() { - return ` -case - when raw_unit = 'ns' then 'ms/iter' - when raw_unit = 'bytes' then 'MiB' - else raw_unit -end -`; -} - -function buildDisplayValueExpressionSql() { - return ` -case - when raw_unit = 'ns' then raw_value / 1000000.0 - when raw_unit = 'bytes' then raw_value / 1048576.0 - else raw_value -end -`; -} - -function buildClassifiedBenchmarksStatement() { - return ` -create or replace table classified_benchmarks as -with base as ( - select - b.*, - ms.prefix as suite_prefix, - ms.display_name as suite_display_name, - ms.query_prefix as suite_query_prefix, - ms.dataset_key as suite_dataset_key, - coalesce(ms.fan_out, false) as suite_fan_out, - coalesce(json_extract_string(b.commit_json, '$.id'), b.commit_id) as resolved_commit_id, - case - when ms.dataset_key = 'tpch' then json_extract_string(b.dataset_json, '$.tpch.scale_factor') - when ms.dataset_key = 'tpcds' then json_extract_string(b.dataset_json, '$.tpcds.scale_factor') - else null - end as suite_scale_factor - from benchmarks_base b - left join matched_suites ms using (benchmark_row) -), -named as ( - select - *, - ${buildGroupNameExpressionSql()} as group_name, - ${buildRawChartNameExpressionSql()} as raw_chart_name, - ${buildRawSeriesNameExpressionSql()} as raw_series_name, - ${buildRawUnitExpressionSql()} as raw_unit, - ${buildSortPositionExpressionSql()} as sort_position - from base -), -renamed as ( - select - n.*, - ${renameSeriesInSql("n.raw_series_name")} as series_name - from named n -) -select - benchmark_row, - name, - resolved_commit_id, - group_name, - ${buildChartNameExpressionSql()} as chart_name, - series_name, - sort_position, - ${buildDisplayUnitExpressionSql()} as unit, - ${buildDisplayValueExpressionSql()} as value -from renamed -`; -} - -function buildBenchmarkPointsStatement() { - return ` -create or replace table benchmark_points as -select - cb.group_name, - cb.chart_name, - cb.series_name, - cb.sort_position, - cb.unit, - cb.value, - cd.commit_idx -from classified_benchmarks cb -join valid_groups vg on vg.group_name = cb.group_name -join commit_dim cd on cd.id = cb.resolved_commit_id -where cb.group_name is not null - and cb.chart_name not like '%PARQUET-UNC%' - and cb.name not like '% throughput%' - and cb.value is not null -`; -} - -function buildActiveCommitsStatement() { - return ` -create or replace table active_commits as -with first_active as ( - select coalesce(min(commit_idx), 0) as min_commit_idx - from benchmark_points -) -select - cd.commit_idx as original_commit_idx, - cd.commit_idx - fa.min_commit_idx as commit_idx, - cd.id, - cd.message, - cd.timestamp_text as timestamp, - cd.author, - cd.url -from commit_dim cd -cross join first_active fa -where cd.commit_idx >= fa.min_commit_idx -order by cd.commit_idx -`; -} - -function buildBenchmarkPointsActiveStatement() { - return ` -create or replace table benchmark_points_active as -select - bp.group_name, - bp.chart_name, - bp.series_name, - bp.sort_position, - bp.unit, - bp.value, - ac.commit_idx -from benchmark_points bp -join active_commits ac - on ac.original_commit_idx = bp.commit_idx -`; -} - -function buildChartDefinitionsStatement() { - return ` -create or replace table chart_defs as -select - group_name, - chart_name, - min(sort_position) as sort_position, - min(unit) as unit -from benchmark_points_active -group by 1, 2 -`; -} - -function buildChartLatestIndexStatement() { - return ` -create or replace table chart_latest_idx as -select - group_name, - chart_name, - max(commit_idx) as latest_commit_idx -from benchmark_points_active -group by 1, 2 -`; -} - -function buildChartLatestValuesStatement() { - return ` -create or replace table chart_latest_values as -select - bpa.group_name, - bpa.chart_name, - bpa.series_name, - bpa.value -from benchmark_points_active bpa -join chart_latest_idx cli - on cli.group_name = bpa.group_name - and cli.chart_name = bpa.chart_name - and cli.latest_commit_idx = bpa.commit_idx -`; -} - -function buildChartSeriesLatestValuesStatement() { - return ` -create or replace table chart_series_latest_values as -select - group_name, - chart_name, - series_name, - arg_max(value, commit_idx) as latest_value -from benchmark_points_active -group by 1, 2, 3 -`; -} - -export function buildBootstrapSql(dataPath, commitsPath) { - return joinStatements([ - buildQuerySuitesStatement(), - buildValidGroupsStatement(), - buildEngineRenamesStatement(), - buildRawCommitsViewStatement(commitsPath), - buildCommitDimStatement(), - buildRawBenchmarksViewStatement(dataPath), - buildBenchmarksBaseStatement(), - buildMatchedSuitesStatement(), - buildClassifiedBenchmarksStatement(), - buildBenchmarkPointsStatement(), - buildActiveCommitsStatement(), - buildBenchmarkPointsActiveStatement(), - buildChartDefinitionsStatement(), - buildChartLatestIndexStatement(), - buildChartLatestValuesStatement(), - buildChartSeriesLatestValuesStatement(), - ]); -} diff --git a/benchmarks-website/store/utils.js b/benchmarks-website/store/utils.js deleted file mode 100644 index 462c974fbce..00000000000 --- a/benchmarks-website/store/utils.js +++ /dev/null @@ -1,3 +0,0 @@ -export function firstLine(message) { - return String(message || "").split("\n")[0] || ""; -} From 042e4fb4f068109dc4c2e1852fb0c8e1e27a6d30 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 24 Apr 2026 19:19:27 +0100 Subject: [PATCH 187/250] Improve stats reporting to DF (#7628) ## Summary By comparing the physical plans between Parquet and Vortex, I've found a few things we're doing that are either wrong or suboptimal: 1. Reporting inexact column and file byte sizes if they are missing (which they are), which at the very least leads DF down a suboptimal path which introduces an extra RepartitionExec which isn't necessary. 2. He have a cute trick to report cardinality estimation for constant columns, problem is if we said we had exactly one record even if `IsConstant` was false. There are still a few stats things that could probably be improved here, especially which stats are calculated and stored where and how that's configured, but I think for now this is a very nice improvement. --------- Signed-off-by: Adam Gutglick --- vortex-datafusion/src/convert/stats.rs | 39 +++++++- vortex-datafusion/src/persistent/format.rs | 106 ++++++++++----------- 2 files changed, 84 insertions(+), 61 deletions(-) diff --git a/vortex-datafusion/src/convert/stats.rs b/vortex-datafusion/src/convert/stats.rs index 2a6b1d0996f..3c7b31de95d 100644 --- a/vortex-datafusion/src/convert/stats.rs +++ b/vortex-datafusion/src/convert/stats.rs @@ -9,6 +9,7 @@ use vortex::dtype::Nullability; use vortex::dtype::PType; use vortex::error::VortexExpect; use vortex::error::VortexResult; +use vortex::expr::stats::Precision as VortexPrecision; use vortex::expr::stats::Stat; use vortex::scalar::Scalar; @@ -83,10 +84,40 @@ pub(crate) fn stats_set_to_df( min_value: min.to_df(), max_value: max.to_df(), sum_value: sum.to_df(), - distinct_count: stats_set - .get_as::(Stat::IsConstant, &DType::Bool(Nullability::NonNullable)) - .and_then(|is_constant| is_constant.as_exact().map(|_| Precision::Exact(1))) - .unwrap_or(Precision::Absent), + distinct_count: is_constant_to_distinct_count( + stats_set.get_as::(Stat::IsConstant, &DType::Bool(Nullability::NonNullable)), + ), byte_size: column_size.to_df(), }) } + +pub(crate) fn is_constant_to_distinct_count( + is_constant: Option>, +) -> Precision { + match is_constant.and_then(VortexPrecision::as_exact) { + Some(true) => Precision::Exact(1), + Some(false) | None => Precision::Absent, + } +} + +#[cfg(test)] +mod tests { + use vortex::expr::stats::Precision as VortexPrecision; + + use super::*; + + #[test] + fn is_constant_false_does_not_imply_one_distinct_value() -> VortexResult<()> { + let false_constant = StatsSet::of(Stat::IsConstant, VortexPrecision::exact(false)); + let false_stats = stats_set_to_df(&false_constant, &DType::Bool(Nullability::NonNullable))?; + + assert_eq!(false_stats.distinct_count, Precision::Absent); + + let true_constant = StatsSet::of(Stat::IsConstant, VortexPrecision::exact(true)); + let true_stats = stats_set_to_df(&true_constant, &DType::Bool(Nullability::NonNullable))?; + + assert_eq!(true_stats.distinct_count, Precision::Exact(1)); + + Ok(()) + } +} diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index d5c7fe913f1..cb24be972c2 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -14,6 +14,7 @@ use datafusion_common::ColumnStatistics; use datafusion_common::DataFusionError; use datafusion_common::GetExt; use datafusion_common::Result as DFResult; +use datafusion_common::ScalarValue as DFScalarValue; use datafusion_common::Statistics; use datafusion_common::config::ConfigField; use datafusion_common::config_namespace; @@ -60,6 +61,7 @@ use vortex::file::VORTEX_FILE_EXTENSION; use vortex::io::object_store::ObjectStoreReadAt; use vortex::io::session::RuntimeSessionExt; use vortex::scalar::Scalar; +use vortex::scalar::ScalarValue as VortexScalarValue; use vortex::session::VortexSession; use super::cache::CachedVortexMetadata; @@ -67,6 +69,7 @@ use super::sink::VortexSink; use super::source::VortexSource; use crate::PrecisionExt as _; use crate::convert::TryToDataFusion; +use crate::convert::stats::is_constant_to_distinct_count; const DEFAULT_FOOTER_INITIAL_READ_SIZE_BYTES: usize = MAX_POSTSCRIPT_SIZE as usize + EOF_SIZE; @@ -500,11 +503,13 @@ impl FileFormat for VortexFormat { .vortex_expect("Row count overflow"), ), total_byte_size: Precision::Absent, - column_statistics: vec![ColumnStatistics::default(); struct_dtype.nfields()], + column_statistics: vec![ + ColumnStatistics::default(); + table_schema.fields().len() + ], }); }; - let mut sum_of_column_byte_sizes = stats::Precision::exact(0_usize); let mut column_statistics = Vec::with_capacity(table_schema.fields().len()); for field in table_schema.fields().iter() { @@ -518,55 +523,22 @@ impl FileFormat for VortexFormat { let (stats_set, stats_dtype) = file_stats.get(col_idx); // Update the total size in bytes. - let column_size = stats_set - .get_as::(Stat::UncompressedSizeInBytes, &PType::U64.into()) - .unwrap_or_else(|| stats::Precision::inexact(0_usize)); - sum_of_column_byte_sizes = sum_of_column_byte_sizes - .zip(column_size) - .map(|(acc, size)| acc + size); - - // TODO(connor): There's a lot that can go wrong here, should probably handle this - // more gracefully... - // Find the min statistic. - let min = stats_set.get(Stat::Min).and_then(|pstat_val| { - pstat_val - .map(|stat_val| { - // Because of DataFusion's Schema evolution, it is possible that the - // type of the min/max stat has changed. Thus we construct the stat as - // the file datatype first and only then do we cast accordingly. - Scalar::try_new( - Stat::Min - .dtype(stats_dtype) - .vortex_expect("must have a valid dtype"), - Some(stat_val), - ) - .vortex_expect("`Stat::Min` somehow had an incompatible `DType`") - .cast(&DType::from_arrow(field.as_ref())) - .vortex_expect("Unable to cast to target type that DataFusion wants") - .try_to_df() - .ok() - }) - .transpose() - }); - - // Find the max statistic. - let max = stats_set.get(Stat::Max).and_then(|pstat_val| { - pstat_val - .map(|stat_val| { - Scalar::try_new( - Stat::Max - .dtype(stats_dtype) - .vortex_expect("must have a valid dtype"), - Some(stat_val), - ) - .vortex_expect("`Stat::Max` somehow had an incompatible `DType`") - .cast(&DType::from_arrow(field.as_ref())) - .vortex_expect("Unable to cast to target type that DataFusion wants") - .try_to_df() - .ok() - }) - .transpose() - }); + let column_size = + stats_set.get_as::(Stat::UncompressedSizeInBytes, &PType::U64.into()); + + let target_dtype = DType::from_arrow(field.as_ref()); + let min = scalar_stat_to_df( + Stat::Min, + stats_set.get(Stat::Min), + stats_dtype, + &target_dtype, + ); + let max = scalar_stat_to_df( + Stat::Max, + stats_set.get(Stat::Max), + stats_dtype, + &target_dtype, + ); let null_count = stats_set.get_as::(Stat::NullCount, &PType::U64.into()); @@ -575,16 +547,19 @@ impl FileFormat for VortexFormat { min_value: min.to_df(), max_value: max.to_df(), sum_value: Precision::Absent, - distinct_count: stats_set - .get_as::(Stat::IsConstant, &DType::Bool(Nullability::NonNullable)) - .and_then(|is_constant| is_constant.as_exact().map(|_| Precision::Exact(1))) - .unwrap_or(Precision::Absent), - // TODO(connor): Is this correct? + distinct_count: is_constant_to_distinct_count( + stats_set.get_as::( + Stat::IsConstant, + &DType::Bool(Nullability::NonNullable), + ), + ), byte_size: column_size.to_df(), }) } - let total_byte_size = sum_of_column_byte_sizes.to_df(); + let total_byte_size = column_statistics + .iter() + .fold(Precision::Exact(0), |acc, cs| acc.add(&cs.byte_size)); Ok(Statistics { num_rows: Precision::Exact( @@ -651,6 +626,23 @@ impl FileFormat for VortexFormat { } } +fn scalar_stat_to_df( + stat: Stat, + value: Option>, + stats_dtype: &DType, + target_dtype: &DType, +) -> Option> { + let stat_dtype = stat.dtype(stats_dtype)?; + + value? + .try_map(|stat_value| { + Scalar::try_new(stat_dtype, Some(stat_value))? + .cast(target_dtype)? + .try_to_df() + }) + .ok() +} + #[cfg(test)] mod tests { From 193d655f1c5950731d6dd67b6b9841078f26218a Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:19:40 -0400 Subject: [PATCH 188/250] Validate zoned build (#7627) ## Summary Adds some validation to the `ZonedLayout` building. ## Testing Some error catching tests. Signed-off-by: Connor Tsui --- vortex-layout/src/layouts/zoned/mod.rs | 111 ++++++++++++++++++++- vortex-layout/src/layouts/zoned/reader.rs | 115 ++++++++++++++++++++-- vortex-layout/src/layouts/zoned/writer.rs | 6 ++ 3 files changed, 221 insertions(+), 11 deletions(-) diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 3c524bedead..15a5203c0d5 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -20,6 +20,8 @@ use vortex_array::stats::stats_from_bitset_bytes; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_error::vortex_ensure; +use vortex_error::vortex_ensure_eq; use vortex_error::vortex_panic; use vortex_session::VortexSession; use vortex_session::registry::ReadContext; @@ -118,6 +120,11 @@ impl VTable for Zoned { children: &dyn LayoutChildren, _ctx: &ReadContext, ) -> VortexResult { + vortex_ensure_eq!( + children.nchildren(), + 2, + "ZonedLayout expects exactly 2 children (data, zones)" + ); Ok(ZonedLayout { dtype: dtype.clone(), children: children.to_arc(), @@ -194,8 +201,18 @@ impl DeserializeMetadata for ZonedMetadata { type Output = Self; fn deserialize(metadata: &[u8]) -> VortexResult { + vortex_ensure!( + metadata.len() >= 4, + "Zoned metadata must contain at least 4 bytes for zone length, got {}", + metadata.len() + ); + + // Backward compat: older files may encode `zone_len == 0`. Preserve the raw metadata on + // read and let the reader disable zoned pruning for those layouts instead of rejecting + // deserialization outright. let zone_len = u32::try_from_le_bytes(&metadata[0..4])?; let present_stats: Arc<[Stat]> = stats_from_bitset_bytes(&metadata[4..]).into(); + Ok(Self { zone_len, present_stats, @@ -216,19 +233,25 @@ impl SerializeMetadata for ZonedMetadata { #[cfg(test)] mod tests { + use std::panic; + use rstest::rstest; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_session::registry::ReadContext; use super::*; + use crate::IntoLayout; + use crate::children::OwnedLayoutChildren; + use crate::layouts::flat::FlatLayout; + use crate::segments::SegmentId; #[rstest] #[case(ZonedMetadata { zone_len: u32::MAX, present_stats: Arc::new([]), })] - #[case(ZonedMetadata { - zone_len: 0, - present_stats: Arc::new([Stat::IsConstant]), - })] #[case::all_sorted(ZonedMetadata { zone_len: 314, present_stats: Arc::new([Stat::IsConstant, Stat::IsSorted, Stat::IsStrictSorted, Stat::Max, Stat::Min, Stat::Sum, Stat::NullCount, Stat::UncompressedSizeInBytes, Stat::NaNCount]), @@ -258,4 +281,84 @@ mod tests { ); assert_ne!(deserialized.present_stats, metadata.present_stats); } + + #[rstest] + #[case(vec![])] + #[case(vec![0])] + #[case(vec![0, 0])] + #[case(vec![0, 0, 0])] + fn test_deserialize_short_metadata_errors(#[case] metadata: Vec) { + assert!(ZonedMetadata::deserialize(&metadata).is_err()); + } + + #[test] + fn test_deserialize_short_metadata_returns_error_not_panic() { + let result = panic::catch_unwind(|| ZonedMetadata::deserialize(&[])); + assert!( + result.is_ok(), + "deserialize should return an error, not panic" + ); + assert!(result.unwrap().is_err()); + } + + #[test] + fn test_deserialize_zero_zone_len_is_allowed_for_backcompat() { + let metadata = 0u32.to_le_bytes(); + let deserialized = ZonedMetadata::deserialize(&metadata).unwrap(); + assert_eq!(deserialized.zone_len, 0); + assert!(deserialized.present_stats.is_empty()); + } + + #[test] + fn test_build_allows_zero_zone_len_for_backcompat() -> VortexResult<()> { + let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); + let read_ctx = ReadContext::new([]); + let children = OwnedLayoutChildren::layout_children(vec![ + FlatLayout::new(0, dtype.clone(), SegmentId::from(0), read_ctx.clone()).into_layout(), + FlatLayout::new( + 0, + ZoneMap::dtype_for_stats_table(&dtype, &[]), + SegmentId::from(1), + read_ctx, + ) + .into_layout(), + ]); + + let layout = ::build( + &ZonedLayoutEncoding, + &dtype, + 0, + &ZonedMetadata { + zone_len: 0, + present_stats: Arc::new([]), + }, + vec![], + children.as_ref(), + &ReadContext::new([]), + )?; + + assert_eq!(layout.zone_len, 0); + Ok(()) + } + + #[test] + fn test_build_rejects_invalid_child_count() { + let metadata = ZonedMetadata { + zone_len: 3, + present_stats: Arc::new([]), + }; + let children = OwnedLayoutChildren::layout_children(vec![]); + + let result = ::build( + &ZonedLayoutEncoding, + &DType::Primitive(PType::I32, Nullability::NonNullable), + 0, + &metadata, + vec![], + children.as_ref(), + &ReadContext::new([]), + ); + + assert!(result.is_err()); + } } diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 10f2c1ff598..75e36c9648b 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -200,8 +200,10 @@ impl ZonedReader { /// Get the range of zone IDs containing a row range. pub(crate) fn zone_range(&self, row_range: &Range) -> Range { - // Zone length is guaranteed to be > 0 by ZonedLayout::new validation + // Caller must ensure zone_len > 0. Legacy files may deserialize with zone_len == 0, but + // pruning_evaluation disables zoned pruning for those layouts before calling this helper. debug_assert!(self.layout.zone_len > 0, "zone_len must be > 0"); + let zone_len_u64 = self.layout.zone_len as u64; let zone_start = row_range.start / zone_len_u64; let zone_end = row_range.end.div_ceil(zone_len_u64); @@ -250,6 +252,13 @@ impl LayoutReader for ZonedReader { .data_child()? .pruning_evaluation(row_range, expr, mask.clone())?; + if self.layout.zone_len == 0 { + tracing::debug!( + "Stats pruning evaluation: skipping zoned pruning for legacy zero-length zones" + ); + return Ok(data_eval); + } + let Some(pruning_mask_future) = self.pruning_mask_future(expr.clone()) else { tracing::debug!("Stats pruning evaluation: not prune-able {expr}"); return Ok(data_eval); @@ -412,22 +421,44 @@ mod test { use vortex_array::expr::gt; use vortex_array::expr::lit; use vortex_array::expr::root; + use vortex_array::scalar_fn::session::ScalarFnSession; + use vortex_array::session::ArraySession; use vortex_buffer::buffer; + use vortex_error::VortexResult; + use vortex_io::runtime::Handle; use vortex_io::runtime::single::block_on; + use vortex_io::session::RuntimeSession; use vortex_io::session::RuntimeSessionExt; use vortex_mask::Mask; + use vortex_session::VortexSession; + use vortex_session::registry::ReadContext; + use crate::IntoLayout; use crate::LayoutRef; use crate::LayoutStrategy; + use crate::VTable; + use crate::children::OwnedLayoutChildren; use crate::layouts::chunked::writer::ChunkedLayoutStrategy; use crate::layouts::flat::writer::FlatLayoutStrategy; + use crate::layouts::zoned::Zoned; + use crate::layouts::zoned::ZonedLayoutEncoding; + use crate::layouts::zoned::ZonedMetadata; use crate::layouts::zoned::writer::ZonedLayoutOptions; use crate::layouts::zoned::writer::ZonedStrategy; use crate::segments::SegmentSource; use crate::segments::TestSegments; use crate::sequence::SequenceId; use crate::sequence::SequentialArrayStreamExt; - use crate::test::SESSION; + use crate::session::LayoutSession; + + fn session_with_handle(handle: Handle) -> VortexSession { + VortexSession::empty() + .with::() + .with::() + .with::() + .with::() + .with_handle(handle) + } #[fixture] /// Create a stats layout with three chunks of primitive arrays. @@ -453,7 +484,7 @@ mod test { .sequenced(ptr); let segments2 = Arc::::clone(&segments); let layout = block_on(|handle| async move { - let session = SESSION.clone().with_handle(handle); + let session = session_with_handle(handle); strategy .write_stream(ctx, segments2, array_stream, eof, &session) .await @@ -466,9 +497,10 @@ mod test { fn test_stats_evaluator( #[from(stats_layout)] (segments, layout): (Arc, LayoutRef), ) { - block_on(|_| async { + block_on(|handle| async { + let session = session_with_handle(handle); let result = layout - .new_reader("".into(), segments, &SESSION) + .new_reader("".into(), segments, &session) .unwrap() .projection_evaluation( &(0..layout.row_count()), @@ -488,9 +520,10 @@ mod test { fn test_stats_pruning_mask( #[from(stats_layout)] (segments, layout): (Arc, LayoutRef), ) { - block_on(|_| async { + block_on(|handle| async { let row_count = layout.row_count(); - let reader = layout.new_reader("".into(), segments, &SESSION).unwrap(); + let session = session_with_handle(handle); + let reader = layout.new_reader("".into(), segments, &session).unwrap(); // Choose a prune-able expression let expr = gt(root(), lit(7)); @@ -511,4 +544,72 @@ mod test { ); }) } + + #[rstest] + fn test_legacy_zero_zone_len_skips_zoned_pruning( + #[from(stats_layout)] (segments, layout): (Arc, LayoutRef), + ) -> VortexResult<()> { + let zoned_layout = layout.as_::(); + let children = + OwnedLayoutChildren::layout_children(vec![layout.child(0)?, layout.child(1)?]); + let legacy_layout = ::build( + &ZonedLayoutEncoding, + layout.dtype(), + layout.row_count(), + &ZonedMetadata { + zone_len: 0, + present_stats: Arc::clone(zoned_layout.present_stats()), + }, + vec![], + children.as_ref(), + &ReadContext::new([]), + )? + .into_layout(); + + block_on(|handle| async { + let row_count = legacy_layout.row_count(); + let session = session_with_handle(handle); + let reader = legacy_layout.new_reader("".into(), segments, &session)?; + + let result = reader + .pruning_evaluation( + &(0..row_count), + >(root(), lit(7)), + Mask::new_true(row_count.try_into().unwrap()), + )? + .await?; + + assert!(result.all_true()); + Ok(()) + }) + } + + #[test] + fn test_writer_rejects_zero_block_size() { + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let strategy = ZonedStrategy::new( + ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()), + FlatLayoutStrategy::default(), + ZonedLayoutOptions { + block_size: 0, + ..Default::default() + }, + ); + let array_stream = ChunkedArray::from_iter([buffer![1, 2, 3].into_array()]) + .into_array() + .to_array_stream() + .sequenced(ptr); + let segments2 = Arc::::clone(&segments); + + let result = block_on(|handle| async move { + let session = session_with_handle(handle); + strategy + .write_stream(ctx, segments2, array_stream, eof, &session) + .await + }); + + assert!(result.is_err()); + } } diff --git a/vortex-layout/src/layouts/zoned/writer.rs b/vortex-layout/src/layouts/zoned/writer.rs index 8ec37f8bae4..c28b46558a9 100644 --- a/vortex-layout/src/layouts/zoned/writer.rs +++ b/vortex-layout/src/layouts/zoned/writer.rs @@ -12,6 +12,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::expr::stats::Stat; use vortex_array::stats::PRUNING_STATS; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; use vortex_io::session::RuntimeSessionExt; use vortex_session::VortexSession; use vortex_utils::parallelism::get_available_parallelism; @@ -80,6 +81,11 @@ impl LayoutStrategy for ZonedStrategy { mut eof: SequencePointer, session: &VortexSession, ) -> VortexResult { + vortex_ensure!( + self.options.block_size > 0, + "ZonedStrategy requires block_size > 0 when writing" + ); + let stats = Arc::clone(&self.options.stats); let session = session.clone(); let compute_session = session.clone(); From 7c2df2c98bd7dd7a37e8f923a78efc55974b3fcb Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Fri, 24 Apr 2026 21:12:41 +0100 Subject: [PATCH 189/250] feat(cuda): make GPU dispatch mode configurable (#7621) Allow for explicitly choosing dyn dispatch or standalone kernels. We primarily do this to be able to test and bench the standalone CUDA kernels. In a follow up, I'll change the logic to prefer standalone kernels, if available for an array tree, over dyn dispatch. Signed-off-by: Alexander Droste --- vortex-cuda/benches/alp_cuda.rs | 2 ++ vortex-cuda/benches/bitpacked_cuda.rs | 3 ++ vortex-cuda/benches/date_time_parts_cuda.rs | 2 ++ vortex-cuda/benches/dict_cuda.rs | 2 ++ vortex-cuda/benches/for_cuda.rs | 3 ++ vortex-cuda/src/executor.rs | 35 +++++++++++++++++++++ vortex-cuda/src/hybrid_dispatch/mod.rs | 21 +++++++++++++ vortex-cuda/src/lib.rs | 1 + 8 files changed, 69 insertions(+) diff --git a/vortex-cuda/benches/alp_cuda.rs b/vortex-cuda/benches/alp_cuda.rs index 11e9b894dd6..0f006018a58 100644 --- a/vortex-cuda/benches/alp_cuda.rs +++ b/vortex-cuda/benches/alp_cuda.rs @@ -31,6 +31,7 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::alp_encode; use vortex::error::VortexExpect; use vortex::session::VortexSession; +use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -109,6 +110,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index 449061e6978..7568db82ec5 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -31,6 +31,7 @@ use vortex::encodings::fastlanes::BitPackedData; use vortex::encodings::fastlanes::unpack_iter::BitPacked; use vortex::error::VortexExpect; use vortex::session::VortexSession; +use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -127,6 +128,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { @@ -172,6 +174,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index da696681418..59a8c8aa9df 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -30,6 +30,7 @@ use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; use vortex::session::VortexSession; +use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -68,6 +69,7 @@ fn benchmark_datetimeparts(c: &mut Criterion) { let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { diff --git a/vortex-cuda/benches/dict_cuda.rs b/vortex-cuda/benches/dict_cuda.rs index 1e1a51d3a8a..4f1b72a49e3 100644 --- a/vortex-cuda/benches/dict_cuda.rs +++ b/vortex-cuda/benches/dict_cuda.rs @@ -26,6 +26,7 @@ use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::error::VortexExpect; use vortex::session::VortexSession; +use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -96,6 +97,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index 29402737a52..c1c098f1f4b 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -33,6 +33,7 @@ use vortex::encodings::fastlanes::FoRArray; use vortex::error::VortexExpect; use vortex::scalar::Scalar; use vortex::session::VortexSession; +use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -91,6 +92,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { @@ -129,6 +131,7 @@ where let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 8234bfb8efe..d490d3f03fe 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -57,6 +57,25 @@ impl CudaKernelEvents { } } +/// Controls which GPU dispatch strategy is used when executing arrays. +/// +/// By default, `execute_cuda` tries fused dynamic dispatch first, +/// then falls back to standalone per-encoding kernels. Benchmarks and tests +/// can force a specific strategy to get stable, isolated measurements. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum CudaDispatchMode { + /// Automatically choose the best strategy (current default behavior). + /// Tries fused → partially-fused → standalone → CPU fallback. + #[default] + Auto, + /// Only use standalone per-encoding `CudaExecute` kernels. + /// Skips the fused/partially-fused dynamic dispatch planner entirely. + StandaloneOnly, + /// Only use fused or partially-fused dynamic dispatch. + /// Returns an error if the array is not dyn-dispatch-compatible. + DynDispatchOnly, +} + /// CUDA execution context. /// /// Provides access to the CUDA context and stream for kernel execution. @@ -66,6 +85,7 @@ pub struct CudaExecutionCtx { ctx: ExecutionCtx, cuda_session: CudaSession, strategy: Arc, + dispatch_mode: CudaDispatchMode, } impl CudaExecutionCtx { @@ -77,6 +97,7 @@ impl CudaExecutionCtx { ctx, cuda_session, strategy: Arc::new(DefaultLaunchStrategy), + dispatch_mode: CudaDispatchMode::Auto, } } @@ -94,6 +115,15 @@ impl CudaExecutionCtx { self } + /// Set the dispatch mode for the execution context. + /// + /// This controls whether `execute_cuda` uses fused dynamic dispatch, + /// standalone per-encoding kernels, or the automatic (default) strategy. + pub fn with_dispatch_mode(mut self, dispatch_mode: CudaDispatchMode) -> Self { + self.dispatch_mode = dispatch_mode; + self + } + /// Perform an external kernel launch, with events created and logged via the configured /// [`LaunchStrategy`]. /// @@ -276,6 +306,11 @@ impl CudaExecutionCtx { self.ctx.session() } + /// Returns the current dispatch mode. + pub fn dispatch_mode(&self) -> CudaDispatchMode { + self.dispatch_mode + } + /// Returns a reference to the CUDA session. pub(crate) fn cuda_session(&self) -> &CudaSession { &self.cuda_session diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 36636076da7..59ec18aa938 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -70,8 +70,21 @@ pub async fn try_gpu_dispatch( array: &ArrayRef, ctx: &mut CudaExecutionCtx, ) -> VortexResult { + use crate::executor::CudaDispatchMode; + trace!(encoding = %array.encoding_id(), dtype = ?array.dtype(), len = array.len(), "try GPU dispatch"); + // Short-circuit: standalone-only skips the plan builder entirely. + if ctx.dispatch_mode() == CudaDispatchMode::StandaloneOnly { + trace!(encoding = %array.encoding_id(), "standalone-only dispatch"); + return ctx + .cuda_session() + .kernel(&array.encoding_id()) + .ok_or_else(|| vortex_err!("No CUDA kernel for encoding {:?}", array.encoding_id()))? + .execute(array.clone(), ctx) + .await; + } + match DispatchPlan::new(array)? { DispatchPlan::Fused(plan) => { let materialized = plan.materialize(ctx).await?; @@ -99,6 +112,14 @@ pub async fn try_gpu_dispatch( materialized.execute(array.len(), ctx) } DispatchPlan::Unfused => { + // In dyn-dispatch-only mode, don't fall back to standalone kernels. + if ctx.dispatch_mode() == CudaDispatchMode::DynDispatchOnly { + return Err(vortex_err!( + "Array with encoding {:?} is not dyn-dispatch-compatible", + array.encoding_id() + )); + } + // Unfused kernel dispatch fallback. ctx.cuda_session() .kernel(&array.encoding_id()) diff --git a/vortex-cuda/src/lib.rs b/vortex-cuda/src/lib.rs index 7d9caf6aa99..bbebef9430d 100644 --- a/vortex-cuda/src/lib.rs +++ b/vortex-cuda/src/lib.rs @@ -27,6 +27,7 @@ pub use canonical::CanonicalCudaExt; pub use device_buffer::CudaBufferExt; pub use device_buffer::CudaDeviceBuffer; pub use device_read_at::CopyDeviceReadAt; +pub use executor::CudaDispatchMode; pub use executor::CudaExecutionCtx; pub use executor::CudaKernelEvents; use kernel::ALPExecutor; From 1838a7a204b2cb385923f5f4a966879ad8ad45d9 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:13:38 -0400 Subject: [PATCH 190/250] `zoned` cleanup (#7634) ## Summary Cleans up `ZonedLayout`. This is just a cosmetic change as I was trying to understand it. Also adds some tests that I thought would be helpful. ## Testing N/A Signed-off-by: Connor Tsui --- docs/concepts/scanning.md | 11 +- vortex-layout/src/layouts/file_stats.rs | 2 +- vortex-layout/src/layouts/zoned/builder.rs | 246 +++++++++++++++++-- vortex-layout/src/layouts/zoned/mod.rs | 44 +++- vortex-layout/src/layouts/zoned/pruning.rs | 208 ++++++++++++++++ vortex-layout/src/layouts/zoned/reader.rs | 211 +--------------- vortex-layout/src/layouts/zoned/schema.rs | 94 ++++++++ vortex-layout/src/layouts/zoned/writer.rs | 9 +- vortex-layout/src/layouts/zoned/zone_map.rs | 253 +------------------- 9 files changed, 598 insertions(+), 480 deletions(-) create mode 100644 vortex-layout/src/layouts/zoned/pruning.rs create mode 100644 vortex-layout/src/layouts/zoned/schema.rs diff --git a/docs/concepts/scanning.md b/docs/concepts/scanning.md index 9e21c589f25..393b7d2d83c 100644 --- a/docs/concepts/scanning.md +++ b/docs/concepts/scanning.md @@ -68,10 +68,12 @@ independently. The scan tracks the selectivity of each conjunct using a probabil and dynamically reorders them so that the most selective predicates are evaluated first. This means that as a scan progresses, it learns the most efficient evaluation order for the filter. -Filters are evaluated in two stages. First, pruning evaluation uses statistics stored in -`ZonedLayout` (such as per-zone min/max values) to eliminate entire regions without reading any -data. Second, filter evaluation materializes only the filter-referenced columns and computes a -row mask of matching rows. +Filters are evaluated in two stages. First, pruning evaluation uses statistics stored in a +`ZonedLayout` auxiliary `zones` child to eliminate entire row zones without reading the underlying +data child. These pruning predicates are falsification checks derived from the original filter, for +example by comparing a zone's min/max values against the requested predicate. Second, filter +evaluation materializes only the filter-referenced columns and computes a row mask of matching +rows. ## Projection Pushdown @@ -89,4 +91,3 @@ Query engines integrate with the Scan API by translating their internal plan rep scan requests and consuming the resulting array stream in their preferred format. Integrations exist for DuckDB, DataFusion, Spark, and Trino, with each engine converting its native filter and projection representations into Vortex [expressions](expressions.md). - diff --git a/vortex-layout/src/layouts/file_stats.rs b/vortex-layout/src/layouts/file_stats.rs index a970cfbedb3..f12af8a98fc 100644 --- a/vortex-layout/src/layouts/file_stats.rs +++ b/vortex-layout/src/layouts/file_stats.rs @@ -21,7 +21,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use crate::layouts::zoned::zone_map::StatsAccumulator; +use crate::layouts::zoned::StatsAccumulator; use crate::sequence::SendableSequentialStream; use crate::sequence::SequenceId; use crate::sequence::SequentialStreamAdapter; diff --git a/vortex-layout/src/layouts/zoned/builder.rs b/vortex-layout/src/layouts/zoned/builder.rs index 6a7c7eff0e6..7f24eaca886 100644 --- a/vortex-layout/src/layouts/zoned/builder.rs +++ b/vortex-layout/src/layouts/zoned/builder.rs @@ -1,32 +1,137 @@ +//! Write-time accumulation and builders for zoned layout stats tables. + // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::marker::PhantomData; +use itertools::Itertools; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::StructArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::BoolBuilder; use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::DType; use vortex_array::dtype::FieldName; use vortex_array::dtype::Nullability; +use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; +use vortex_array::expr::stats::StatsProvider; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarTruncation; use vortex_array::scalar::lower_bound; use vortex_array::scalar::upper_bound; +use vortex_array::validity::Validity; use vortex_buffer::BufferString; use vortex_buffer::ByteBuffer; +use vortex_error::VortexExpect; use vortex_error::VortexResult; -pub const MAX_IS_TRUNCATED: &str = "max_is_truncated"; -pub const MIN_IS_TRUNCATED: &str = "min_is_truncated"; +use crate::layouts::zoned::schema::MAX_IS_TRUNCATED; +use crate::layouts::zoned::schema::MIN_IS_TRUNCATED; +use crate::layouts::zoned::zone_map::ZoneMap; + +/// Accumulates write-time statistics for each logical zone. +pub struct StatsAccumulator { + builders: Vec>, + length: usize, +} + +impl StatsAccumulator { + pub fn new(dtype: &DType, stats: &[Stat], max_variable_length_statistics_size: usize) -> Self { + let builders = stats + .iter() + .filter_map(|&stat| { + stat.dtype(dtype).map(|stat_dtype| { + stats_builder_with_capacity( + stat, + &stat_dtype.as_nullable(), + 1024, + max_variable_length_statistics_size, + ) + }) + }) + .collect::>(); + + Self { + builders, + length: 0, + } + } + + pub fn push_chunk_without_compute(&mut self, array: &ArrayRef) -> VortexResult<()> { + for builder in &mut self.builders { + if let Some(Precision::Exact(value)) = array.statistics().get(builder.stat()) { + builder.append_scalar(value.cast(&value.dtype().as_nullable())?)?; + } else { + builder.append_null(); + } + } + self.length += 1; + Ok(()) + } -pub fn stats_builder_with_capacity( + pub fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { + for builder in &mut self.builders { + if let Some(value) = array.statistics().compute_stat(builder.stat(), ctx)? { + builder.append_scalar(value.cast(&value.dtype().as_nullable())?)?; + } else { + builder.append_null(); + } + } + self.length += 1; + Ok(()) + } + + /// Finishes the accumulator into a [`ZoneMap`]. + /// + /// Returns `None` if none of the requested statistics can be computed, for example they are + /// not applicable to the column's data type. + pub fn as_stats_table(&mut self) -> VortexResult> { + let mut names = Vec::new(); + let mut fields = Vec::new(); + let mut stats = Vec::new(); + + for builder in self + .builders + .iter_mut() + // We sort the stats so the DType is deterministic based on which stats are present. + .sorted_unstable_by_key(|builder| builder.stat()) + { + let values = builder.finish(); + + // We drop any all-null stats columns. + if values.all_invalid()? { + continue; + } + + stats.push(builder.stat()); + names.extend(values.names); + fields.extend(values.arrays); + } + + if names.is_empty() { + return Ok(None); + } + + let array = StructArray::try_new(names.into(), fields, self.length, Validity::NonNullable) + .vortex_expect("Failed to create zone map"); + let stats = stats.into(); + + // SAFETY: `StatsAccumulator` builds the struct fields from `stats_builder_with_capacity` + // using the same field-ordering and truncation-column rules as `stats_table_dtype`. + // The `stats` list is collected in that same sorted order, so the resulting struct array + // matches the expected zoned stats-table dtype by construction. + Ok(Some(unsafe { ZoneMap::new_unchecked(array, stats) })) + } +} + +fn stats_builder_with_capacity( stat: Stat, dtype: &DType, capacity: usize, @@ -64,21 +169,20 @@ pub fn stats_builder_with_capacity( } } -/// Arrays with their associated names, reduced version of a StructArray -pub struct NamedArrays { - pub names: Vec, - pub arrays: Vec, +/// Arrays with their associated names, reduced version of a `StructArray`. +struct NamedArrays { + names: Vec, + arrays: Vec, } impl NamedArrays { - pub fn all_invalid(&self) -> VortexResult { - // by convention we assume that the first array is the one we care about for logical validity + fn all_invalid(&self) -> VortexResult { + // By convention the first array is the logical validity signal for the stat column. self.arrays[0].all_invalid(&mut LEGACY_SESSION.create_execution_ctx()) } } -/// Minimal array builder interface for use by StatsTable for building stats arrays -pub trait StatsArrayBuilder: Send { +trait StatsArrayBuilder: Send { fn stat(&self) -> Stat; fn append_scalar(&mut self, value: Scalar) -> VortexResult<()>; @@ -88,13 +192,13 @@ pub trait StatsArrayBuilder: Send { fn finish(&mut self) -> NamedArrays; } -pub struct StatNameArrayBuilder { +struct StatNameArrayBuilder { stat: Stat, builder: Box, } impl StatNameArrayBuilder { - pub fn new(stat: Stat, builder: Box) -> Self { + fn new(stat: Stat, builder: Box) -> Self { Self { stat, builder } } } @@ -140,7 +244,7 @@ struct TruncatedMaxBinaryStatsBuilder { } impl TruncatedMaxBinaryStatsBuilder { - pub fn new( + fn new( values: Box, is_truncated: BoolBuilder, max_value_length: usize, @@ -162,7 +266,7 @@ struct TruncatedMinBinaryStatsBuilder { } impl TruncatedMinBinaryStatsBuilder { - pub fn new( + fn new( values: Box, is_truncated: BoolBuilder, max_value_length: usize, @@ -194,7 +298,6 @@ impl StatsArrayBuilder for TruncatedMaxBinaryStatsBuilder StatsArrayBuilder for TruncatedMinBinaryStatsBuilder StatsArrayBuilder for TruncatedMinBinaryStatsBuilder(&mut ctx) + .unwrap(); + assert_eq!( + field1_bool.to_bit_buffer(), + BitBuffer::from(vec![false, true]) + ); + let field3_bool = stats_table + .array() + .unmasked_field(3) + .clone() + .execute::(&mut ctx) + .unwrap(); + assert_eq!( + field3_bool.to_bit_buffer(), + BitBuffer::from(vec![true, false]) + ); + } + + #[test] + fn always_adds_is_truncated_column() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let array = buffer![0, 1, 2].into_array(); + let mut acc = StatsAccumulator::new(array.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); + acc.push_chunk(&array, &mut ctx) + .vortex_expect("push_chunk should succeed for test array"); + let stats_table = acc + .as_stats_table() + .unwrap() + .expect("Must have stats table"); + assert_eq!( + stats_table.array().names().as_ref(), + &[ + Stat::Max.name(), + MAX_IS_TRUNCATED, + Stat::Min.name(), + MIN_IS_TRUNCATED, + Stat::Sum.name(), + ] + ); + let field1_bool = stats_table + .array() + .unmasked_field(1) + .clone() + .execute::(&mut ctx) + .unwrap(); + assert_eq!(field1_bool.to_bit_buffer(), BitBuffer::from(vec![false])); + let field3_bool = stats_table + .array() + .unmasked_field(3) + .clone() + .execute::(&mut ctx) + .unwrap(); + assert_eq!(field3_bool.to_bit_buffer(), BitBuffer::from(vec![false])); + } +} diff --git a/vortex-layout/src/layouts/zoned/mod.rs b/vortex-layout/src/layouts/zoned/mod.rs index 15a5203c0d5..18ae57f368f 100644 --- a/vortex-layout/src/layouts/zoned/mod.rs +++ b/vortex-layout/src/layouts/zoned/mod.rs @@ -1,15 +1,28 @@ +//! Zoned layouts wrap a data layout with an auxiliary per-zone statistics layout. +//! +//! The zoned layout tree has exactly two children: +//! - a transparent `data` child containing the underlying column data +//! - an auxiliary `zones` child containing one row of aggregate statistics per zone +//! +//! Metadata stores the logical zone length in rows plus the sorted list of statistics present in +//! the auxiliary table. During scans, pruning first evaluates a falsification predicate against +//! the `zones` child and only forwards surviving rows to the underlying `data` child. + // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors mod builder; +mod pruning; mod reader; +mod schema; pub mod writer; pub mod zone_map; use std::sync::Arc; -pub use builder::MAX_IS_TRUNCATED; -pub use builder::MIN_IS_TRUNCATED; +pub(crate) use builder::StatsAccumulator; +pub use schema::MAX_IS_TRUNCATED; +pub use schema::MIN_IS_TRUNCATED; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; use vortex_array::dtype::DType; @@ -35,7 +48,7 @@ use crate::VTable; use crate::children::LayoutChildren; use crate::children::OwnedLayoutChildren; use crate::layouts::zoned::reader::ZonedReader; -use crate::layouts::zoned::zone_map::ZoneMap; +use crate::layouts::zoned::schema::stats_table_dtype; use crate::segments::SegmentId; use crate::segments::SegmentSource; use crate::vtable; @@ -48,7 +61,8 @@ impl VTable for Zoned { type Metadata = ZonedMetadata; fn id(_encoding: &Self::Encoding) -> LayoutId { - LayoutId::new("vortex.stats") // For legacy reasons, this is called stats + // For legacy reasons the serialized layout encoding ID is still `vortex.stats`. + LayoutId::new("vortex.stats") } fn encoding(_layout: &Self::Layout) -> LayoutEncodingRef { @@ -81,10 +95,9 @@ impl VTable for Zoned { fn child(layout: &Self::Layout, idx: usize) -> VortexResult { match idx { 0 => layout.children.child(0, layout.dtype()), - 1 => layout.children.child( - 1, - &ZoneMap::dtype_for_stats_table(layout.dtype(), &layout.present_stats), - ), + 1 => layout + .children + .child(1, &stats_table_dtype(layout.dtype(), &layout.present_stats)), _ => vortex_bail!("Invalid child index: {}", idx), } } @@ -145,12 +158,15 @@ impl VTable for Zoned { } } +/// Encoding marker for the zoned layout. #[derive(Debug)] pub struct ZonedLayoutEncoding; -/// Annotates a data layout with per-zone aggregate statistics (e.g. min, max, null count). +/// A layout that annotates a data child with one row of aggregate statistics per zone. /// -/// During reads, zone maps allow entire zones to be skipped when a filter predicate cannot match. +/// The first child is the underlying data layout. The second child is an auxiliary stats table +/// whose rows align with logical row zones of length `zone_len`, except for the final partial zone. +/// During reads, pruning uses the stats table to skip zones whose rows cannot satisfy a filter. #[derive(Clone, Debug)] pub struct ZonedLayout { dtype: DType, @@ -169,7 +185,7 @@ impl ZonedLayout { if zone_len == 0 { vortex_panic!("Zone length must be greater than 0"); } - let expected_dtype = ZoneMap::dtype_for_stats_table(data.dtype(), &present_stats); + let expected_dtype = stats_table_dtype(data.dtype(), &present_stats); if zones.dtype() != &expected_dtype { vortex_panic!("Invalid zone map layout: zones dtype does not match expected dtype"); } @@ -191,6 +207,10 @@ impl ZonedLayout { } } +/// Serialized zoned-layout metadata. +/// +/// `zone_len` is the logical row length of each zone. `present_stats` is the sorted list of +/// statistics stored in the auxiliary stats-table child. #[derive(Debug, PartialEq, Eq, Clone)] pub struct ZonedMetadata { pub(super) zone_len: u32, @@ -317,7 +337,7 @@ mod tests { FlatLayout::new(0, dtype.clone(), SegmentId::from(0), read_ctx.clone()).into_layout(), FlatLayout::new( 0, - ZoneMap::dtype_for_stats_table(&dtype, &[]), + stats_table_dtype(&dtype, &[]), SegmentId::from(1), read_ctx, ) diff --git a/vortex-layout/src/layouts/zoned/pruning.rs b/vortex-layout/src/layouts/zoned/pruning.rs new file mode 100644 index 00000000000..7fccafddebb --- /dev/null +++ b/vortex-layout/src/layouts/zoned/pruning.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Read-time pruning support for zoned layouts. + +use std::sync::Arc; +use std::sync::LazyLock; +use std::sync::OnceLock; + +use futures::FutureExt; +use futures::TryFutureExt; +use futures::future::BoxFuture; +use futures::future::Shared; +use parking_lot::RwLock; +use vortex_array::MaskFuture; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::StructArray; +use vortex_array::dtype::FieldPath; +use vortex_array::dtype::FieldPathSet; +use vortex_array::expr::Expression; +use vortex_array::expr::pruning::checked_pruning_expr; +use vortex_array::expr::root; +use vortex_array::expr::stats::Stat; +use vortex_array::scalar_fn::fns::dynamic::DynamicExprUpdates; +use vortex_error::SharedVortexResult; +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_mask::Mask; +use vortex_session::VortexSession; +use vortex_utils::aliases::dash_map::DashMap; + +use crate::LazyReaderChildren; +use crate::layouts::zoned::ZonedLayout; +use crate::layouts::zoned::zone_map::ZoneMap; + +type SharedZoneMap = Shared>>; +pub(super) type SharedPruningResult = + Shared>>>; +type PredicateCache = Arc>>; + +pub(super) struct PruningState { + zone_count: usize, + present_stats: Arc<[Stat]>, + lazy_children: Arc, + session: VortexSession, + + pruning_result: LazyLock>>, + zone_map: OnceLock, + pruning_predicates: LazyLock>>, +} + +impl PruningState { + pub(super) fn new( + layout: &ZonedLayout, + lazy_children: Arc, + session: VortexSession, + ) -> Self { + Self { + zone_count: layout.nzones(), + present_stats: Arc::clone(layout.present_stats()), + lazy_children, + session, + pruning_result: Default::default(), + zone_map: Default::default(), + pruning_predicates: Default::default(), + } + } + + pub(super) fn pruning_mask_future(&self, expr: Expression) -> Option { + if let Some(result) = self.pruning_result.get(&expr) { + return result.value().clone(); + } + + self.pruning_result + .entry(expr.clone()) + .or_insert_with(|| match self.pruning_predicate(expr.clone()) { + None => { + tracing::debug!("No pruning predicate for expr: {expr}"); + None + } + Some(predicate) => { + tracing::debug!( + "Constructed pruning predicate for expr: {expr}: {predicate:?}" + ); + let zone_map = self.zone_map(); + let dynamic_updates = DynamicExprUpdates::new(&expr); + let session = self.session.clone(); + + Some( + async move { + let zone_map = zone_map.await?; + let initial_mask = + zone_map.prune(&predicate, &session).map_err(|err| { + err.with_context(format!( + "While evaluating pruning predicate {} (derived from {})", + predicate, expr + )) + })?; + Ok(Arc::new(PruningResult { + zone_map, + predicate, + dynamic_updates, + latest_result: RwLock::new((0, initial_mask)), + session, + })) + } + .boxed() + .shared(), + ) + } + }) + .clone() + } + + fn pruning_predicate(&self, expr: Expression) -> Option { + self.pruning_predicates + .entry(expr.clone()) + .or_default() + .get_or_init(move || { + let available_stats = FieldPathSet::from_iter( + self.present_stats + .iter() + .map(|stat| FieldPath::from_name(stat.name())), + ); + checked_pruning_expr(&expr, &available_stats).map(|(expr, _)| expr) + }) + .clone() + } + + fn zone_map(&self) -> SharedZoneMap { + self.zone_map + .get_or_init(move || { + let zone_count = self.zone_count; + let present_stats = Arc::clone(&self.present_stats); + let zones_eval = self + .lazy_children + .get(1) + .vortex_expect("failed to get zone child") + .projection_evaluation( + &(0..zone_count as u64), + &root(), + MaskFuture::new_true(zone_count), + ) + .vortex_expect("Failed construct zone map evaluation"); + let session = self.session.clone(); + + async move { + let mut ctx = session.create_execution_ctx(); + let zones_array = zones_eval.await?.execute::(&mut ctx)?; + // SAFETY: zoned layout validation ensures the zones child matches the expected + // stats-table schema for `present_stats`. + Ok(unsafe { ZoneMap::new_unchecked(zones_array, present_stats) }) + } + .map_err(Arc::new) + .boxed() + .shared() + }) + .clone() + } +} + +pub(super) struct PruningResult { + zone_map: ZoneMap, + predicate: Expression, + dynamic_updates: Option, + latest_result: RwLock<(u64, Mask)>, + session: VortexSession, +} + +impl PruningResult { + pub(super) fn mask(&self) -> VortexResult { + let Some(dynamic_updates) = &self.dynamic_updates else { + return Ok(self.latest_result.read().1.clone()); + }; + + let version = dynamic_updates.version(); + + { + let read_guard = self.latest_result.read(); + if read_guard.0 >= version { + return Ok(read_guard.1.clone()); + } + } + + let mut guard = self.latest_result.write(); + if guard.0 >= version { + return Ok(guard.1.clone()); + } + + tracing::debug!( + "Re-computing pruning mask for version {version} on {}", + self.predicate + ); + + let next_mask = self + .zone_map + .prune(&self.predicate, &self.session) + .map_err(|err| { + err.with_context(format!( + "While evaluating pruning predicate {}", + self.predicate + )) + })?; + *guard = (version, next_mask.clone()); + + Ok(next_mask) + } +} diff --git a/vortex-layout/src/layouts/zoned/reader.rs b/vortex-layout/src/layouts/zoned/reader.rs index 75e36c9648b..70bad53db9e 100644 --- a/vortex-layout/src/layouts/zoned/reader.rs +++ b/vortex-layout/src/layouts/zoned/reader.rs @@ -5,62 +5,33 @@ use std::collections::BTreeSet; use std::ops::BitAnd; use std::ops::Range; use std::sync::Arc; -use std::sync::LazyLock; -use std::sync::OnceLock; -use futures::FutureExt; -use futures::TryFutureExt; use futures::future::BoxFuture; -use futures::future::Shared; use itertools::Itertools; -use parking_lot::RwLock; use vortex_array::ArrayRef; use vortex_array::MaskFuture; -use vortex_array::VortexSessionExecute; -use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; -use vortex_array::dtype::FieldPath; -use vortex_array::dtype::FieldPathSet; use vortex_array::expr::Expression; -use vortex_array::expr::pruning::checked_pruning_expr; -use vortex_array::expr::root; -use vortex_array::scalar_fn::fns::dynamic::DynamicExprUpdates; use vortex_buffer::BitBufferMut; -use vortex_error::SharedVortexResult; use vortex_error::VortexError; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; -use vortex_utils::aliases::dash_map::DashMap; use crate::LayoutReader; use crate::LayoutReaderRef; use crate::LazyReaderChildren; use crate::layouts::zoned::ZonedLayout; -use crate::layouts::zoned::zone_map::ZoneMap; +use crate::layouts::zoned::pruning::PruningState; +use crate::layouts::zoned::schema::stats_table_dtype; use crate::segments::SegmentSource; -type SharedZoneMap = Shared>>; -type SharedPruningResult = Shared>>>; -type PredicateCache = Arc>>; - pub struct ZonedReader { layout: ZonedLayout, name: Arc, - lazy_children: LazyReaderChildren, - session: VortexSession, - - /// A cache of expr -> optional pruning result (applying the pruning expr to the zone map) - pruning_result: LazyLock>>, - - /// Shared zone map - zone_map: OnceLock, - - /// A cache of expr -> optional pruning predicate. - /// This also uses the present_stats from the `ZonedLayout` - pruning_predicates: LazyLock>>, + lazy_children: Arc, + pruning: PruningState, } impl ZonedReader { @@ -72,25 +43,22 @@ impl ZonedReader { ) -> VortexResult { let dtypes = vec![ layout.dtype.clone(), - ZoneMap::dtype_for_stats_table(layout.dtype(), layout.present_stats()), + stats_table_dtype(layout.dtype(), layout.present_stats()), ]; let names = vec![Arc::clone(&name), format!("{}.zones", name).into()]; - let lazy_children = LazyReaderChildren::new( + let lazy_children = Arc::new(LazyReaderChildren::new( Arc::clone(&layout.children), dtypes, names, Arc::clone(&segment_source), session.clone(), - ); + )); Ok(Self { + pruning: PruningState::new(&layout, Arc::clone(&lazy_children), session), layout, name, lazy_children, - session, - pruning_result: Default::default(), - zone_map: Default::default(), - pruning_predicates: Default::default(), }) } @@ -98,106 +66,6 @@ impl ZonedReader { self.lazy_children.get(0) } - /// Get or create the pruning predicate for a given expression. - fn pruning_predicate(&self, expr: Expression) -> Option { - self.pruning_predicates - .entry(expr.clone()) - .or_default() - .get_or_init(move || { - let field_path_set = FieldPathSet::from_iter( - self.layout - .present_stats - .iter() - .map(|s| FieldPath::from_name(s.name())), - ); - checked_pruning_expr(&expr, &field_path_set).map(|(expr, _)| expr) - }) - .clone() - } - - /// Get or initialize the zone map. - /// - /// Only the first successful caller will initialize the zone map, all other callers will - /// resolve to the same result. - fn zone_map(&self) -> SharedZoneMap { - self.zone_map - .get_or_init(move || { - let nzones = self.layout.nzones(); - let present_stats = Arc::clone(&self.layout.present_stats); - - let zones_eval = self - .lazy_children - .get(1) - .vortex_expect("failed to get zone child") - .projection_evaluation( - &(0..nzones as u64), - &root(), - MaskFuture::new_true(nzones), - ) - .vortex_expect("Failed construct zone map evaluation"); - let session = self.session.clone(); - - async move { - let mut ctx = session.create_execution_ctx(); - let zones_array = zones_eval.await?.execute::(&mut ctx)?; - // SAFETY: This is only fine to call because we perform validation above - Ok(unsafe { ZoneMap::new_unchecked(zones_array, present_stats) }) - } - .map_err(Arc::new) - .boxed() - .shared() - }) - .clone() - } - - /// Returns a pruning mask where `true` means the chunk _can be pruned_. - fn pruning_mask_future(&self, expr: Expression) -> Option { - // Check cache first with read-only lock - if let Some(result) = self.pruning_result.get(&expr) { - return result.value().clone(); - } - - self.pruning_result - .entry(expr.clone()) - .or_insert_with(|| match self.pruning_predicate(expr.clone()) { - None => { - tracing::debug!("No pruning predicate for expr: {expr}"); - None - } - Some(predicate) => { - tracing::debug!( - "Constructed pruning predicate for expr: {expr}: {predicate:?}" - ); - let zone_map = self.zone_map(); - let dynamic_updates = DynamicExprUpdates::new(&expr); - let session = self.session.clone(); - - Some( - async move { - let zone_map = zone_map.await?; - let initial_mask = - zone_map.prune(&predicate, &session).map_err(|err| { - err.with_context(format!( - "While evaluating pruning predicate {} (derived from {})", - predicate, expr - )) - })?; - Ok(Arc::new(PruningResult { - zone_map, - predicate, - dynamic_updates, - latest_result: RwLock::new((0, initial_mask)), - session, - })) - } - .boxed() - .shared(), - ) - } - }) - .clone() - } - /// Get the range of zone IDs containing a row range. pub(crate) fn zone_range(&self, row_range: &Range) -> Range { // Caller must ensure zone_len > 0. Legacy files may deserialize with zone_len == 0, but @@ -259,7 +127,7 @@ impl LayoutReader for ZonedReader { return Ok(data_eval); } - let Some(pruning_mask_future) = self.pruning_mask_future(expr.clone()) else { + let Some(pruning_mask_future) = self.pruning.pruning_mask_future(expr.clone()) else { tracing::debug!("Stats pruning evaluation: not prune-able {expr}"); return Ok(data_eval); }; @@ -346,67 +214,6 @@ impl LayoutReader for ZonedReader { } } -/// A wrapper for the result of pruning an expression against a zone map such that we can refresh -/// it each time the dynamic expressions are updated. -struct PruningResult { - zone_map: ZoneMap, - predicate: Expression, - dynamic_updates: Option, - latest_result: RwLock<(u64, Mask)>, - session: VortexSession, -} - -impl PruningResult { - /// Return the pruning mask, computed for _at least_ the given version. - /// - /// The version typically comes from the dynamic expression updates, but zero can be passed - /// to fetch any version. - fn mask(&self) -> VortexResult { - // If we're not dynamic, then the result is always the latest result. - let Some(dynamic_updates) = &self.dynamic_updates else { - return Ok(self.latest_result.read().1.clone()); - }; - - // Compute the latest version of the dynamic expression values. - let version = dynamic_updates.version(); - - { - let read_guard = self.latest_result.read(); - if read_guard.0 >= version { - // We're up to date, so we can return the cached result. - return Ok(read_guard.1.clone()); - } - } - - // Otherwise, we re-compute the mask for the given version number. - let mut guard = self.latest_result.write(); - - // Once we've taken the write lock, we check again in case another thread has already - // beaten us to it. - if guard.0 >= version { - return Ok(guard.1.clone()); - } - - tracing::debug!( - "Re-computing pruning mask for version {version} on {}", - self.predicate - ); - - let next_mask = self - .zone_map - .prune(&self.predicate, &self.session) - .map_err(|err| { - err.with_context(format!( - "While evaluating pruning predicate {}", - self.predicate - )) - })?; - *guard = (version, next_mask.clone()); - - Ok(next_mask) - } -} - #[cfg(test)] mod test { use std::sync::Arc; diff --git a/vortex-layout/src/layouts/zoned/schema.rs b/vortex-layout/src/layouts/zoned/schema.rs new file mode 100644 index 00000000000..e14388ba379 --- /dev/null +++ b/vortex-layout/src/layouts/zoned/schema.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Shared helpers for the zoned layout's auxiliary stats-table schema. + +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::StructFields; +use vortex_array::expr::stats::Stat; + +pub const MAX_IS_TRUNCATED: &str = "max_is_truncated"; +pub const MIN_IS_TRUNCATED: &str = "min_is_truncated"; + +/// Return the auxiliary stats-table schema for a zoned layout. +pub(crate) fn stats_table_dtype(column_dtype: &DType, present_stats: &[Stat]) -> DType { + assert!(present_stats.is_sorted(), "Stats must be sorted"); + DType::Struct( + StructFields::from_iter( + present_stats + .iter() + .filter_map(|stat| { + stat.dtype(column_dtype) + .or_else(|| { + // Backward compat: older files may have stored stats (e.g. Sum) + // for extension types by resolving through the storage dtype. + if let DType::Extension(ext) = column_dtype { + stat.dtype(ext.storage_dtype()) + } else { + None + } + }) + .map(|dtype| (stat, dtype.as_nullable())) + }) + .flat_map(|(stat, dtype)| match stat { + Stat::Max => vec![ + (stat.name(), dtype), + (MAX_IS_TRUNCATED, DType::Bool(Nullability::NonNullable)), + ], + Stat::Min => vec![ + (stat.name(), dtype), + (MIN_IS_TRUNCATED, DType::Bool(Nullability::NonNullable)), + ], + _ => vec![(stat.name(), dtype)], + }), + ), + Nullability::NonNullable, + ) +} + +#[cfg(test)] +mod tests { + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::extension::datetime::Date; + use vortex_array::extension::datetime::TimeUnit; + + use super::*; + + #[test] + fn stats_table_dtype_adds_truncation_flags() { + let dtype = stats_table_dtype( + &DType::Primitive(PType::I32, Nullability::NonNullable), + &[Stat::Max, Stat::Min, Stat::Sum], + ); + + assert_eq!( + dtype.as_struct_fields().names().as_ref(), + &[ + Stat::Max.name(), + MAX_IS_TRUNCATED, + Stat::Min.name(), + MIN_IS_TRUNCATED, + Stat::Sum.name(), + ] + ); + } + + #[test] + fn stats_table_dtype_uses_storage_dtype_for_extensions() { + let dtype = DType::Extension(Date::new(TimeUnit::Days, Nullability::NonNullable).erased()); + let stats_dtype = stats_table_dtype(&dtype, &[Stat::Max, Stat::Min]); + + assert_eq!( + stats_dtype.as_struct_fields().names().as_ref(), + &[ + Stat::Max.name(), + MAX_IS_TRUNCATED, + Stat::Min.name(), + MIN_IS_TRUNCATED, + ] + ); + } +} diff --git a/vortex-layout/src/layouts/zoned/writer.rs b/vortex-layout/src/layouts/zoned/writer.rs index c28b46558a9..9c895d986d2 100644 --- a/vortex-layout/src/layouts/zoned/writer.rs +++ b/vortex-layout/src/layouts/zoned/writer.rs @@ -1,3 +1,5 @@ +//! Write-time assembly for zoned layouts. + // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors @@ -20,8 +22,8 @@ use vortex_utils::parallelism::get_available_parallelism; use crate::IntoLayout; use crate::LayoutRef; use crate::LayoutStrategy; +use crate::layouts::zoned::StatsAccumulator; use crate::layouts::zoned::ZonedLayout; -use crate::layouts::zoned::zone_map::StatsAccumulator; use crate::segments::SegmentSinkRef; use crate::sequence::SendableSequentialStream; use crate::sequence::SequencePointer; @@ -29,6 +31,10 @@ use crate::sequence::SequentialArrayStreamExt; use crate::sequence::SequentialStreamAdapter; use crate::sequence::SequentialStreamExt; +/// Configuration for building zoned layouts. +/// +/// The input stream is assumed to already be partitioned into one chunk per zone, except +/// possibly the final partial zone. pub struct ZonedLayoutOptions { /// The size of a statistics block pub block_size: usize, @@ -58,6 +64,7 @@ pub struct ZonedStrategy { } impl ZonedStrategy { + /// Create a writer that emits a data child plus an auxiliary per-zone stats child. pub fn new( child: Child, stats: Stats, diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 00df7f72dd4..9a0173c0035 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -1,9 +1,10 @@ +//! Runtime view of a zoned layout's auxiliary per-zone statistics table. + // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::sync::Arc; -use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -14,23 +15,17 @@ use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::dtype::StructFields; use vortex_array::expr::Expression; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; -use vortex_array::expr::stats::StatsProvider; use vortex_array::stats::StatsSet; -use vortex_array::validity::Validity; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_mask::Mask; use vortex_session::VortexSession; -use crate::layouts::zoned::builder::MAX_IS_TRUNCATED; -use crate::layouts::zoned::builder::MIN_IS_TRUNCATED; -use crate::layouts::zoned::builder::StatsArrayBuilder; -use crate::layouts::zoned::builder::stats_builder_with_capacity; +pub use crate::layouts::zoned::builder::StatsAccumulator; +use crate::layouts::zoned::schema::stats_table_dtype; /// A zone map containing statistics for a column. /// Each row of the zone map corresponds to a chunk of the column. @@ -52,12 +47,12 @@ impl ZoneMap { array: StructArray, stats: Arc<[Stat]>, ) -> VortexResult { - let expected_dtype = Self::dtype_for_stats_table(&column_dtype, &stats); + let expected_dtype = stats_table_dtype(&column_dtype, &stats); if &expected_dtype != array.dtype() { vortex_bail!("Array dtype does not match expected zone map dtype: {expected_dtype}"); } - // SAFETY: We checked that the + // SAFETY: We checked that the array matches the expected stats-table schema. Ok(unsafe { Self::new_unchecked(array, stats) }) } @@ -66,45 +61,17 @@ impl ZoneMap { /// # Safety /// /// Assumes that the input struct array has the correct statistics as fields. Or in other words, - /// the [`DType`] of the input array is equal to the result of [`Self::dtype_for_stats_table`]. + /// the [`DType`] of the input array is equal to the result of `stats_table_dtype`. pub unsafe fn new_unchecked(array: StructArray, stats: Arc<[Stat]>) -> Self { Self { array, stats } } /// Returns the [`DType`] of the statistics table given a set of statistics and column [`DType`]. + /// + /// This remains as a compatibility wrapper around the zoned schema helper. + #[deprecated(note = "use `stats_table_dtype` from `crate::layouts::zoned::schema` instead")] pub fn dtype_for_stats_table(column_dtype: &DType, present_stats: &[Stat]) -> DType { - assert!(present_stats.is_sorted(), "Stats must be sorted"); - DType::Struct( - StructFields::from_iter( - present_stats - .iter() - .filter_map(|stat| { - stat.dtype(column_dtype) - .or_else(|| { - // Backward compat: older files may have stored stats (e.g. Sum) - // for extension types by resolving through the storage dtype. - if let DType::Extension(ext) = column_dtype { - stat.dtype(ext.storage_dtype()) - } else { - None - } - }) - .map(|dtype| (stat, dtype.as_nullable())) - }) - .flat_map(|(s, dt)| match s { - Stat::Max => vec![ - (s.name(), dt), - (MAX_IS_TRUNCATED, DType::Bool(Nullability::NonNullable)), - ], - Stat::Min => vec![ - (s.name(), dt), - (MIN_IS_TRUNCATED, DType::Bool(Nullability::NonNullable)), - ], - _ => vec![(s.name(), dt)], - }), - ), - Nullability::NonNullable, - ) + stats_table_dtype(column_dtype, present_stats) } /// Returns the underlying [`StructArray`] backing the zone map @@ -174,121 +141,17 @@ impl ZoneMap { } } -// TODO(ngates): we should make it such that the zone map stores a mirror of the DType -// underneath each stats column. For example, `min: i32` for an `i32` array. -// Or `min: {a: i32, b: i32}` for a struct array of type `{a: i32, b: i32}`. -// See: -/// Accumulates statistics for a column. -pub struct StatsAccumulator { - builders: Vec>, - length: usize, -} - -impl StatsAccumulator { - pub fn new(dtype: &DType, stats: &[Stat], max_variable_length_statistics_size: usize) -> Self { - let builders = stats - .iter() - .filter_map(|&s| { - s.dtype(dtype).map(|stat_dtype| { - stats_builder_with_capacity( - s, - &stat_dtype.as_nullable(), - 1024, - max_variable_length_statistics_size, - ) - }) - }) - .collect::>(); - - Self { - builders, - length: 0, - } - } - - pub fn push_chunk_without_compute(&mut self, array: &ArrayRef) -> VortexResult<()> { - for builder in self.builders.iter_mut() { - if let Some(Precision::Exact(v)) = array.statistics().get(builder.stat()) { - builder.append_scalar(v.cast(&v.dtype().as_nullable())?)?; - } else { - builder.append_null(); - } - } - self.length += 1; - Ok(()) - } - - pub fn push_chunk(&mut self, array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<()> { - for builder in self.builders.iter_mut() { - if let Some(v) = array.statistics().compute_stat(builder.stat(), ctx)? { - builder.append_scalar(v.cast(&v.dtype().as_nullable())?)?; - } else { - builder.append_null(); - } - } - self.length += 1; - Ok(()) - } - - /// Finishes the accumulator into a [`ZoneMap`]. - /// - /// Returns `None` if none of the requested statistics can be computed, for example they are - /// not applicable to the column's data type. - pub fn as_stats_table(&mut self) -> VortexResult> { - let mut names = Vec::new(); - let mut fields = Vec::new(); - let mut stats = Vec::new(); - - for builder in self - .builders - .iter_mut() - // We sort the stats so the DType is deterministic based on which stats are present. - .sorted_unstable_by_key(|b| b.stat()) - { - let values = builder.finish(); - - // We drop any all-null stats columns - if values.all_invalid()? { - continue; - } - - stats.push(builder.stat()); - names.extend(values.names); - fields.extend(values.arrays); - } - - if names.is_empty() { - return Ok(None); - } - - Ok(Some(ZoneMap { - array: StructArray::try_new(names.into(), fields, self.length, Validity::NonNullable) - .vortex_expect("Failed to create zone map"), - stats: stats.into(), - })) - } -} - #[cfg(test)] mod tests { use std::sync::Arc; - use rstest::rstest; use vortex_array::IntoArray; - use vortex_array::LEGACY_SESSION; - use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; - use vortex_array::arrays::bool::BoolArrayExt; - use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::assert_arrays_eq; - use vortex_array::builders::ArrayBuilder; - use vortex_array::builders::VarBinViewBuilder; - use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; use vortex_array::dtype::FieldPathSet; - use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::expr::gt; use vortex_array::expr::gt_eq; @@ -298,106 +161,12 @@ mod tests { use vortex_array::expr::root; use vortex_array::expr::stats::Stat; use vortex_array::validity::Validity; - use vortex_buffer::BitBuffer; use vortex_buffer::buffer; - use vortex_error::VortexExpect; - use crate::layouts::zoned::MAX_IS_TRUNCATED; - use crate::layouts::zoned::MIN_IS_TRUNCATED; - use crate::layouts::zoned::zone_map::StatsAccumulator; use crate::layouts::zoned::zone_map::ZoneMap; use crate::test::SESSION; - #[rstest] - #[case(DType::Utf8(Nullability::NonNullable))] - #[case(DType::Binary(Nullability::NonNullable))] - fn truncates_accumulated_stats(#[case] dtype: DType) { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let mut builder = VarBinViewBuilder::with_capacity(dtype.clone(), 2); - builder.append_value("Value to be truncated"); - builder.append_value("untruncated"); - let mut builder2 = VarBinViewBuilder::with_capacity(dtype, 2); - builder2.append_value("Another"); - builder2.append_value("wait a minute"); - let mut acc = - StatsAccumulator::new(builder.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); - acc.push_chunk(&builder.finish(), &mut ctx) - .vortex_expect("push_chunk should succeed for test data"); - acc.push_chunk(&builder2.finish(), &mut ctx) - .vortex_expect("push_chunk should succeed for test data"); - let stats_table = acc - .as_stats_table() - .unwrap() - .expect("Must have stats table"); - assert_eq!( - stats_table.array.names().as_ref(), - &[ - Stat::Max.name(), - MAX_IS_TRUNCATED, - Stat::Min.name(), - MIN_IS_TRUNCATED, - ] - ); - let field1_bool = stats_table - .array - .unmasked_field(1) - .clone() - .execute::(&mut ctx) - .unwrap(); - assert_eq!( - field1_bool.to_bit_buffer(), - BitBuffer::from(vec![false, true]) - ); - let field3_bool = stats_table - .array - .unmasked_field(3) - .clone() - .execute::(&mut ctx) - .unwrap(); - assert_eq!( - field3_bool.to_bit_buffer(), - BitBuffer::from(vec![true, false]) - ); - } - #[test] - fn always_adds_is_truncated_column() { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let array = buffer![0, 1, 2].into_array(); - let mut acc = StatsAccumulator::new(array.dtype(), &[Stat::Max, Stat::Min, Stat::Sum], 12); - acc.push_chunk(&array, &mut ctx) - .vortex_expect("push_chunk should succeed for test array"); - let stats_table = acc - .as_stats_table() - .unwrap() - .expect("Must have stats table"); - assert_eq!( - stats_table.array.names().as_ref(), - &[ - Stat::Max.name(), - MAX_IS_TRUNCATED, - Stat::Min.name(), - MIN_IS_TRUNCATED, - Stat::Sum.name(), - ] - ); - let field1_bool = stats_table - .array - .unmasked_field(1) - .clone() - .execute::(&mut ctx) - .unwrap(); - assert_eq!(field1_bool.to_bit_buffer(), BitBuffer::from(vec![false])); - let field3_bool = stats_table - .array - .unmasked_field(3) - .clone() - .execute::(&mut ctx) - .unwrap(); - assert_eq!(field3_bool.to_bit_buffer(), BitBuffer::from(vec![false])); - } - - #[rstest] fn test_zone_map_prunes() { // All stats that are known at pruning time. let stats = FieldPathSet::from_iter([ From a1d949306c3fe1fb46905b14e75621d29001dc60 Mon Sep 17 00:00:00 2001 From: Connor Tsui <87130162+connortsui20@users.noreply.github.com> Date: Sun, 26 Apr 2026 13:59:27 -0400 Subject: [PATCH 191/250] remove gcs dead code from vortex-bench (#7640) ## Summary AFAIK we do not use gcs as a storage backend. I'm guessing this is a historical artifact? ## Testing N/A Signed-off-by: Connor Tsui --- vortex-bench/src/measurements.rs | 2 +- vortex-bench/src/utils/constants.rs | 1 - vortex-bench/src/utils/file.rs | 6 ++---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vortex-bench/src/measurements.rs b/vortex-bench/src/measurements.rs index 91036db3f5e..4f6abe65563 100644 --- a/vortex-bench/src/measurements.rs +++ b/vortex-bench/src/measurements.rs @@ -244,7 +244,7 @@ pub struct QueryMeasurement { pub target: Target, pub benchmark_dataset: BenchmarkDataset, pub benchmark_runner: String, - /// The storage backend against which this test was run. One of: s3, gcs, nvme. + /// The storage backend against which this test was run. One of: s3, nvme. pub storage: String, pub runs: Vec, } diff --git a/vortex-bench/src/utils/constants.rs b/vortex-bench/src/utils/constants.rs index 959ad63b7f4..c9f5ca4c442 100644 --- a/vortex-bench/src/utils/constants.rs +++ b/vortex-bench/src/utils/constants.rs @@ -4,4 +4,3 @@ /// Storage type constants pub const STORAGE_NVME: &str = "nvme"; pub const STORAGE_S3: &str = "s3"; -pub const STORAGE_GCS: &str = "gcs"; diff --git a/vortex-bench/src/utils/file.rs b/vortex-bench/src/utils/file.rs index c8916d68398..fb7ecc668fb 100644 --- a/vortex-bench/src/utils/file.rs +++ b/vortex-bench/src/utils/file.rs @@ -154,20 +154,18 @@ pub fn resolve_data_url(remote_data_dir: Option<&str>, local_subdir: &str) -> Re /// Convert a URL scheme to a storage type string /// -/// Maps URL schemes (s3, gcs, file) to storage type identifiers +/// Maps URL schemes (s3, file) to storage type identifiers /// for benchmark reporting. /// /// # Returns -/// - A storage type string ("s3", "gcs", "nvme") +/// - A storage type string ("s3", "nvme") /// - Or an error if the scheme is unknown pub fn url_scheme_to_storage(url: &Url) -> Result { - use super::constants::STORAGE_GCS; use super::constants::STORAGE_NVME; use super::constants::STORAGE_S3; match url.scheme() { STORAGE_S3 => Ok(STORAGE_S3.to_owned()), - STORAGE_GCS => Ok(STORAGE_GCS.to_owned()), "file" => Ok(STORAGE_NVME.to_owned()), otherwise => { bail!("unknown URL scheme: {}", otherwise) From a001ba86267353f56d5c55d430800f920e8571cd Mon Sep 17 00:00:00 2001 From: Brian Hart <12980763+brainhart@users.noreply.github.com> Date: Sun, 26 Apr 2026 15:27:34 -0700 Subject: [PATCH 192/250] Fix dtype mismatch in FileStatsLayoutReader for stat scalars (#7639) ## Summary Use the stat's own dtype (e.g. u64 for NullCount) rather than the field dtype when constructing stat scalars in stats_ref. This fixes IS NULL pruning on nullable timestamp columns which previously failed with a dtype mismatch. ## Testing Added regression test Signed-off-by: Brian Hart Co-authored-by: Brian Hart --- vortex-file/src/v2/file_stats_reader.rs | 65 ++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/vortex-file/src/v2/file_stats_reader.rs b/vortex-file/src/v2/file_stats_reader.rs index e5b504d1af4..5634ec09bd4 100644 --- a/vortex-file/src/v2/file_stats_reader.rs +++ b/vortex-file/src/v2/file_stats_reader.rs @@ -126,7 +126,10 @@ impl StatsCatalog for FileStatsLayoutReader { let stat_value = field_stats.get(stat)?.as_exact()?; let field_dtype = self.struct_fields.field_by_index(field_idx)?; - let stat_scalar = Scalar::try_new(field_dtype, Some(stat_value)).ok()?; + // Use the stat's own dtype rather than the field dtype. For example, + // NullCount is always u64 regardless of the column type. + let stat_dtype = stat.dtype(&field_dtype)?; + let stat_scalar = Scalar::try_new(stat_dtype, Some(stat_value)).ok()?; Some(lit(stat_scalar)) } @@ -209,16 +212,20 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray as _; + use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; + use vortex_array::arrays::datetime::TemporalData; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::expr::get_item; use vortex_array::expr::gt; + use vortex_array::expr::is_null; use vortex_array::expr::lit; use vortex_array::expr::root; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; + use vortex_array::extension::datetime::TimeUnit; use vortex_array::scalar::ScalarValue; use vortex_array::scalar_fn::session::ScalarFnSession; use vortex_array::session::ArraySession; @@ -232,6 +239,7 @@ mod tests { use vortex_layout::LayoutStrategy; use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; use vortex_layout::layouts::table::TableStrategy; + use vortex_layout::segments::SegmentSink; use vortex_layout::segments::TestSegments; use vortex_layout::sequence::SequenceId; use vortex_layout::sequence::SequentialArrayStreamExt; @@ -337,4 +345,59 @@ mod tests { Ok(()) }) } + + /// Regression test: `IS NULL` on a nullable timestamp column must not fail with a + /// dtype mismatch. The bug was that `stats_ref` used the *field* dtype (timestamp) + /// for the `NullCount` stat scalar instead of the stat's own dtype (u64). + #[test] + fn is_null_pruning_on_nullable_timestamp_column() -> VortexResult<()> { + block_on(|handle| async { + let session = SESSION.clone().with_handle(handle); + let ctx = ArrayContext::empty(); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + + // Build a struct with a nullable timestamp column containing some nulls. + let prim_array = + PrimitiveArray::from_option_iter([Some(1_000_000i64), None, Some(3_000_000)]) + .into_array(); + let ts_data = TemporalData::new_timestamp(prim_array, TimeUnit::Microseconds, None); + let ts_dtype = ts_data.dtype().clone(); + let ts_array = ts_data.into_array(); + + let struct_array = StructArray::from_fields([("deleted_at", ts_array)].as_slice())?; + + let strategy = TableStrategy::new( + Arc::new(FlatLayoutStrategy::default()), + Arc::new(FlatLayoutStrategy::default()), + ); + let layout = strategy + .write_stream( + ctx, + Arc::clone(&segments) as Arc, + struct_array.into_array().to_array_stream().sequenced(ptr), + eof, + &session, + ) + .await?; + + let child = layout.new_reader("".into(), segments, &SESSION)?; + + // File-level stats: 1 null in deleted_at. + let mut stats = StatsSet::default(); + stats.set(Stat::NullCount, Precision::exact(ScalarValue::from(1u64))); + let file_stats = FileStatistics::new(Arc::from([stats]), Arc::from([ts_dtype])); + + let reader = FileStatsLayoutReader::new(child, file_stats, SESSION.clone()); + + // `is_null(deleted_at)` — should NOT panic or error due to dtype mismatch. + let expr = is_null(get_item("deleted_at", root())); + let mask = Mask::new_true(3); + let result = reader.pruning_evaluation(&(0..3), &expr, mask)?.await?; + // null_count is 1 (non-zero), so is_null is not falsified => not pruned. + assert_eq!(result, Mask::new_true(3)); + + Ok(()) + }) + } } From f41884486fd05b2374fc065d6c1bc6c39fd1eb6f Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 27 Apr 2026 05:14:21 -0400 Subject: [PATCH 193/250] Add ability to override function behaviour via registry in VortexSession (#7588) This logic isn't used yet but will be used to allow us to customise behaviour of functions depending on an integration point, i.e. Datafusion can have it's casting logic that is different from arrow casting logic while everyone using vortex can still continue calling `cast` and not specialize for the engine they're using Thing to consider is whether we want require passing session to optimise or whether we should remove the implicit optimise calls and defer them to execute loop The next pr will replace struct casting logic with Arrow and DF specific behaviour. --------- Signed-off-by: Robert Kruszewski --- Cargo.lock | 2 +- Cargo.toml | 2 +- vortex-array/Cargo.toml | 1 + vortex-array/public-api.lock | 42 ++++++- .../src/arrays/extension/compute/rules.rs | 9 +- vortex-array/src/executor.rs | 27 +++-- vortex-array/src/optimizer/kernels.rs | 113 ++++++++++++++++++ vortex-array/src/optimizer/mod.rs | 84 +++++++++++-- vortex-duckdb/src/datasource.rs | 2 +- vortex-session/Cargo.toml | 1 - vortex-session/src/registry.rs | 1 + vortex/src/lib.rs | 5 +- 12 files changed, 260 insertions(+), 29 deletions(-) create mode 100644 vortex-array/src/optimizer/kernels.rs diff --git a/Cargo.lock b/Cargo.lock index 727918f14e7..22b8445fcfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10018,6 +10018,7 @@ name = "vortex-array" version = "0.1.0" dependencies = [ "arbitrary", + "arc-swap", "arcref", "arrow-arith 58.1.0", "arrow-array 58.1.0", @@ -10815,7 +10816,6 @@ dependencies = [ name = "vortex-session" version = "0.1.0" dependencies = [ - "arcref", "dashmap", "lasso", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 92e08f1338f..4ddcfbe3d43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ version = "0.1.0" aho-corasick = "1.1.3" anyhow = "1.0.97" arbitrary = "1.3.2" -arc-swap = "1.8" +arc-swap = "1.9" arcref = "0.2.0" arrow-arith = "58" arrow-array = "58" diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index f9adbeb99db..f8676d76ef0 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -21,6 +21,7 @@ workspace = true [dependencies] arbitrary = { workspace = true, optional = true } +arc-swap = { workspace = true } arcref = { workspace = true } arrow-arith = { workspace = true } arrow-array = { workspace = true, features = ["ffi"] } diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 8ee3c97f17b..02d767b3cb1 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -13222,6 +13222,36 @@ pub vortex_array::normalize::NormalizeOptions::operation: vortex_array::normaliz pub mod vortex_array::optimizer +pub mod vortex_array::optimizer::kernels + +pub struct vortex_array::optimizer::kernels::ArrayKernels + +impl vortex_array::optimizer::kernels::ArrayKernels + +pub fn vortex_array::optimizer::kernels::ArrayKernels::empty() -> Self + +pub fn vortex_array::optimizer::kernels::ArrayKernels::find_reduce_parent(&self, parent: vortex_session::registry::Id, child: vortex_session::registry::Id) -> core::option::Option> + +pub fn vortex_array::optimizer::kernels::ArrayKernels::register_reduce_parent>(&self, parent: vortex_session::registry::Id, child: vortex_session::registry::Id, fns: I) + +impl core::default::Default for vortex_array::optimizer::kernels::ArrayKernels + +pub fn vortex_array::optimizer::kernels::ArrayKernels::default() -> vortex_array::optimizer::kernels::ArrayKernels + +impl core::fmt::Debug for vortex_array::optimizer::kernels::ArrayKernels + +pub fn vortex_array::optimizer::kernels::ArrayKernels::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +pub trait vortex_array::optimizer::kernels::ArrayKernelsExt: vortex_session::SessionExt + +pub fn vortex_array::optimizer::kernels::ArrayKernelsExt::kernels(&self) -> vortex_session::Ref<'_, vortex_array::optimizer::kernels::ArrayKernels> + +impl vortex_array::optimizer::kernels::ArrayKernelsExt for S + +pub fn S::kernels(&self) -> vortex_session::Ref<'_, vortex_array::optimizer::kernels::ArrayKernels> + +pub type vortex_array::optimizer::kernels::ReduceParentFn = fn(child: &vortex_array::ArrayRef, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + pub mod vortex_array::optimizer::rules pub struct vortex_array::optimizer::rules::ParentReduceRuleAdapter @@ -13364,13 +13394,17 @@ pub trait vortex_array::optimizer::ArrayOptimizer pub fn vortex_array::optimizer::ArrayOptimizer::optimize(&self) -> vortex_error::VortexResult -pub fn vortex_array::optimizer::ArrayOptimizer::optimize_recursive(&self) -> vortex_error::VortexResult +pub fn vortex_array::optimizer::ArrayOptimizer::optimize_ctx(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::optimizer::ArrayOptimizer::optimize_recursive(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::optimizer::ArrayOptimizer for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::optimize(&self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::optimize_recursive(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::optimize_ctx(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::optimize_recursive(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult pub mod vortex_array::patches @@ -22328,7 +22362,9 @@ impl vortex_array::optimizer::ArrayOptimizer for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::optimize(&self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::optimize_recursive(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::optimize_ctx(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayRef::optimize_recursive(&self, session: &vortex_session::VortexSession) -> vortex_error::VortexResult impl vortex_array::scalar_fn::ReduceNode for vortex_array::ArrayRef diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index b6e2d5a1e06..d7c8469d1dc 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -78,9 +78,12 @@ impl ArrayParentReduceRule for ExtensionFilterPushDownRule { #[cfg(test)] mod tests { + use std::sync::LazyLock; + use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_mask::Mask; + use vortex_session::VortexSession; use crate::IntoArray; #[expect(deprecated)] @@ -108,6 +111,10 @@ mod tests { use crate::scalar::ScalarValue; use crate::scalar_fn::fns::binary::Binary; use crate::scalar_fn::fns::operators::Operator; + use crate::session::ArraySession; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] struct TestExt; @@ -220,7 +227,7 @@ mod tests { .try_new_array(3, Operator::Lt, [constant_ext, ext_array]) .unwrap(); - let optimized = scalar_fn_array.optimize_recursive().unwrap(); + let optimized = scalar_fn_array.optimize_recursive(&SESSION).unwrap(); let scalar_fn = optimized.as_opt::().unwrap(); let children = scalar_fn.children(); let constant = children[0] diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index e35b485972a..1ca27bc1de5 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -10,9 +10,10 @@ //! 3. **`execute_parent`** -- child-driven fused execution (may read buffers). //! 4. **`execute`** -- the encoding's own decode step (most expensive). //! -//! The main entry point is [`DynArray::execute_until`], which uses an explicit work stack +//! The main entry point is [`ArrayRef::execute_until`], which uses an explicit work stack //! to drive execution iteratively without recursion. Between steps, the optimizer runs -//! reduce/reduce_parent rules to fixpoint. +//! reduce/reduce_parent rules to fixpoint using the active [`ExecutionCtx`] session, so +//! session-registered optimizer kernels participate during execution. //! //! See for a full description //! of the model. @@ -88,17 +89,21 @@ impl ArrayRef { /// /// Each iteration proceeds through three steps in order: /// - /// 1. **Done / canonical check** — if `current` satisfies the active done predicate or is + /// 1. **Done / canonical check** - if `current` satisfies the active done predicate or is /// canonical, splice it back into the stacked parent (if any) and continue, or return. - /// 2. **`execute_parent` on children** — try each child's `execute_parent` against `current` + /// 2. **`execute_parent` on children** - try each child's `execute_parent` against `current` /// as the parent (e.g. `Filter(RunEnd)` → `FilterExecuteAdaptor` fires from RunEnd). /// If there is a stacked parent frame, the rewritten child is spliced back into it so /// that optimize and further `execute_parent` can fire on the reconstructed parent /// (e.g. `Slice(RunEnd)` → `RunEnd` spliced into stacked `Filter` → `Filter(RunEnd)` /// whose `FilterExecuteAdaptor` fires on the next iteration). - /// 3. **`execute`** — call the encoding's own execute step, which either returns `Done` or + /// 3. **`execute`** - call the encoding's own execute step, which either returns `Done` or /// `ExecuteSlot(i)` to push a child onto the stack for focused execution. /// + /// Optimizer calls in this loop use [`ExecutionCtx::session`], so kernels registered on the + /// session's [`ArrayKernels`](crate::optimizer::kernels::ArrayKernels) are visible between + /// execution steps. + /// /// Note: the returned array may not match `M`. If execution converges to a canonical form /// that does not match `M`, the canonical array is returned since no further execution /// progress is possible. @@ -110,7 +115,7 @@ impl ArrayRef { let mut stack: Vec = Vec::new(); for _ in 0..max_iterations() { - // Step 1: done / canonical — splice back into stacked parent or return. + // Step 1: done / canonical - splice back into stacked parent or return. let is_done = stack .last() .map_or(M::matches as DonePredicate, |frame| frame.done); @@ -121,7 +126,7 @@ impl ArrayRef { return Ok(current); } Some(frame) => { - current = frame.put_back(current)?.optimize()?; + current = frame.put_back(current)?.optimize_ctx(ctx.session())?; continue; } } @@ -137,9 +142,9 @@ impl ArrayRef { "execute_parent rewrote {} -> {}", current, rewritten )); - current = rewritten.optimize()?; + current = rewritten.optimize_ctx(ctx.session())?; if let Some(frame) = stack.pop() { - current = frame.put_back(current)?.optimize()?; + current = frame.put_back(current)?.optimize_ctx(ctx.session())?; } continue; } @@ -158,7 +163,7 @@ impl ArrayRef { )); let frame = StackFrame::new(parent, i, done, &child); stack.push(frame); - current = child.optimize()?; + current = child.optimize_ctx(ctx.session())?; } ExecutionStep::Done => { ctx.log(format_args!("Done: {}", array)); @@ -523,7 +528,7 @@ macro_rules! require_child { /// execution of child `$idx`. /// /// Unlike `require_child!`, this is a statement macro (no value produced) and does not clone -/// `$parent` — it is moved into the early-return path. +/// `$parent` - it is moved into the early-return path. /// /// ```ignore /// require_opt_child!(array, array.patches().map(|p| p.indices()), 1 => Primitive); diff --git a/vortex-array/src/optimizer/kernels.rs b/vortex-array/src/optimizer/kernels.rs new file mode 100644 index 00000000000..af70d0f9b9b --- /dev/null +++ b/vortex-array/src/optimizer/kernels.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Session-scoped registry for optimizer kernels. +//! +//! [`ArrayKernels`] stores function pointers that participate in array optimization without +//! adding rules to an encoding vtable. The optimizer currently consults it for parent-reduce +//! rewrites before the child encoding's static `PARENT_RULES`. A registered function can +//! therefore add a rule for an extension encoding or take precedence over a built-in rule. +//! +//! Kernel entries are addressed by `(outer_id, child_id, kind)`. For parent-reduce kernels, +//! `outer_id` is the id returned by the parent array's `encoding_id()` and `child_id` is the +//! child array's `encoding_id()`. For [`ScalarFn`](crate::arrays::ScalarFn) parents, the parent +//! id is the scalar function id. +//! +//! Sessions created by the top-level `vortex` crate install an empty registry by default. Other +//! sessions can add it with [`VortexSession::with`](vortex_session::VortexSession::with) or rely +//! on [`ArrayKernelsExt::kernels`] to insert the default value. + +use std::hash::BuildHasher; +use std::sync::Arc; +use std::sync::LazyLock; + +use arc_swap::ArcSwap; +use vortex_error::VortexResult; +use vortex_session::Ref; +use vortex_session::SessionExt; +use vortex_session::registry::Id; +use vortex_utils::aliases::DefaultHashBuilder; +use vortex_utils::aliases::hash_map::HashMap; + +use crate::ArrayRef; + +/// Shared hasher used to combine `(outer, child, FnKind)` tuples into [`FnRegistry`] keys. +static FN_HASHER: LazyLock = LazyLock::new(DefaultHashBuilder::default); + +/// Function pointer for a plugin-provided parent-reduce rewrite. +/// +/// The optimizer calls this with the matched `child`, its `parent`, and the slot index where the +/// child appears. Return `Ok(Some(new_parent))` to replace the parent, or `Ok(None)` when the +/// rewrite does not apply. +/// +/// Implementations must preserve the parent's logical length and dtype, matching the invariant +/// required of static parent-reduce rules. +pub type ReduceParentFn = + fn(child: &ArrayRef, parent: &ArrayRef, child_idx: usize) -> VortexResult>; + +/// Session-scoped registry of optimizer kernel functions. +#[derive(Debug, Default)] +pub struct ArrayKernels { + reduce_parent: ArcSwap>>, +} + +impl ArrayKernels { + /// Create an empty [`ArrayKernels`] with no kernels registered. + pub fn empty() -> Self { + Self::default() + } + + /// Register a [`ReduceParentFn`] for `(outer, child)`. + /// + /// The optimizer will invoke `f` when it sees a parent with encoding id `outer` holding a + /// child with encoding id `child` during a `reduce_parent` step, before trying the child + /// encoding's static `PARENT_RULES`. `outer` is usually the parent array's encoding id. For + /// `ScalarFnArray`, it is the scalar function id, for example `Cast.id()`. + /// + /// Replaces any function already registered for the same pair. + pub fn register_reduce_parent>( + &self, + parent: Id, + child: Id, + fns: I, + ) { + let registry = self.reduce_parent.load(); + let id = self.hash_fn_ids(parent, child); + let mut owned_registry = registry.as_ref().clone(); + if let Some(existing) = owned_registry.remove(&id) { + owned_registry.insert(id, existing.as_ref().iter().cloned().chain(fns).collect()); + } else { + owned_registry.insert(id, fns.into_iter().collect()); + } + self.reduce_parent.store(Arc::new(owned_registry)); + } + + /// Look up the [`ReduceParentFn`] registered for `(outer, child)`. + /// + /// Returns an owned [`Arc`] so the session-variable borrow can be dropped before invoking the + /// function. + pub fn find_reduce_parent(&self, parent: Id, child: Id) -> Option> { + let id = self.hash_fn_ids(parent, child); + let map = self.reduce_parent.load(); + let entry = map.get(&id)?; + Some(Arc::clone(entry)) + } + + /// Combine a typed kernel id tuple into the `u64` key expected by the underlying + /// [`FnRegistry`]. All typed helpers use this path so registration and lookup agree. + fn hash_fn_ids(&self, parent: Id, child: Id) -> u64 { + FN_HASHER.hash_one((parent, child)) + } +} + +/// Extension trait for accessing optimizer kernels from a +/// [`VortexSession`](vortex_session::VortexSession). +pub trait ArrayKernelsExt: SessionExt { + /// Returns the [`ArrayKernels`] session variable, inserting a default-constructed one if + /// none has been registered on the session yet. + fn kernels(&self) -> Ref<'_, ArrayKernels> { + self.get::() + } +} + +impl ArrayKernelsExt for S {} diff --git a/vortex-array/src/optimizer/mod.rs b/vortex-array/src/optimizer/mod.rs index 0401e69cb9a..70a041bcc18 100644 --- a/vortex-array/src/optimizer/mod.rs +++ b/vortex-array/src/optimizer/mod.rs @@ -6,34 +6,83 @@ //! //! Optimization runs between execution steps, which is what enables cross-step optimizations: //! after a child is decoded, new `reduce_parent` rules may match that were previously blocked. +//! +//! There are three public entry points on [`ArrayOptimizer`]: +//! +//! - [`ArrayOptimizer::optimize`] uses only static rules registered on encoding vtables. +//! - [`ArrayOptimizer::optimize_ctx`] also consults session-scoped [`ArrayKernels`] before +//! static parent-reduce rules, so this is the entry point used by execution. +//! - [`ArrayOptimizer::optimize_recursive`] applies the session-aware optimizer to the root and +//! every descendant. + +use std::sync::Arc; use vortex_error::VortexResult; use vortex_error::vortex_bail; +use vortex_session::SessionExt; +use vortex_session::VortexSession; use crate::ArrayRef; +use crate::optimizer::kernels::ArrayKernels; +use crate::optimizer::kernels::ReduceParentFn; +pub mod kernels; pub mod rules; /// Extension trait for optimizing array trees using reduce/reduce_parent rules. pub trait ArrayOptimizer { - /// Optimize the root array node only by running reduce and reduce_parent rules to fixpoint. + /// Optimize the root array node by running reduce and reduce_parent rules to fixpoint. + /// + /// This uses only static rules registered on encoding vtables. Use [`Self::optimize_ctx`] + /// when session-registered [`ArrayKernels`] should participate. fn optimize(&self) -> VortexResult; + /// Optimize the root array node using static rules and any [`ArrayKernels`] on `session`. + /// + /// Session kernels are checked for each `(parent_encoding_id, child_encoding_id)` pair before + /// the child's static `PARENT_RULES`. If `session` does not contain [`ArrayKernels`], this + /// behaves like [`Self::optimize`]. + fn optimize_ctx(&self, session: &VortexSession) -> VortexResult; + /// Optimize the entire array tree recursively (root and all descendants). - fn optimize_recursive(&self) -> VortexResult; + /// + /// This uses the same session-aware rule ordering as [`Self::optimize_ctx`] for every node in + /// the tree. + fn optimize_recursive(&self, session: &VortexSession) -> VortexResult; } impl ArrayOptimizer for ArrayRef { fn optimize(&self) -> VortexResult { - Ok(try_optimize(self)?.unwrap_or_else(|| self.clone())) + Ok(try_optimize(self, None)?.unwrap_or_else(|| self.clone())) + } + + fn optimize_ctx(&self, session: &VortexSession) -> VortexResult { + Ok(try_optimize(self, Some(session))?.unwrap_or_else(|| self.clone())) } - fn optimize_recursive(&self) -> VortexResult { - Ok(try_optimize_recursive(self)?.unwrap_or_else(|| self.clone())) + fn optimize_recursive(&self, session: &VortexSession) -> VortexResult { + Ok(try_optimize_recursive(self, session)?.unwrap_or_else(|| self.clone())) } } -fn try_optimize(array: &ArrayRef) -> VortexResult> { +/// Resolve a session-registered [`ReduceParentFn`] for the `(parent, child)` pair. +/// +/// The returned [`Arc`] is owned so the caller can drop the [`ArrayKernels`] borrow before +/// invoking the function. +fn plugin_reduce_parent( + session: &VortexSession, + parent: &ArrayRef, + child: &ArrayRef, +) -> Option> { + session + .get_opt::() + .and_then(|s| s.find_reduce_parent(parent.encoding_id(), child.encoding_id())) +} + +fn try_optimize( + array: &ArrayRef, + session: Option<&VortexSession>, +) -> VortexResult> { let mut current_array = array.clone(); let mut any_optimizations = false; @@ -55,6 +104,20 @@ fn try_optimize(array: &ArrayRef) -> VortexResult> { // Its important to take all slots here, as `current_array` can change inside the loop. for (slot_idx, slot) in current_array.slots().iter().enumerate() { let Some(child) = slot else { continue }; + + // Session kernels take precedence over the child encoding's static PARENT_RULES. + if let Some(session) = session + && let Some(plugins) = plugin_reduce_parent(session, ¤t_array, child) + { + for plugin in plugins.as_ref() { + if let Some(new_array) = plugin(child, ¤t_array, slot_idx)? { + current_array = new_array; + any_optimizations = true; + continue 'outer; + } + } + } + if let Some(new_array) = child.reduce_parent(¤t_array, slot_idx)? { // If the parent was replaced, then we attempt to reduce it again. current_array = new_array; @@ -76,11 +139,14 @@ fn try_optimize(array: &ArrayRef) -> VortexResult> { } } -fn try_optimize_recursive(array: &ArrayRef) -> VortexResult> { +fn try_optimize_recursive( + array: &ArrayRef, + session: &VortexSession, +) -> VortexResult> { let mut current_array = array.clone(); let mut any_optimizations = false; - if let Some(new_array) = try_optimize(¤t_array)? { + if let Some(new_array) = try_optimize(¤t_array, Some(session))? { current_array = new_array; any_optimizations = true; } @@ -90,7 +156,7 @@ fn try_optimize_recursive(array: &ArrayRef) -> VortexResult> { for slot in current_array.slots() { match slot { Some(child) => { - if let Some(new_child) = try_optimize_recursive(child)? { + if let Some(new_child) = try_optimize_recursive(child, session)? { new_slots.push(Some(new_child)); any_slot_optimized = true; } else { diff --git a/vortex-duckdb/src/datasource.rs b/vortex-duckdb/src/datasource.rs index 1bff3ad6a3a..84bc82f2db3 100644 --- a/vortex-duckdb/src/datasource.rs +++ b/vortex-duckdb/src/datasource.rs @@ -399,7 +399,7 @@ impl TableFunction for T { return Ok(()); }; let (array_result, conversion_cache) = result?; - let array_result = array_result.optimize_recursive()?; + let array_result = array_result.optimize_recursive(ctx.session())?; let array_result: StructArray = if let Some(array) = array_result.as_opt::() { diff --git a/vortex-session/Cargo.toml b/vortex-session/Cargo.toml index 263c8c8500c..d4e047223c1 100644 --- a/vortex-session/Cargo.toml +++ b/vortex-session/Cargo.toml @@ -20,7 +20,6 @@ all-features = true workspace = true [dependencies] -arcref = { workspace = true } dashmap = { workspace = true } lasso = { workspace = true } parking_lot = { workspace = true } diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index a739b9fdda3..36c03828844 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -9,6 +9,7 @@ use std::fmt; use std::fmt::Debug; use std::fmt::Display; use std::fmt::Formatter; +use std::hash::Hash; use std::ops::Deref; use std::sync::Arc; use std::sync::LazyLock; diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 12b71709b72..ae803ee98ae 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -12,6 +12,7 @@ use vortex_array::dtype::session::DTypeSession; // vortex::expr is in the process of having its dependencies inverted, and will eventually be // pulled back out into a vortex_expr crate. pub use vortex_array::expr; +use vortex_array::optimizer::kernels::ArrayKernels; pub use vortex_array::scalar_fn; use vortex_array::scalar_fn::session::ScalarFnSession; use vortex_array::session::ArraySession; @@ -154,7 +155,8 @@ pub mod encodings { /// Extension trait to create a default Vortex session. pub trait VortexSessionDefault { - /// Creates a default Vortex session with the standard arrays, layouts, and expressions. + /// Creates a default Vortex session with standard arrays, layouts, scalar functions, + /// optimizer kernels, expressions, aggregate functions, and runtime support. fn default() -> VortexSession; } @@ -165,6 +167,7 @@ impl VortexSessionDefault for VortexSession { .with::() .with::() .with::() + .with::() .with::() .with::(); From f58b59a622378ac6ce7a495c4fc94f728b475835 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Mon, 27 Apr 2026 11:43:04 +0100 Subject: [PATCH 194/250] small metal benchmark runners (#7456) Use metal runners to try and reduce run variance. --- .github/workflows/nightly-bench.yml | 2 +- .github/workflows/sql-benchmarks.yml | 4 ++-- scripts/bench-taskset.sh | 11 ++++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index f2689bc70a4..b89395516be 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -72,4 +72,4 @@ jobs: matrix: machine_type: - id: x86 - instance_name: c6id.8xlarge + instance_name: i7i.metal-24xl diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index e2da59838e9..8dcb56bceda 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -9,7 +9,7 @@ on: machine_type: required: false type: string - default: c6id.8xlarge + default: i7i.metal-24xl benchmark_matrix: required: false type: string @@ -270,7 +270,7 @@ jobs: runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + && format('runs-on={0}/runner=bench-dedicated/instance-type={1}/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 diff --git a/scripts/bench-taskset.sh b/scripts/bench-taskset.sh index a86f6818ab3..4b8d362dd9c 100644 --- a/scripts/bench-taskset.sh +++ b/scripts/bench-taskset.sh @@ -18,12 +18,17 @@ fi if [[ -z "${BENCH_CPUS:-}" ]]; then - cpu_count="$(nproc)" - BENCH_CPUS="2-$((cpu_count - 1))" + if command -v numactl >/dev/null 2>&1; then + # All CPUs on NUMA node 0, skipping CPUs 0-1 to avoid OS interference + BENCH_CPUS=$(numactl --hardware | awk '/^node 0 cpus:/{sep=""; for(i=4;i<=NF;i++){if($i+0>1){printf "%s%s",sep,$i; sep=","}}}') + else + cpu_count="$(nproc)" + BENCH_CPUS="2-$((cpu_count - 1))" + fi fi if command -v numactl >/dev/null 2>&1; then - exec numactl --physcpubind="$BENCH_CPUS" --localalloc "$@" + exec numactl --physcpubind="$BENCH_CPUS" --membind=0 "$@" fi exec taskset -c "$BENCH_CPUS" "$@" From 06603536539fd96b0cba1525ecfdd4b2010910a2 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 27 Apr 2026 12:01:41 +0100 Subject: [PATCH 195/250] C API scan examples (#7564) - Add C scan api examples: dtype print, single-thread scan to arrow, multi-thread scan. - Remove vx_data_source_row_count in favour of vx_estimate. - Remove old C example. - Allow reading local relative paths in Datasource Signed-off-by: Mikhail Kot --- .github/workflows/rust-instrumented.yml | 19 ++- vortex-ffi/CMakeLists.txt | 20 ++- vortex-ffi/README.md | 72 ++++---- vortex-ffi/cbindgen.toml | 13 +- vortex-ffi/cinclude/vortex.h | 86 +++++----- vortex-ffi/examples/CMakeLists.txt | 18 ++ vortex-ffi/examples/Makefile | 75 --------- vortex-ffi/examples/README.md | 30 ---- vortex-ffi/examples/dtype.c | 156 ++++++++++++++++++ vortex-ffi/examples/hello-vortex.c | 116 ------------- vortex-ffi/examples/scan.c | 209 ++++++++++++++++++++++++ vortex-ffi/examples/scan_to_arrow.c | 126 ++++++++++++++ vortex-ffi/examples/write_sample.c | 143 ++++++++++++++++ vortex-ffi/src/data_source.rs | 42 ++--- vortex-ffi/src/scan.rs | 31 ++-- vortex-ffi/test/CMakeLists.txt | 9 - vortex-ffi/test/scan.cpp | 6 +- vortex-file/src/multi/mod.rs | 15 +- 18 files changed, 821 insertions(+), 365 deletions(-) create mode 100644 vortex-ffi/examples/CMakeLists.txt delete mode 100644 vortex-ffi/examples/Makefile delete mode 100644 vortex-ffi/examples/README.md create mode 100644 vortex-ffi/examples/dtype.c delete mode 100644 vortex-ffi/examples/hello-vortex.c create mode 100644 vortex-ffi/examples/scan.c create mode 100644 vortex-ffi/examples/scan_to_arrow.c create mode 100644 vortex-ffi/examples/write_sample.c diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index 3bcdad8af53..a2c0d0f842d 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -187,15 +187,28 @@ jobs: cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ --target x86_64-unknown-linux-gnu -Zbuild-std \ -p vortex-ffi - - name: Build FFI library tests + - name: Build FFI library tests and examples run: | cd vortex-ffi - cmake -Bbuild -DBUILD_TESTS=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" + cmake -Bbuild -DBUILD_TESTS=1 -DBUILD_EXAMPLES=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" cmake --build build -j - name: Run tests run: | set -o pipefail - ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt -i- + ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt + - name: Run examples + run: | + set -o pipefail + + # Failed to create data source: Object store error: Generic LocalFileSystem + # error: Unable to walk dir: File system loop found + rm -fr vortex-ffi/build/_deps/nanoarrow-src/python + + ./vortex-ffi/build/examples/write_sample file.vortex 2>&1 | rustfilt + ./vortex-ffi/build/examples/write_sample file2.vortex 2>&1 | rustfilt + ./vortex-ffi/build/examples/dtype '*.vortex' 2>&1 | rustfilt + ./vortex-ffi/build/examples/scan '*.vortex' 2>&1 | rustfilt + ./vortex-ffi/build/examples/scan_to_arrow '*.vortex' 2>&1 | rustfilt miri: name: "Rust tests (miri)" diff --git a/vortex-ffi/CMakeLists.txt b/vortex-ffi/CMakeLists.txt index dc6a231fc55..68107b0d244 100644 --- a/vortex-ffi/CMakeLists.txt +++ b/vortex-ffi/CMakeLists.txt @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors cmake_minimum_required(VERSION 3.10) +include(FetchContent) + project(VortexFFI VERSION 0.0.1 LANGUAGES C) @@ -10,6 +12,7 @@ set(CMAKE_C_STANDARD 17) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -Wextra -Wpedantic") option(BUILD_TESTS "Build tests" OFF) +option(BUILD_EXAMPLES "Build examples" OFF) set(SANITIZER "" CACHE STRING "Build with sanitizers") set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") @@ -76,10 +79,10 @@ Static library path ${LIBRARY_PATH} Headers path ${LIBRARY_HEADERS}") if (NOT EXISTS "${LIBRARY_PATH_SHARED}") - message(FATAL_ERROR "Shared library not found") + message(FATAL_ERROR "Shared library not found, run `cargo build --release -p vortex-ffi`") endif() if (NOT EXISTS "${LIBRARY_PATH}") - message(FATAL_ERROR "Static library not found") + message(FATAL_ERROR "Static library not found, run `cargo build --release -p vortex-ffi`") endif() add_library(vortex_ffi STATIC IMPORTED) @@ -95,6 +98,15 @@ set_target_properties(vortex_ffi_shared PROPERTIES INTERFACE_LINK_OPTIONS "LINKER:-rpath,${LIBRARY_DIR}" ) +if (BUILD_TESTS OR BUILD_EXAMPLES) + FetchContent_Declare( + Nanoarrow + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0 + ) + FetchContent_MakeAvailable(Nanoarrow) +endif() + if (BUILD_TESTS) enable_language(CXX) set(CMAKE_CXX_STANDARD 20) @@ -103,3 +115,7 @@ if (BUILD_TESTS) enable_testing() add_subdirectory(test) endif() + +if (BUILD_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/vortex-ffi/README.md b/vortex-ffi/README.md index 472bc89f71c..34df8cc2a1a 100644 --- a/vortex-ffi/README.md +++ b/vortex-ffi/README.md @@ -1,58 +1,50 @@ -# Foreign Function Interface +# Vortex C interface +## Usage from a CMake project -Vortex is a file format that can be used by any execution engine. Nearly every programming language supports -the C ABI (Application Binary Interface), so by providing an FFI interface to work with Vortex objects we can -make it easy to support a variety of languages. - -Check out the [`examples`](./examples/) directory to see an example of how to use the API to build -a real native application. - -## Design - -The FFI is designed to be very simple and follows a very object-oriented approach: - -- **Constructors** are simple C functions that return opaque pointers -- **Methods** are functions that receive an opaque pointer as the first argument, followed by subsequent arguments. - Methods may return a value or void. -- **Destructors** free native resources (allocations, file handles, network sockets) and must be explicitly called by - the foreign language to avoid leaking resources. - -Constructors will generally allocate rust memory, and destructors free that memory. - -## Documentation - -The FFI API is documented in `docs/api/c` with explicit inclusion of types, enums, and functions, etc. Note that an -item cannot be referenced in the documentation if it does not have a documentation comment. +``` +# in vortex folder +cargo build --release -p vortex-ffi -## Updating Headers +# in your CMakeLists.txt +include_directory(vortex/vortex-ffi) +target_link_libraries(my_target, vortex_ffi_shared) +# or target_link_libraries(my_target, vortex_ffi) +``` -To rebuild the header file: +## Running C examples: ```sh -cargo +nightly build -p vortex-ffi +cmake -Bbuild -DBUILD_EXAMPLES=1 +cmake --build build +./build/examples/dtype +./build/examples/scan +./build/examples/scan_to_arrow +./build/examples/write_sample ``` -The header generation uses cbindgen's macro expansion feature which requires nightly. -Stable builds use the checked-in header file at `cinclude/vortex.h`. +## Updating Headers + +If you're developing FFI and want to rebuild `cinclude/vortex.h`, run +`cargo +nightly build -p vortex-ffi`. -### Testing C part +## Testing C part -Build the test library +Build the test library: ```sh -cmake -Bbuild -cmake --build build -j $(nproc) +cmake -Bbuild -DBUILD_TESTS=1 +cmake --build build ``` -Run the tests +Run the tests: ```sh ctest --test-dir build -j $(nproc) ``` -You would need C++ compiler toolchain to run the tests since they use Catch2. +You will need C++ compiler toolchain to run the tests since they use Catch2. -### Testing Rust part with sanitizers +## Testing Rust part with sanitizers AddressSanitizer: @@ -90,20 +82,20 @@ with sanitizers. - `allow-abi-mismatch` is safe because in our dependency graph only crates like `compiler_builtins` unset sanitization, and they do it on purpose. - Make sure to use `cargo test` and not `cargo nextest` as nextest reports less -leaks. + leaks. - If you want stack trace symbolization, install `llvm-symbolizer`. -### Testing Rust and C with sanitizers +## Testing Rust and C with sanitizers 1. Build FFI library with external sanitizer runtime: ```sh RUSTFLAGS="-Zsanitizer=address -Zexternal-clangrt" \ cargo +nightly build -Zbuild-std --target= \ ---no-default-features -p vortex-ffi + --no-default-features -p vortex-ffi ``` -2. Build tests with target triple +2. Build tests with target triple: ```sh cmake -Bbuild -DWITH_ASAN=1 -DTARGET_TRIPLE= diff --git a/vortex-ffi/cbindgen.toml b/vortex-ffi/cbindgen.toml index 5153eb0f9fc..46d80cd72c7 100644 --- a/vortex-ffi/cbindgen.toml +++ b/vortex-ffi/cbindgen.toml @@ -16,10 +16,15 @@ header = """ // // https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions -// We don't want to bundle nanoarrow or similar just for these two definitions. -// If you use your own Arrow library, define this macro and -// typedef FFI_ArrowSchema ArrowSchema; -// typedef FFI_ArrowArrayStream ArrowArrayStream; +// If you want to use your own Arrow library like nanoarrow, define this macro +// and typedef your types: +// +// #include "nanoarrow/common/inline_types.h" +// #define USE_OWN_ARROW +// typedef struct ArrowSchema FFI_ArrowSchema; +// typedef struct ArrowArrayStream FFI_ArrowArrayStream; +// #include "vortex.h" +// #ifndef USE_OWN_ARROW struct ArrowSchema { const char* format; diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index bb4495b34de..ae152752ba2 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -8,10 +8,15 @@ // // https://arrow.apache.org/docs/format/CDataInterface.html#structure-definitions -// We don't want to bundle nanoarrow or similar just for these two definitions. -// If you use your own Arrow library, define this macro and -// typedef FFI_ArrowSchema ArrowSchema; -// typedef FFI_ArrowArrayStream ArrowArrayStream; +// If you want to use your own Arrow library like nanoarrow, define this macro +// and typedef your types: +// +// #include "nanoarrow/common/inline_types.h" +// #define USE_OWN_ARROW +// typedef struct ArrowSchema FFI_ArrowSchema; +// typedef struct ArrowArrayStream FFI_ArrowArrayStream; +// #include "vortex.h" +// #ifndef USE_OWN_ARROW struct ArrowSchema { const char *format; @@ -175,10 +180,19 @@ typedef enum { } vx_validity_type; typedef enum { - VX_CARD_UNKNOWN = 0, - VX_CARD_ESTIMATE = 1, - VX_CARD_MAXIMUM = 2, -} vx_cardinality; + /** + * No estimate is available. + */ + VX_ESTIMATE_UNKNOWN = 0, + /** + * The value in vx_estimate.estimate is exact. + */ + VX_ESTIMATE_EXACT = 1, + /** + * The value in vx_estimate.estimate is an upper bound. + */ + VX_ESTIMATE_INEXACT = 2, +} vx_estimate_type; /** * Equalities, inequalities, and boolean operations over possibly null values. @@ -282,21 +296,6 @@ typedef enum { VX_SELECTION_EXCLUDE_RANGE = 2, } vx_scan_selection_include; -typedef enum { - /** - * No estimate is available. - */ - VX_ESTIMATE_UNKNOWN = 0, - /** - * The value in vx_estimate.estimate is exact. - */ - VX_ESTIMATE_EXACT = 1, - /** - * The value in vx_estimate.estimate is an upper bound. - */ - VX_ESTIMATE_INEXACT = 2, -} vx_estimate_type; - /** * Physical type enum, represents the in-memory physical layout but might represent a different logical type. */ @@ -490,6 +489,10 @@ typedef struct vx_file vx_file; */ typedef struct vx_partition vx_partition; +/** + * A scan is a single traversal of a data source with projections and + * filters. A scan can be consumed only once. + */ typedef struct vx_scan vx_scan; /** @@ -537,13 +540,17 @@ typedef struct { const char *paths; } vx_data_source_options; +/** + * Used for estimating number of partitions in a data source or number of rows + * in a partition. + */ typedef struct { - vx_cardinality cardinality; + vx_estimate_type type; /** - * Set only when "cardinality" is not VX_CARD_UNKNOWN + * Set only when "type" is not VX_ESTIMATE_UNKNOWN. */ - uint64_t rows; -} vx_data_source_row_count; + uint64_t estimate; +} vx_estimate; /** * Options supplied for opening a file. @@ -662,18 +669,6 @@ typedef struct { bool ordered; } vx_scan_options; -/** - * Used for estimating number of partitions in a data source or number of rows - * in a partition. - */ -typedef struct { - vx_estimate_type type; - /** - * Set only when "type" is not VX_ESTIMATE_UNKNOWN. - */ - uint64_t estimate; -} vx_estimate; - #ifdef __cplusplus extern "C" { #endif // __cplusplus @@ -921,7 +916,7 @@ const vx_dtype *vx_data_source_dtype(const vx_data_source *ds); /** * Write data source's row count estimate into "row_count". */ -void vx_data_source_get_row_count(const vx_data_source *ds, vx_data_source_row_count *row_count); +void vx_data_source_get_row_count(const vx_data_source *ds, vx_estimate *row_count); /** * Clone a borrowed [`vx_dtype`], returning an owned [`vx_dtype`]. @@ -1319,6 +1314,17 @@ vx_partition *vx_scan_next_partition(vx_scan *scan, vx_error **err); */ int vx_partition_row_count(const vx_partition *partition, vx_estimate *count, vx_error **err); +/** + * Scan partition to ArrowArrayStream. + * Consumes partition fully: subsequent calls to vx_partition_scan_arrow or + * vx_partition_next are undefined behaviour. + * This call blocks current thread until underlying stream is fully consumed. + * + * Caller must not free partition after calling this function. + * + * On success, sets "stream" and returns 0. + * On error, sets "err" and returns 1, freeing the partition. + */ int vx_partition_scan_arrow(const vx_session *session, vx_partition *partition, FFI_ArrowArrayStream *stream, diff --git a/vortex-ffi/examples/CMakeLists.txt b/vortex-ffi/examples/CMakeLists.txt new file mode 100644 index 00000000000..47228d9a903 --- /dev/null +++ b/vortex-ffi/examples/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: CC-BY-4.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# allow linking with vortex_ffi_shared although it's not in current folder +cmake_policy(SET CMP0079 NEW) + +add_executable(scan scan.c) +target_link_libraries(scan PRIVATE vortex_ffi_shared) + +add_executable(scan_to_arrow scan_to_arrow.c) +target_link_libraries(scan_to_arrow PRIVATE + nanoarrow_shared vortex_ffi_shared) + +add_executable(dtype dtype.c) +target_link_libraries(dtype PRIVATE vortex_ffi_shared) + +add_executable(write_sample write_sample.c) +target_link_libraries(write_sample PRIVATE vortex_ffi_shared) diff --git a/vortex-ffi/examples/Makefile b/vortex-ffi/examples/Makefile deleted file mode 100644 index c22056524fd..00000000000 --- a/vortex-ffi/examples/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: CC-BY-4.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -# Directory containing the Rust crate -ROOT_DIR = ../.. -RUST_CRATE_DIR = .. -# Directory where cargo places the built libraries -RUST_TARGET_DIR = $(ROOT_DIR)/target -# The library name (without lib prefix and .so/.dylib suffix) -LIBRARY_NAME = vortex_ffi - -# Determine platform-specific library extension -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - LIB_EXT = so -else ifeq ($(UNAME_S),Darwin) - LIB_EXT = dylib -else - # Default to .dll for Windows - LIB_EXT = dll -endif - -# Path to the library depending on build mode -DEBUG_LIB = $(RUST_TARGET_DIR)/debug/lib$(LIBRARY_NAME).$(LIB_EXT) -RELEASE_LIB = $(RUST_TARGET_DIR)/release/lib$(LIBRARY_NAME).$(LIB_EXT) - -# Default to debug mode -LIB = $(DEBUG_LIB) - -# C flags -CFLAGS = -Wall -I$(RUST_CRATE_DIR)/cinclude - -# Linking flags -LDFLAGS = -L$(RUST_TARGET_DIR)/debug -l$(LIBRARY_NAME) - -# Final executable name -EXECUTABLE = hello_vortex - -.PHONY: all clean run release debug - -# Default target -all: debug - -# Debug build -debug: LDFLAGS = -L$(RUST_TARGET_DIR)/debug -l$(LIBRARY_NAME) -debug: build - -# Release build -release: LDFLAGS = -L$(RUST_TARGET_DIR)/release -l$(LIBRARY_NAME) -release: LIB = $(RELEASE_LIB) -release: build - -# Build rule -build: $(LIB) $(EXECUTABLE) - -# Build the Rust library -$(DEBUG_LIB): - cd $(RUST_CRATE_DIR) && cargo build - -$(RELEASE_LIB): - cd $(RUST_CRATE_DIR) && cargo build --release - -# Build the C executable -$(EXECUTABLE): hello-vortex.c - $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) - -# Run the executable with the correct library path -run: build - @echo "Running example..." - LD_LIBRARY_PATH=$(dir $(LIB)) ./$(EXECUTABLE) - -# Clean build artifacts -clean: - rm -f $(EXECUTABLE) - cd $(RUST_CRATE_DIR) && cargo clean diff --git a/vortex-ffi/examples/README.md b/vortex-ffi/examples/README.md deleted file mode 100644 index 5ebc99f9bfe..00000000000 --- a/vortex-ffi/examples/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# C FFI example - -This example shows how to interface with the FFI of this crate using C code. - -Run `make` to build the `hello_vortex` binary. - -The binary expects a single argument, which is the path to a Vortex file. A new streaming -scan will be created that will materialize file splits as Vortex in-memory arrays, and print -some information about them. - -Here's an example from the TPC-H `partsupp` dataset: - -``` -$ make run file:///tmp/partsupp.vortex -Scanning file: file:///tmp/partsupp.vortex -Chunk 0: 65536 -Chunk 1: 65536 -Chunk 2: 65536 -Chunk 3: 65536 -Chunk 4: 65536 -Chunk 5: 65536 -Chunk 6: 65536 -Chunk 7: 65536 -Chunk 8: 65536 -Chunk 9: 65536 -Chunk 10: 65536 -Chunk 11: 65536 -Chunk 12: 13568 -Scanning complete -``` diff --git a/vortex-ffi/examples/dtype.c b/vortex-ffi/examples/dtype.c new file mode 100644 index 00000000000..007c3993e33 --- /dev/null +++ b/vortex-ffi/examples/dtype.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex.h" +#include + +const char *usage = "Print dtype of files\n" + "Usage: dtype \n"; + +void print_dtype(const vx_dtype *dtype); + +void print_ptype(const vx_dtype *type) { + const char *ptype = NULL; + + switch (vx_dtype_primitive_ptype(type)) { + case PTYPE_U8: + ptype = "uint8_t"; + break; + case PTYPE_U16: + ptype = "uint16_t"; + break; + case PTYPE_U32: + ptype = "uint32_t"; + break; + case PTYPE_U64: + ptype = "uint64_t"; + break; + case PTYPE_I8: + ptype = "int8_t"; + break; + case PTYPE_I16: + ptype = "int16_t"; + break; + case PTYPE_I32: + ptype = "int32_t"; + break; + case PTYPE_I64: + ptype = "int64_t"; + break; + case PTYPE_F16: + ptype = "float16"; + break; + case PTYPE_F32: + ptype = "float"; + break; + case PTYPE_F64: + ptype = "double"; + break; + default: + __builtin_unreachable(); + } + + printf("primitive(%s)", ptype); +} + +void print_struct_dtype(const vx_dtype *dtype) { + const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); + + printf("struct(\n"); + for (uint64_t i = 0; i < vx_struct_fields_nfields(fields); ++i) { + const vx_dtype *field_dtype = vx_struct_fields_field_dtype(fields, i); + const vx_string *field_name = vx_struct_fields_field_name(fields, i); + printf(" %.*s = ", (int)vx_string_len(field_name), vx_string_ptr(field_name)); + print_dtype(field_dtype); + vx_dtype_free(field_dtype); + } + printf(")"); +} + +void print_list_dtype(const vx_dtype *dtype) { + printf("list("); + print_dtype(vx_dtype_list_element(dtype)); + printf(")"); +} + +void print_fixed_list_dtype(const vx_dtype *dtype) { + printf("fixed list(size=%d, ", vx_dtype_fixed_size_list_size(dtype)); + print_dtype(vx_dtype_fixed_size_list_element(dtype)); + printf(")"); +} + +void print_decimal_dtype(const vx_dtype *dtype) { + const uint8_t precision = vx_dtype_decimal_precision(dtype); + const int8_t scale = vx_dtype_decimal_scale(dtype); + printf("decimal(precision=%u, scale=%d)", precision, scale); +} + +void print_dtype(const vx_dtype *dtype) { + switch (vx_dtype_get_variant(dtype)) { + case DTYPE_NULL: + printf("null"); + break; + case DTYPE_BOOL: + printf("bool"); + break; + case DTYPE_UTF8: + printf("utf8"); + break; + case DTYPE_BINARY: + printf("binary"); + break; + case DTYPE_EXTENSION: + printf("extension"); + break; + case DTYPE_PRIMITIVE: + print_ptype(dtype); + break; + case DTYPE_STRUCT: + print_struct_dtype(dtype); + break; + case DTYPE_LIST: + print_list_dtype(dtype); + break; + case DTYPE_FIXED_SIZE_LIST: + print_fixed_list_dtype(dtype); + break; + case DTYPE_DECIMAL: + print_decimal_dtype(dtype); + break; + } + printf("%c\n", vx_dtype_is_nullable(dtype) ? '?' : ' '); +} + +void print_error(const char *what, const vx_error *error) { + const vx_string *str = vx_error_get_message(error); + fprintf(stderr, "%s: %.*s\n", what, (int)vx_string_len(str), vx_string_ptr(str)); +} + +int main(int argc, char **argv) { + if (argc != 2) { + fprintf(stderr, "%s", usage); + return 1; + } + + vx_error *error = NULL; + vx_session *const session = vx_session_new(); + if (session == NULL) { + fprintf(stderr, "Failed to create Vortex session\n"); + return 1; + } + + vx_data_source_options ds_options = {.paths = argv[1]}; + const vx_data_source *data_source = vx_data_source_new(session, &ds_options, &error); + if (data_source == NULL) { + print_error("Failed to create data source", error); + vx_error_free(error); + vx_session_free(session); + return 1; + } + + printf("dtype: "); + print_dtype(vx_data_source_dtype(data_source)); + + vx_data_source_free(data_source); + vx_session_free(session); + return 0; +} diff --git a/vortex-ffi/examples/hello-vortex.c b/vortex-ffi/examples/hello-vortex.c deleted file mode 100644 index 192e4d73ae3..00000000000 --- a/vortex-ffi/examples/hello-vortex.c +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: CC-BY-4.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex.h" -#include -#include -#include - -int main(int argc, char *argv[]) { - if (argc < 2) { - printf("Usage: %s \n", argv[0]); - return 1; - } - - vx_error *error = NULL; - vx_session *session = vx_session_new(); - if (session == NULL) { - fprintf(stderr, "Failed to create Vortex session\n"); - return -1; - } - - // Open the file - char *uri = argv[1]; - printf("Opening file: %s\n", uri); - - vx_file_open_options open_opts = { - .uri = uri, - .property_keys = NULL, - .property_vals = NULL, - .property_len = 0, - }; - - const vx_file *file = vx_file_open_reader(session, &open_opts, &error); - if (error != NULL) { - fprintf(stderr, "Failed to open file: %s\n%s", uri, vx_string_ptr(vx_error_get_message(error))); - vx_error_free(error); - vx_session_free(session); - return -1; - } - - // Print file metadata - this will satisfy the "File contains" check - uint64_t row_count = vx_file_row_count(file); - printf("File contains %llu total rows\n", (unsigned long long)row_count); - - // Get and display file dtype - const vx_dtype *file_dtype = vx_file_dtype(file); - vx_dtype_variant variant = vx_dtype_get_variant(file_dtype); - bool nullable = vx_dtype_is_nullable(file_dtype); - printf("File DType variant: %d, nullable: %s\n", variant, - nullable ? "true" : "false"); - - // Start scanning - printf("\nScanning file...\n"); - vx_array_iterator *scan = vx_file_scan(session, file, NULL, &error); - if (error != NULL) { - fprintf(stderr, "Failed to create file scan iterator\n"); - vx_error_free(error); - vx_file_free(file); - vx_session_free(session); - return -1; - } - - int chunk_count = 0; - const vx_array *batch = vx_array_iterator_next(scan, &error); - - while (batch != NULL && error == NULL) { - size_t batch_len = vx_array_len(batch); - printf("Chunk %d: %zu rows\n", chunk_count, batch_len); - - // For the first chunk, show additional API coverage including struct introspection - if (chunk_count == 0 && batch_len > 0) { - const vx_dtype *dtype = vx_array_dtype(batch); - vx_dtype_variant batch_variant = vx_dtype_get_variant(dtype); - printf(" First chunk DType variant: %d\n", batch_variant); - - // Test null count API - uint32_t null_count = vx_array_null_count(batch, &error); - if (error == NULL) { - printf(" Null count: %u\n", null_count); - } else { - printf(" Null count check failed (expected for some array types)\n"); - vx_error_free(error); - error = NULL; - } - - // Test struct field count if it's a struct - if (batch_variant == DTYPE_STRUCT) { - const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); - size_t n_fields = vx_struct_fields_nfields(fields); - printf(" Struct with %zu fields\n", n_fields); - } - } - - vx_array_free(batch); - batch = vx_array_iterator_next(scan, &error); - chunk_count++; - } - - printf("Total chunks processed: %d\n", chunk_count); - - // Clean up resources - vx_array_iterator_free(scan); - vx_file_free(file); - - if (error != NULL) { - fprintf(stderr, "Error during scan operation\n"); - vx_error_free(error); - vx_session_free(session); - return -1; - } - - printf("Scanning completed successfully\n"); - vx_session_free(session); - - return 0; -} diff --git a/vortex-ffi/examples/scan.c b/vortex-ffi/examples/scan.c new file mode 100644 index 00000000000..65479df8f40 --- /dev/null +++ b/vortex-ffi/examples/scan.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex.h" +#include +#include +#include + +#define MAX_THREADS 64 + +const char *usage = "Multi-threaded file scan\n" + "Usage: scan [-j threads] \n"; + +void print_estimate(const char *what, const vx_estimate *estimate) { + switch (estimate->type) { + case VX_ESTIMATE_UNKNOWN: + printf("%s: unknown\n", what); + return; + case VX_ESTIMATE_EXACT: + printf("%s: %lu\n", what, estimate->estimate); + return; + case VX_ESTIMATE_INEXACT: + printf("%s: approximately %lu\n", what, estimate->estimate); + break; + } +} + +void print_error(const char *what, const vx_error *error) { + const vx_string *str = vx_error_get_message(error); + fprintf(stderr, "%s: %.*s\n", what, (int)vx_string_len(str), vx_string_ptr(str)); +} + +struct scan_thread_info { + pthread_t thread_id; + pthread_mutex_t *mutex; + vx_scan *scan; + size_t partitions, arrays, rows; + vx_error *error; +}; + +void *execute_scan_thread(void *arg) { + struct scan_thread_info *info = arg; + while (true) { + // A partition is an independent unit of work a thread can work on. + pthread_mutex_lock(info->mutex); + vx_partition *partition = vx_scan_next_partition(info->scan, &info->error); + pthread_mutex_unlock(info->mutex); + + if (partition == NULL && info->error == NULL) { + break; // partition iterator exhausted + } + if (partition == NULL && info->error != NULL) { + return NULL; // partition was not scanned due to an error + } + ++info->partitions; + + vx_estimate row_count; + if (vx_partition_row_count(partition, &row_count, &info->error)) { + vx_partition_free(partition); + return NULL; + } + + printf("Thread %lu processing partition %lu, ", info->thread_id + 1, info->partitions); + print_estimate("row count", &row_count); + + // An array is a batch of rows from a partition + const vx_array *array = NULL; + while ((array = vx_partition_next(partition, &info->error)) != NULL) { + ++info->arrays; + info->rows += vx_array_len(array); + vx_array_free(array); + } + + vx_partition_free(partition); + + if (info->error != NULL) { + return NULL; + } + } + + printf("Thread %lu finished, processed %lu partitions, %lu arrays, %lu rows\n", + info->thread_id + 1, + info->partitions, + info->arrays, + info->rows); + return NULL; +} + +vx_error *execute_scan(vx_scan *scan, pthread_t num_threads) { + pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + pthread_t threads[MAX_THREADS]; + struct scan_thread_info infos[MAX_THREADS] = {0}; + + printf("Starting scan, using %lu threads\n", num_threads); + for (pthread_t id = 0; id < num_threads; ++id) { + struct scan_thread_info *info = &infos[id]; + info->thread_id = id; + info->mutex = &mutex; + info->scan = scan; + pthread_create(&threads[id], NULL, execute_scan_thread, info); + } + + size_t partitions = 0, arrays = 0, rows = 0; + for (pthread_t id = 0; id < num_threads; ++id) { + pthread_join(threads[id], NULL); + struct scan_thread_info *info = &infos[id]; + + if (info->error != NULL) { + // Don't join other threads as program will be terminated early + return info->error; + } + + partitions += info->partitions; + arrays += info->arrays; + rows += info->rows; + } + + printf("Finished scan, processed %lu partitions, %lu arrays, %lu rows\n", partitions, arrays, rows); + return NULL; +} + +int parse_options(int argc, char *argv[], pthread_t *threads, char **paths) { + int opt; + while ((opt = getopt(argc, argv, "j:")) != -1) { + switch (opt) { + case 'j': + *threads = atoi(optarg); + break; + default: + fprintf(stderr, "%s", usage); + return 1; + } + } + + if (*threads != 0 && (*threads < 1 || *threads > MAX_THREADS)) { + fprintf(stderr, "Invalid thread count %lu, expected [1; 64]\n", *threads); + return 1; + } + + if (optind + 1 != argc) { + fprintf(stderr, "%s", usage); + return 1; + } + + *paths = argv[optind]; + return 0; +} + +int main(int argc, char *argv[]) { + pthread_t threads = 0; + char *paths; + if (parse_options(argc, argv, &threads, &paths)) { + return 1; + } + + vx_session *session = vx_session_new(); + if (session == NULL) { + fprintf(stderr, "Failed to create Vortex session\n"); + return 1; + } + + printf("Opening files: %s\n", paths); + + // A datasource is a reference to some files. + // We can request multiple scans from a data source. + vx_data_source_options ds_options = {.paths = paths}; + vx_error *error = NULL; + const vx_data_source *data_source = vx_data_source_new(session, &ds_options, &error); + if (data_source == NULL) { + print_error("Failed to create data source", error); + // Returned errors are owned and need to be freed + vx_error_free(error); + vx_session_free(session); + return 1; + } + + vx_estimate row_count; + vx_data_source_get_row_count(data_source, &row_count); + print_estimate("Data source row count", &row_count); + + // A scan is a single traversal of a data source. + // Here we request a scan without any filters, projections, or limiting. + vx_scan_options scan_options = {.max_threads = threads}; + vx_estimate partition_estimate; + vx_scan *scan = vx_data_source_scan(data_source, &scan_options, &partition_estimate, &error); + if (scan == NULL) { + print_error("Failed to create scan", error); + vx_error_free(error); + vx_data_source_free(data_source); + vx_session_free(session); + return 1; + } + + // Caller can use partition estimates to schedule worker threads. + print_estimate("Partition count", &partition_estimate); + if (threads == 0) { + threads = partition_estimate.type == VX_ESTIMATE_UNKNOWN ? 1 : partition_estimate.estimate; + } + + error = execute_scan(scan, threads); + if (error != NULL) { + print_error("Failed to scan", error); + vx_error_free(error); + } + + vx_scan_free(scan); + vx_data_source_free(data_source); + vx_session_free(session); + return 0; +} diff --git a/vortex-ffi/examples/scan_to_arrow.c b/vortex-ffi/examples/scan_to_arrow.c new file mode 100644 index 00000000000..7085a8f39b8 --- /dev/null +++ b/vortex-ffi/examples/scan_to_arrow.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "nanoarrow/common/inline_types.h" +#include "nanoarrow/nanoarrow.h" + +#define USE_OWN_ARROW +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#include "vortex.h" + +#include +#include + +const char *usage = "Scan vortex files to Arrow\n" + "Usage: scan_to_arrow \n"; + +void print_error(const char *what, const vx_error *error) { + const vx_string *str = vx_error_get_message(error); + fprintf(stderr, "%s: %.*s\n", what, (int)vx_string_len(str), vx_string_ptr(str)); +} + +void execute_scan(vx_session *session, vx_scan *scan) { + vx_error *error = NULL; + + // Returned dtype is owned and mustn't be freed + const vx_dtype *dtype = vx_scan_dtype(scan, &error); + if (dtype == NULL) { + print_error("Failed to get scan dtype", error); + vx_error_free(error); + return; + } + + struct ArrowSchema schema; + if (vx_dtype_to_arrow_schema(dtype, &schema, &error)) { + print_error("Failed to convert dtype to Arrow schema", error); + vx_error_free(error); + return; + } + + char schema_buf[1024 * 10]; + const int schema_len = ArrowSchemaToString(&schema, schema_buf, sizeof schema_buf, 1); + printf("arrow schema: %.*s\n", schema_len, schema_buf); + if (schema.release) { + schema.release(&schema); + } + + struct ArrowError arrow_error; + ArrowErrorInit(&arrow_error); + + vx_partition *partition; + size_t partitions = 0, arrays = 0, rows = 0; + + while ((partition = vx_scan_next_partition(scan, &error)) != NULL) { + struct ArrowArrayStream stream; + // Partition is consumed, we must not free it or use it after + if (vx_partition_scan_arrow(session, partition, &stream, &error)) { + print_error("Failed to scan partition to Arrow", error); + vx_error_free(error); + error = NULL; + break; + } + + struct ArrowArray array = {0}; + while (ArrowArrayStreamGetNext(&stream, &array, &arrow_error) == NANOARROW_OK && + array.release != NULL) { + rows += array.length; + ++arrays; + array.release(&array); + memset(&array, 0, sizeof(array)); + } + + printf("Read Partition %lu to arrow, %lu arrays, %lu rows\n", partitions, arrays, rows); + rows = 0; + arrays = 0; + + stream.release(&stream); + ++partitions; + } + + if (error) { + print_error("Failed scanning partition", error); + vx_error_free(error); + } +} + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "%s", usage); + return 1; + } + const char *paths = argv[1]; + + vx_session *session = vx_session_new(); + if (session == NULL) { + fprintf(stderr, "Failed to create Vortex session\n"); + return -1; + } + + vx_data_source_options ds_options = {.paths = paths}; + vx_error *error = NULL; + const vx_data_source *data_source = vx_data_source_new(session, &ds_options, &error); + if (data_source == NULL) { + print_error("Failed to create data source", error); + // Returned errors are owned and need to be freed + vx_error_free(error); + vx_session_free(session); + return 1; + } + + vx_scan *scan = vx_data_source_scan(data_source, NULL, NULL, &error); + if (scan == NULL) { + print_error("Failed to create scan", error); + vx_error_free(error); + vx_data_source_free(data_source); + vx_session_free(session); + return 1; + } + + execute_scan(session, scan); + + vx_scan_free(scan); + vx_data_source_free(data_source); + vx_session_free(session); + + return 0; +} diff --git a/vortex-ffi/examples/write_sample.c b/vortex-ffi/examples/write_sample.c new file mode 100644 index 00000000000..e807b21b5a9 --- /dev/null +++ b/vortex-ffi/examples/write_sample.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex.h" +#include +#include +#include + +#define SAMPLE_ROWS 200 + +const char *usage = "Write a sample 200 rows .vortex file\n" + "Usage: write_sample \n"; + +// StructArray { age=u8, height=u16? } +const vx_dtype *sample_dtype(void) { + vx_struct_fields_builder *builder = vx_struct_fields_builder_new(); + + const char *age = "age"; + const vx_string *age_name = vx_string_new(age, strlen(age)); + const vx_dtype *age_type = vx_dtype_new_primitive(PTYPE_U8, false); + vx_struct_fields_builder_add_field(builder, age_name, age_type); + + const char *height = "height"; + const vx_string *height_name = vx_string_new(height, strlen(height)); + const vx_dtype *height_type = vx_dtype_new_primitive(PTYPE_U16, true); + vx_struct_fields_builder_add_field(builder, height_name, height_type); + + vx_struct_fields *fields = vx_struct_fields_builder_finalize(builder); + return vx_dtype_new_struct(fields, false); +} + +void print_error(const char *what, const vx_error *error) { + const vx_string *str = vx_error_get_message(error); + fprintf(stderr, "%s: %.*s\n", what, (int)vx_string_len(str), vx_string_ptr(str)); +} + +const vx_array *sample_array(void) { + vx_validity validity = {.type = VX_VALIDITY_NON_NULLABLE}; + vx_struct_column_builder *builder = vx_struct_column_builder_new(&validity, SAMPLE_ROWS); + + uint8_t age_buffer[SAMPLE_ROWS]; + uint16_t height_buffer[SAMPLE_ROWS]; + for (uint8_t i = 0; i < SAMPLE_ROWS; ++i) { + age_buffer[i] = i; + height_buffer[i] = rand() % (i + 1); + } + + vx_error *error = NULL; + const vx_array *age_array = vx_array_new_primitive(PTYPE_U8, age_buffer, SAMPLE_ROWS, &validity, &error); + if (error != NULL) { + print_error("Error creating age array", error); + vx_error_free(error); + vx_struct_column_builder_free(builder); + return NULL; + } + + vx_struct_column_builder_add_field(builder, "age", age_array, &error); + vx_array_free(age_array); + if (error != NULL) { + print_error("Error adding age array field to root array", error); + vx_error_free(error); + vx_struct_column_builder_free(builder); + return NULL; + } + + validity.type = VX_VALIDITY_ALL_VALID; + const vx_array *height_array = + vx_array_new_primitive(PTYPE_U16, height_buffer, SAMPLE_ROWS, &validity, &error); + if (error != NULL) { + print_error("Error adding height array field to root array", error); + vx_error_free(error); + vx_struct_column_builder_free(builder); + return NULL; + } + + vx_struct_column_builder_add_field(builder, "height", height_array, &error); + vx_array_free(height_array); + if (error != NULL) { + print_error("Error adding height array field to root array", error); + vx_error_free(error); + vx_struct_column_builder_free(builder); + return NULL; + } + + const vx_array *array = vx_struct_column_builder_finalize(builder, &error); + if (error != NULL) { + print_error("Error creating struct array", error); + vx_error_free(error); + return NULL; + } + + return array; +} + +int main(int argc, char *argv[]) { + if (argc != 2) { + fprintf(stderr, "%s", usage); + return 1; + } + const char *output = argv[1]; + + vx_session *const session = vx_session_new(); + if (session == NULL) { + fprintf(stderr, "Failed to create Vortex session\n"); + return 1; + } + + const vx_dtype *dtype = sample_dtype(); + + vx_error *error = NULL; + vx_array_sink *sink = vx_array_sink_open_file(session, output, dtype, &error); + + vx_dtype_free(dtype); + if (error != NULL) { + vx_session_free(session); + return 1; + } + + const vx_array *array = sample_array(); + if (array == NULL) { + // We already have an error, so we can ignore a potential error + // from this operation + vx_array_sink_close(sink, &error); + vx_session_free(session); + return 1; + } + + vx_array_sink_push(sink, array, &error); + if (error != NULL) { + vx_array_sink_close(sink, &error); + vx_session_free(session); + return 1; + } + vx_array_free(array); + + vx_array_sink_close(sink, &error); + if (error != NULL) { + print_error("Error closing output sink", error); + vx_error_free(error); + } + + vx_session_free(session); + return 0; +} diff --git a/vortex-ffi/src/data_source.rs b/vortex-ffi/src/data_source.rs index 2055e697209..33ebe937b3a 100644 --- a/vortex-ffi/src/data_source.rs +++ b/vortex-ffi/src/data_source.rs @@ -20,6 +20,8 @@ use crate::RUNTIME; use crate::dtype::vx_dtype; use crate::error::try_or; use crate::error::vx_error; +use crate::scan::vx_estimate; +use crate::scan::vx_estimate_type; use crate::session::vx_session; use crate::to_string; @@ -106,39 +108,24 @@ pub unsafe extern "C-unwind" fn vx_data_source_dtype(ds: *const vx_data_source) vx_dtype::new_ref(vx_data_source::as_ref(ds).dtype()) } -#[repr(C)] -#[cfg_attr(test, derive(PartialEq, Debug))] -enum vx_cardinality { - VX_CARD_UNKNOWN = 0, - VX_CARD_ESTIMATE = 1, - VX_CARD_MAXIMUM = 2, -} - -#[repr(C)] -pub struct vx_data_source_row_count { - cardinality: vx_cardinality, - /// Set only when "cardinality" is not VX_CARD_UNKNOWN - rows: u64, -} - /// Write data source's row count estimate into "row_count". #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_data_source_get_row_count( ds: *const vx_data_source, - row_count: *mut vx_data_source_row_count, + row_count: *mut vx_estimate, ) { let rc = unsafe { &mut *row_count }; match vx_data_source::as_ref(ds).row_count() { Some(Exact(rows)) => { - rc.cardinality = vx_cardinality::VX_CARD_MAXIMUM; - rc.rows = rows; + rc.r#type = vx_estimate_type::VX_ESTIMATE_EXACT; + rc.estimate = rows; } Some(Inexact(rows)) => { - rc.cardinality = vx_cardinality::VX_CARD_ESTIMATE; - rc.rows = rows; + rc.r#type = vx_estimate_type::VX_ESTIMATE_INEXACT; + rc.estimate = rows; } None => { - rc.cardinality = vx_cardinality::VX_CARD_UNKNOWN; + rc.r#type = vx_estimate_type::VX_ESTIMATE_UNKNOWN; } } } @@ -152,14 +139,14 @@ mod tests { use std::ffi::CString; use std::ptr; - use crate::data_source::vx_cardinality; use crate::data_source::vx_data_source_dtype; use crate::data_source::vx_data_source_free; use crate::data_source::vx_data_source_get_row_count; use crate::data_source::vx_data_source_new; use crate::data_source::vx_data_source_options; - use crate::data_source::vx_data_source_row_count; use crate::dtype::vx_dtype; + use crate::scan::vx_estimate; + use crate::scan::vx_estimate_type; use crate::session::vx_session_free; use crate::session::vx_session_new; use crate::tests::SAMPLE_ROWS; @@ -221,13 +208,10 @@ mod tests { let dtype = vx_dtype::as_ref(vx_data_source_dtype(ds)); assert_eq!(dtype, struct_array.dtype()); - let mut row_count = vx_data_source_row_count { - cardinality: vx_cardinality::VX_CARD_UNKNOWN, - rows: 0, - }; + let mut row_count = vx_estimate::default(); vx_data_source_get_row_count(ds, &raw mut row_count); - assert_eq!(row_count.cardinality, vx_cardinality::VX_CARD_MAXIMUM); - assert_eq!(row_count.rows, SAMPLE_ROWS as u64); + assert_eq!(row_count.r#type, vx_estimate_type::VX_ESTIMATE_EXACT); + assert_eq!(row_count.estimate, SAMPLE_ROWS as u64); vx_data_source_free(ds); vx_session_free(session); diff --git a/vortex-ffi/src/scan.rs b/vortex-ffi/src/scan.rs index fdae46aca4f..7d728aa135c 100644 --- a/vortex-ffi/src/scan.rs +++ b/vortex-ffi/src/scan.rs @@ -49,7 +49,11 @@ pub enum VxScan { Started(PartitionStream), Finished, } -crate::box_wrapper!(VxScan, vx_scan); +crate::box_wrapper!( + /// A scan is a single traversal of a data source with projections and + /// filters. A scan can be consumed only once. + VxScan, + vx_scan); pub enum VxPartitionScan { Pending(Box), @@ -114,8 +118,10 @@ pub struct vx_scan_options { } #[repr(C)] +#[cfg_attr(test, derive(Debug, PartialEq, Eq, Default))] pub enum vx_estimate_type { /// No estimate is available. + #[cfg_attr(test, default)] VX_ESTIMATE_UNKNOWN = 0, /// The value in vx_estimate.estimate is exact. VX_ESTIMATE_EXACT = 1, @@ -126,10 +132,11 @@ pub enum vx_estimate_type { /// Used for estimating number of partitions in a data source or number of rows /// in a partition. #[repr(C)] +#[cfg_attr(test, derive(Default))] pub struct vx_estimate { - r#type: vx_estimate_type, + pub r#type: vx_estimate_type, /// Set only when "type" is not VX_ESTIMATE_UNKNOWN. - estimate: u64, + pub estimate: u64, } fn scan_request(opts: *const vx_scan_options) -> VortexResult { @@ -325,15 +332,15 @@ pub unsafe extern "C-unwind" fn vx_partition_row_count( }) } -// Scan partition to ArrowArrayStream. -// Consumes partition fully: subsequent calls to vx_partition_scan_arrow or -// vx_partition_next are undefined behaviour. -// This call blocks current thread until underlying stream is fully consumed. -// -// Caller must not free partition after calling this function. -// -// On success, sets "stream" and returns 0. -// On error, sets "err" and returns 1, freeing the partition. +/// Scan partition to ArrowArrayStream. +/// Consumes partition fully: subsequent calls to vx_partition_scan_arrow or +/// vx_partition_next are undefined behaviour. +/// This call blocks current thread until underlying stream is fully consumed. +/// +/// Caller must not free partition after calling this function. +/// +/// On success, sets "stream" and returns 0. +/// On error, sets "err" and returns 1, freeing the partition. #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_partition_scan_arrow( session: *const vx_session, diff --git a/vortex-ffi/test/CMakeLists.txt b/vortex-ffi/test/CMakeLists.txt index 6ec4c130800..be0288ef954 100644 --- a/vortex-ffi/test/CMakeLists.txt +++ b/vortex-ffi/test/CMakeLists.txt @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors include(CTest) -include(FetchContent) - FetchContent_Declare( Catch GIT_REPOSITORY https://github.com/catchorg/Catch2.git @@ -13,13 +11,6 @@ include(Catch) # https://github.com/catchorg/Catch2/issues/1833 target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) -FetchContent_Declare( - Nanoarrow - GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow - GIT_TAG apache-arrow-nanoarrow-0.8.0 -) -FetchContent_MakeAvailable(Nanoarrow) - file(GLOB TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") message(NOTICE "Test files ${TEST_FILES}") add_executable(vortex_ffi_test ${TEST_FILES}) diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp index 976cdc7f355..a885e7d1798 100644 --- a/vortex-ffi/test/scan.cpp +++ b/vortex-ffi/test/scan.cpp @@ -251,11 +251,11 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { vx_data_source_free(ds); }; - vx_data_source_row_count row_count = {}; + vx_estimate row_count; vx_data_source_get_row_count(ds, &row_count); - CHECK(row_count.cardinality == VX_CARD_MAXIMUM); - CHECK(row_count.rows == SAMPLE_ROWS); + CHECK(row_count.type == VX_ESTIMATE_EXACT); + CHECK(row_count.estimate == SAMPLE_ROWS); const vx_dtype *data_source_dtype = vx_data_source_dtype(ds); REQUIRE(vx_dtype_get_variant(data_source_dtype) == DTYPE_STRUCT); diff --git a/vortex-file/src/multi/mod.rs b/vortex-file/src/multi/mod.rs index 5e7f5466921..0b97376c362 100644 --- a/vortex-file/src/multi/mod.rs +++ b/vortex-file/src/multi/mod.rs @@ -77,9 +77,20 @@ impl MultiFileDataSource { /// /// The glob path should be relative to the filesystem's base URL. Pass `None` for the /// filesystem to use the local filesystem (auto-created in [`Self::build`]). + /// + /// Relative paths are resolved against the process working directory. pub fn with_glob(mut self, glob: impl Into, fs: Option) -> Self { - let glob_str = glob.into().trim_start_matches('/').to_string(); - self.glob_sources.push((glob_str, fs)); + let glob = glob.into(); + let glob = if fs.is_none() && std::path::Path::new(&glob).is_relative() { + std::env::current_dir() + .map(|cwd| cwd.join(&glob).to_string_lossy().into_owned()) + .unwrap_or(glob) + .trim_start_matches('/') + .to_string() + } else { + glob.trim_start_matches('/').to_string() + }; + self.glob_sources.push((glob, fs)); self } From 3d70d43393929cd42b30ba8f56cca775543ea604 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:57:27 +0100 Subject: [PATCH 196/250] Update dependency typescript to v6 (#7662) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [typescript](https://www.typescriptlang.org/) ([source](https://redirect.github.com/microsoft/TypeScript)) | [`~5.9.0` → `~6.0.0`](https://renovatebot.com/diffs/npm/typescript/5.9.3/6.0.3) | ![age](https://developer.mend.io/api/mc/badges/age/npm/typescript/6.0.3?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/npm/typescript/5.9.3/6.0.3?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Release Notes

() @@ -79,7 +84,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), @@ -93,8 +98,10 @@ pub fn compare_canonical_array( } DType::Decimal(..) => { let decimal = value.as_decimal(); - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array + .clone() + .execute::(ctx) + .vortex_expect("to decimal"); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let dval = decimal .decimal_value() @@ -110,7 +117,7 @@ pub fn compare_canonical_array( array .validity() .vortex_expect("validity_mask") - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .to_mask(array.len(), ctx) .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), @@ -123,8 +130,10 @@ pub fn compare_canonical_array( }) } DType::Utf8(_) => { - #[expect(deprecated)] - let varbinview = array.to_varbinview(); + let varbinview = array + .clone() + .execute::(ctx) + .vortex_expect("to varbinview"); varbinview.with_iterator(|iter| { let utf8_value = value.as_utf8(); compare_to( @@ -136,8 +145,10 @@ pub fn compare_canonical_array( }) } DType::Binary(_) => { - #[expect(deprecated)] - let varbinview = array.to_varbinview(); + let varbinview = array + .clone() + .execute::(ctx) + .vortex_expect("to varbinview"); varbinview.with_iterator(|iter| { let binary_value = value.as_binary(); compare_to( @@ -151,9 +162,8 @@ pub fn compare_canonical_array( }) } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let scalar_vals: Vec = (0..array.len()) - .map(|i| array.execute_scalar(i, &mut ctx).vortex_expect("scalar_at")) + .map(|i| array.execute_scalar(i, ctx).vortex_expect("scalar_at")) .collect(); BoolArray::from_iter(scalar_vals.iter().map(|v| { scalar_cmp(v, value, operator) diff --git a/fuzz/src/array/fill_null.rs b/fuzz/src/array/fill_null.rs index b9705147326..d19a9f2b360 100644 --- a/fuzz/src/array/fill_null.rs +++ b/fuzz/src/array/fill_null.rs @@ -5,11 +5,8 @@ use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::DecimalArray; @@ -34,16 +31,19 @@ use vortex_error::VortexResult; pub fn fill_null_canonical_array( canonical: Canonical, fill_value: &Scalar, + ctx: &mut ExecutionCtx, ) -> VortexResult { let result_nullability = fill_value.dtype().nullability(); Ok(match canonical { Canonical::Null(array) => ConstantArray::new(fill_value.clone(), array.len()).into_array(), - Canonical::Bool(array) => fill_bool_array(array, fill_value, result_nullability), - Canonical::Primitive(array) => fill_primitive_array(array, fill_value, result_nullability), - Canonical::Decimal(array) => fill_decimal_array(array, fill_value, result_nullability), + Canonical::Bool(array) => fill_bool_array(array, fill_value, result_nullability, ctx), + Canonical::Primitive(array) => { + fill_primitive_array(array, fill_value, result_nullability, ctx) + } + Canonical::Decimal(array) => fill_decimal_array(array, fill_value, result_nullability, ctx), Canonical::VarBinView(array) => { - fill_varbinview_array(array, fill_value, result_nullability) + fill_varbinview_array(array, fill_value, result_nullability, ctx) } Canonical::Struct(_) | Canonical::List(_) @@ -57,6 +57,7 @@ fn fill_bool_array( array: BoolArray, fill_value: &Scalar, result_nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> ArrayRef { let fill_bool = fill_value .as_bool() @@ -72,8 +73,9 @@ fn fill_bool_array( } Validity::AllInvalid => ConstantArray::new(fill_value.clone(), array.len()).into_array(), Validity::Array(validity_array) => { - #[expect(deprecated)] - let validity_bool = validity_array.to_bool(); + let validity_bool = validity_array + .execute::(ctx) + .vortex_expect("validity to bool"); let validity_bits = validity_bool.into_bit_buffer(); let data_bits = array.into_bit_buffer(); @@ -103,6 +105,7 @@ fn fill_primitive_array( array: PrimitiveArray, fill_value: &Scalar, result_nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> ArrayRef { match_each_native_ptype!(array.ptype(), |T| { let fill_val = T::try_from(fill_value) @@ -119,8 +122,9 @@ fn fill_primitive_array( ConstantArray::new(fill_value.clone(), array.len()).into_array() } Validity::Array(validity_array) => { - #[expect(deprecated)] - let validity_bool_array = validity_array.to_bool(); + let validity_bool_array = validity_array + .execute::(ctx) + .vortex_expect("validity to bool"); let validity_bits = validity_bool_array.to_bit_buffer(); let data_slice = array.as_slice::(); @@ -144,6 +148,7 @@ fn fill_decimal_array( array: DecimalArray, fill_value: &Scalar, result_nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> ArrayRef { let decimal_dtype = array.decimal_dtype(); let decimal_scalar = fill_value.as_decimal(); @@ -166,8 +171,9 @@ fn fill_decimal_array( ConstantArray::new(fill_value.clone(), array.len()).into_array() } Validity::Array(validity_array) => { - #[expect(deprecated)] - let validity_bool_array = validity_array.to_bool(); + let validity_bool_array = validity_array + .execute::(ctx) + .vortex_expect("validity to bool"); let validity_bits = validity_bool_array.to_bit_buffer(); let data_buffer = array.buffer::(); @@ -192,6 +198,7 @@ fn fill_varbinview_array( array: VarBinViewArray, fill_value: &Scalar, result_nullability: Nullability, + ctx: &mut ExecutionCtx, ) -> ArrayRef { let array_ref = array.clone().into_array(); match array @@ -201,8 +208,9 @@ fn fill_varbinview_array( Validity::NonNullable | Validity::AllValid => array.into_array(), Validity::AllInvalid => ConstantArray::new(fill_value.clone(), array.len()).into_array(), Validity::Array(validity_array) => { - #[expect(deprecated)] - let validity_bool_array = validity_array.to_bool(); + let validity_bool_array = validity_array + .execute::(ctx) + .vortex_expect("validity to bool"); let validity_bits = validity_bool_array.to_bit_buffer(); match array.dtype() { @@ -215,7 +223,7 @@ fn fill_varbinview_array( .map(|i| { if validity_bits.value(i) { array_ref - .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at") .as_utf8() .value() @@ -229,8 +237,10 @@ fn fill_varbinview_array( let string_refs: Vec<&str> = strings.iter().map(|s| s.as_str()).collect(); let result = VarBinViewArray::from_iter_str(string_refs).into_array(); if result_nullability == Nullability::Nullable { - #[expect(deprecated)] - let result_vbv = result.to_varbinview(); + let result_vbv = result + .clone() + .execute::(ctx) + .vortex_expect("to varbinview"); VarBinViewArray::new_handle( result_vbv.views_handle().clone(), Arc::clone(result_vbv.data_buffers()), @@ -251,7 +261,7 @@ fn fill_varbinview_array( .map(|i| { if validity_bits.value(i) { array_ref - .execute_scalar(i, &mut LEGACY_SESSION.create_execution_ctx()) + .execute_scalar(i, ctx) .vortex_expect("scalar_at") .as_binary() .value() @@ -265,8 +275,10 @@ fn fill_varbinview_array( let binary_refs: Vec<&[u8]> = binaries.iter().map(|b| b.as_slice()).collect(); let result = VarBinViewArray::from_iter_bin(binary_refs).into_array(); if result_nullability == Nullability::Nullable { - #[expect(deprecated)] - let result_vbv = result.to_varbinview(); + let result_vbv = result + .clone() + .execute::(ctx) + .vortex_expect("to varbinview"); VarBinViewArray::new_handle( result_vbv.views_handle().clone(), Arc::clone(result_vbv.data_buffers()), @@ -316,10 +328,11 @@ mod tests { #[test] fn test_fill_null_primitive() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)]); let fill_value = Scalar::from(42i32); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = PrimitiveArray::from_iter([1i32, 42, 3, 42, 5]); assert_arrays_eq!(expected, result); @@ -327,12 +340,13 @@ mod tests { #[test] fn test_fill_null_bool() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let data_buffer = BitBuffer::from(vec![true, false, false, false]); let validity_buffer = BitBuffer::from(vec![true, false, true, false]); let array = BoolArray::new(data_buffer, Validity::from(validity_buffer)); let fill_value = Scalar::from(true); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = BoolArray::from(BitBuffer::from(vec![true, true, false, true])); assert_arrays_eq!(expected, result); @@ -340,13 +354,14 @@ mod tests { #[test] fn test_fill_null_string() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = VarBinViewArray::from_iter( [Some("hello"), None, Some("world")].iter().copied(), DType::Utf8(Nullability::Nullable), ); let fill_value = Scalar::utf8("default", Nullability::NonNullable); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = VarBinViewArray::from_iter_str(["hello", "default", "world"]); assert_arrays_eq!(expected, result); @@ -354,10 +369,11 @@ mod tests { #[test] fn test_fill_null_all_invalid() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter([None::, None, None]); let fill_value = Scalar::from(100i32); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = PrimitiveArray::from_iter([100i32, 100, 100]); assert_arrays_eq!(expected, result); @@ -365,10 +381,11 @@ mod tests { #[test] fn test_fill_null_no_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1i32, 2, 3]); let fill_value = Scalar::from(42i32); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = PrimitiveArray::from_iter([1i32, 2, 3]); assert_arrays_eq!(expected, result); @@ -377,10 +394,11 @@ mod tests { #[test] #[should_panic] fn test_fill_null_with_null_value_errors() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3)]); let fill_value = Scalar::null(DType::Primitive(PType::I32, Nullability::Nullable)); - let result = fill_null_canonical_array(canonical(array), &fill_value); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx); assert!(result.is_err()); assert!( @@ -393,6 +411,7 @@ mod tests { #[test] fn test_fill_null_decimal_i32() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = DecimalArray::from_option_iter( [Some(100i32), None, Some(300i32), None, Some(500i32)], DecimalDType::new(10, 2), @@ -403,7 +422,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = DecimalArray::from_iter( [100i32, 999i32, 300i32, 999i32, 500i32], @@ -414,6 +433,7 @@ mod tests { #[test] fn test_fill_null_decimal_i64() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = DecimalArray::from_option_iter( [Some(1000i64), None, Some(3000i64)], DecimalDType::new(15, 3), @@ -424,7 +444,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = DecimalArray::from_iter([1000i64, 9999i64, 3000i64], DecimalDType::new(15, 3)); @@ -433,6 +453,7 @@ mod tests { #[test] fn test_fill_null_decimal_i128() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = DecimalArray::from_option_iter( [Some(10000i128), None, Some(30000i128), None], DecimalDType::new(20, 4), @@ -443,7 +464,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = DecimalArray::from_iter( [10000i128, 99999i128, 30000i128, 99999i128], @@ -454,6 +475,7 @@ mod tests { #[test] fn test_fill_null_decimal_all_invalid() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = DecimalArray::from_option_iter([None::, None, None], DecimalDType::new(10, 2)); let fill_value = Scalar::decimal( @@ -462,7 +484,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = DecimalArray::from_option_iter( [Some(777i64), Some(777i64), Some(777i64)], @@ -476,6 +498,7 @@ mod tests { #[test] fn test_fill_null_decimal_no_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = DecimalArray::from_option_iter( [Some(100i32), Some(200i32), Some(300i32)], DecimalDType::new(10, 2), @@ -486,7 +509,7 @@ mod tests { Nullability::NonNullable, ); - let result = fill_null_canonical_array(canonical(array), &fill_value).unwrap(); + let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap(); let expected = DecimalArray::from_option_iter( [Some(100i32), Some(200i32), Some(300i32)], diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index c96d1044efe..09e10e26e9f 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -2,11 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -25,12 +22,13 @@ use vortex_error::VortexResult; use crate::array::take_canonical_array_non_nullable_indices; -pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult { +pub fn filter_canonical_array( + array: &ArrayRef, + filter: &[bool], + ctx: &mut ExecutionCtx, +) -> VortexResult { let validity = if array.dtype().is_nullable() { - let validity_buff = array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? - .to_bit_buffer(); + let validity_buff = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); Validity::from_iter( filter .iter() @@ -44,8 +42,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult match array.dtype() { DType::Bool(_) => { - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array.clone().execute::(ctx)?; Ok(BoolArray::new( BitBuffer::from_iter( filter @@ -59,8 +56,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult .into_array()) } DType::Primitive(p, _) => match_each_native_ptype!(p, |P| { - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array.clone().execute::(ctx)?; Ok(PrimitiveArray::new( filter .iter() @@ -73,8 +69,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult .into_array()) }), DType::Decimal(d, _) => { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array.clone().execute::(ctx)?; match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); Ok(DecimalArray::new( @@ -91,8 +86,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult }) } DType::Utf8(_) | DType::Binary(_) => { - #[expect(deprecated)] - let utf8 = array.to_varbinview(); + let utf8 = array.clone().execute::(ctx)?; let values = utf8.with_iterator(|iter| { iter.zip(filter.iter()) .filter(|(_, f)| **f) @@ -102,11 +96,10 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult Ok(VarBinViewArray::from_iter(values, array.dtype().clone()).into_array()) } DType::Struct(..) => { - #[expect(deprecated)] - let struct_array = array.to_struct(); + let struct_array = array.clone().execute::(ctx)?; let filtered_children = struct_array .iter_unmasked_fields() - .map(|c| filter_canonical_array(c, filter)) + .map(|c| filter_canonical_array(c, filter, ctx)) .collect::>>()?; StructArray::try_new_with_dtype( @@ -124,7 +117,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult indices.push(idx); } } - take_canonical_array_non_nullable_indices(array, indices.as_slice()) + take_canonical_array_non_nullable_indices(array, indices.as_slice(), ctx) } d @ (DType::Null | DType::Extension(_) | DType::Variant(_)) => { unreachable!("DType {d} not supported for fuzzing") diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 75cf60f6d9c..49fc2ad40f9 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -5,11 +5,8 @@ use std::sync::Arc; use vortex_array::ArrayRef; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::ExtensionArray; @@ -35,7 +32,7 @@ use vortex_mask::Mask; /// This needs to be coherent with applications of Mask. /// The result is always nullable. The result has the same length as self. #[inline] -pub fn mask_validity(validity: &Validity, mask: &Mask) -> Validity { +pub fn mask_validity(validity: &Validity, mask: &Mask, ctx: &mut ExecutionCtx) -> Validity { let out = match mask.bit_buffer() { AllOr::All => validity.clone().into_nullable(), AllOr::None => Validity::AllInvalid, @@ -45,8 +42,10 @@ pub fn mask_validity(validity: &Validity, mask: &Mask) -> Validity { Validity::from_bit_buffer(make_valid.clone(), Nullability::Nullable) } Validity::Array(is_valid) => { - #[expect(deprecated)] - let is_valid = is_valid.to_bool(); + let is_valid = is_valid + .clone() + .execute::(ctx) + .vortex_expect("validity to bool"); Validity::from_bit_buffer( is_valid.to_bit_buffer() & make_valid, Nullability::Nullable, @@ -62,18 +61,22 @@ pub fn mask_validity(validity: &Validity, mask: &Mask) -> Validity { /// Apply mask on the canonical form of the array to get a consistent baseline. /// This implementation manually applies the mask to each canonical type /// without using the mask_fn method, to serve as an independent baseline for testing. -pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { +pub fn mask_canonical_array( + canonical: Canonical, + mask: &Mask, + ctx: &mut ExecutionCtx, +) -> VortexResult { Ok(match canonical { Canonical::Null(array) => { // Null arrays are already all invalid, masking has no effect array.into_array() } Canonical::Bool(array) => { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); BoolArray::new(array.to_bit_buffer(), new_validity).into_array() } Canonical::Primitive(array) => { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); PrimitiveArray::from_buffer_handle( array.buffer_handle().clone(), array.ptype(), @@ -82,14 +85,14 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); match_each_decimal_value_type!(array.values_type(), |D| { DecimalArray::new(array.buffer::(), array.decimal_dtype(), new_validity) .into_array() }) } Canonical::VarBinView(array) => { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); VarBinViewArray::new_handle( array.views_handle().clone(), Arc::clone(array.data_buffers()), @@ -99,7 +102,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); // SAFETY: Since we are only masking the validity and everything else comes from an // already valid `ListViewArray`, all of the invariants are still upheld. @@ -115,7 +118,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); FixedSizeListArray::new( array.elements().clone(), array.list_size(), @@ -125,7 +128,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { - let new_validity = mask_validity(&array.validity()?, mask); + let new_validity = mask_validity(&array.validity()?, mask, ctx); StructArray::try_new_with_dtype( array.unmasked_fields(), array.struct_fields().clone(), @@ -137,14 +140,9 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult { // Recursively mask the storage array - let masked_storage = mask_canonical_array( - array - .storage_array() - .clone() - .execute::(&mut LEGACY_SESSION.create_execution_ctx())?, - mask, - ) - .vortex_expect("mask_canonical_array should succeed in fuzz test"); + let storage_canonical = array.storage_array().clone().execute::(ctx)?; + let masked_storage = mask_canonical_array(storage_canonical, mask, ctx) + .vortex_expect("mask_canonical_array should succeed in fuzz test"); let ext_dtype = array .ext_dtype() @@ -186,28 +184,26 @@ mod tests { #[test] fn test_mask_null_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = NullArray::new(5); let mask = Mask::from_iter([true, false, true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); assert_eq!(result.len(), 5); // All values should still be null for i in 0..5 { - assert!( - !result - .is_valid(i, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(!result.is_valid(i, &mut ctx).unwrap()); } } #[test] fn test_mask_bool_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = BoolArray::from_iter([true, false, true, false, true]); let mask = Mask::from_iter([false, true, true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = BoolArray::from_iter([None, Some(false), Some(true), None, Some(true)]); assert_arrays_eq!(result, expected); @@ -215,10 +211,11 @@ mod tests { #[test] fn test_mask_primitive_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::from_iter([true, false, true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), None, Some(5)]); assert_arrays_eq!(result, expected); @@ -226,10 +223,11 @@ mod tests { #[test] fn test_mask_primitive_array_with_nulls() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_option_iter([Some(1i32), None, Some(3), Some(4), None]); let mask = Mask::from_iter([false, true, true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = PrimitiveArray::from_option_iter([None, None, Some(3i32), None, None]); assert_arrays_eq!(result, expected); @@ -237,6 +235,7 @@ mod tests { #[test] fn test_mask_decimal_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let dtype = DecimalDType::new(10, 2); let array = DecimalArray::from_option_iter( [Some(1i128), Some(2), Some(3), Some(4), Some(5)], @@ -244,7 +243,7 @@ mod tests { ); let mask = Mask::from_iter([true, true, false, true, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = DecimalArray::from_option_iter([Some(1i128), Some(2), None, Some(4), Some(5)], dtype); @@ -253,10 +252,11 @@ mod tests { #[test] fn test_mask_varbinview_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = VarBinViewArray::from_iter_str(["one", "two", "three", "four", "five"]); let mask = Mask::from_iter([false, true, false, true, false]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = VarBinViewArray::from_iter_nullable_str([None, Some("two"), None, Some("four"), None]); @@ -265,6 +265,7 @@ mod tests { #[test] fn test_mask_list_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5, 6]).into_array(); let offsets = PrimitiveArray::from_iter([0i32, 2, 4]).into_array(); let sizes = PrimitiveArray::from_iter([2i32, 2, 2]).into_array(); @@ -275,56 +276,34 @@ mod tests { let mask = Mask::from_iter([true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); assert_eq!(result.len(), 3); - assert!( - result - .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !result - .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - result - .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(result.is_valid(0, &mut ctx).unwrap()); + assert!(!result.is_valid(1, &mut ctx).unwrap()); + assert!(result.is_valid(2, &mut ctx).unwrap()); } #[test] fn test_mask_fixed_size_list_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let elements = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5, 6]).into_array(); let array = FixedSizeListArray::try_new(elements, 2, Nullability::NonNullable.into(), 3).unwrap(); let mask = Mask::from_iter([false, true, false]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); assert_eq!(result.len(), 3); - assert!( - !result - .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - result - .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !result - .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(!result.is_valid(0, &mut ctx).unwrap()); + assert!(result.is_valid(1, &mut ctx).unwrap()); + assert!(!result.is_valid(2, &mut ctx).unwrap()); } #[test] fn test_mask_struct_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let field1 = PrimitiveArray::from_iter([1i32, 2, 3]).into_array(); let field2 = PrimitiveArray::from_iter([4i32, 5, 6]).into_array(); let fields = vec![field1, field2]; @@ -339,32 +318,21 @@ mod tests { let mask = Mask::from_iter([true, false, true]); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); assert_eq!(result.len(), 3); - assert!( - result - .is_valid(0, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - !result - .is_valid(1, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); - assert!( - result - .is_valid(2, &mut LEGACY_SESSION.create_execution_ctx()) - .unwrap() - ); + assert!(result.is_valid(0, &mut ctx).unwrap()); + assert!(!result.is_valid(1, &mut ctx).unwrap()); + assert!(result.is_valid(2, &mut ctx).unwrap()); } #[test] fn test_mask_all_false() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::AllFalse(5); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = PrimitiveArray::from_option_iter([None, None, None, None, None::]); assert_arrays_eq!(result, expected); @@ -372,10 +340,11 @@ mod tests { #[test] fn test_mask_all_true() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter([1i32, 2, 3, 4, 5]); let mask = Mask::AllTrue(5); - let result = mask_canonical_array(canonical(array), &mask).unwrap(); + let result = mask_canonical_array(canonical(array), &mask, &mut ctx).unwrap(); let expected = PrimitiveArray::from_option_iter([Some(1i32), Some(2), Some(3), Some(4), Some(5)]); @@ -384,9 +353,10 @@ mod tests { #[test] fn test_mask_empty_array() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); let array = PrimitiveArray::from_iter(Vec::::new()); for mask in [Mask::AllFalse(0), Mask::AllTrue(0)] { - let result = mask_canonical_array(canonical(array.clone()), &mask).unwrap(); + let result = mask_canonical_array(canonical(array.clone()), &mask, &mut ctx).unwrap(); assert_eq!(result.len(), 0); } } diff --git a/fuzz/src/array/mod.rs b/fuzz/src/array/mod.rs index 6dc126d9254..d030479ac91 100644 --- a/fuzz/src/array/mod.rs +++ b/fuzz/src/array/mod.rs @@ -40,6 +40,7 @@ use strum::EnumIter; use strum::IntoEnumIterator; use tracing::debug; use vortex_array::ArrayRef; +use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::min_max::MinMaxResult; @@ -202,7 +203,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { ActionType::Slice => { let start = u.choose_index(current_array.len())?; let stop = u.int_in_range(start..=current_array.len())?; - current_array = slice_canonical_array(¤t_array, start, stop) + current_array = slice_canonical_array(¤t_array, start, stop, &mut ctx) .vortex_expect("slice_canonical_array should succeed in fuzz test"); ( @@ -218,7 +219,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { let indices = random_vec_in_range(u, 0, current_array.len() - 1)?; let nullable = indices.contains(&None); - current_array = take_canonical_array(¤t_array, &indices) + current_array = take_canonical_array(¤t_array, &indices, &mut ctx) .vortex_expect("take_canonical_array should succeed in fuzz test"); let indices_array = if nullable { PrimitiveArray::from_option_iter( @@ -260,7 +261,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { return Err(EmptyChoose); } - let sorted = sort_canonical_array(¤t_array) + let sorted = sort_canonical_array(¤t_array, &mut ctx) .vortex_expect("sort_canonical_array should succeed in fuzz test"); let side = if u.arbitrary()? { @@ -271,9 +272,10 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { ( Action::SearchSorted(scalar.clone(), side), ExpectedValue::Search( - search_sorted_canonical_array(&sorted, &scalar, side).vortex_expect( - "search_sorted_canonical_array should succeed in fuzz test", - ), + search_sorted_canonical_array(&sorted, &scalar, side, &mut ctx) + .vortex_expect( + "search_sorted_canonical_array should succeed in fuzz test", + ), ), ) } @@ -281,7 +283,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { let mask = (0..current_array.len()) .map(|_| bool::arbitrary(u)) .collect::>>()?; - current_array = filter_canonical_array(¤t_array, &mask) + current_array = filter_canonical_array(¤t_array, &mask, &mut ctx) .vortex_expect("filter_canonical_array should succeed in fuzz test"); ( Action::Filter(Mask::from_iter(mask)), @@ -300,7 +302,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { }; let op = u.arbitrary()?; - current_array = compare_canonical_array(¤t_array, &scalar, op); + current_array = compare_canonical_array(¤t_array, &scalar, op, &mut ctx); ( Action::Compare(scalar, op), ExpectedValue::Array(current_array.clone()), @@ -312,7 +314,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { { return Err(EmptyChoose); } - let Some(result) = cast_canonical_array(¤t_array, &to) + let Some(result) = cast_canonical_array(¤t_array, &to, &mut ctx) .vortex_expect("should fail to create array") else { return Err(EmptyChoose); @@ -327,20 +329,20 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { } // Sum - returns a scalar, does NOT update current_array (terminal operation) - #[expect(deprecated)] let current_array_canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); let sum_result = sum_canonical_array(current_array_canonical, &mut ctx) .vortex_expect("sum_canonical_array should succeed in fuzz test"); (Action::Sum, ExpectedValue::Scalar(sum_result)) } ActionType::MinMax => { // MinMax - returns a scalar, does NOT update current_array (terminal operation) - #[expect(deprecated)] let current_array_canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); let min_max_result = min_max_canonical_array(current_array_canonical, &mut ctx) .vortex_expect("min_max_canonical_array should succeed in fuzz test"); (Action::MinMax, ExpectedValue::MinMax(min_max_result)) @@ -368,12 +370,12 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { } // Compute expected result on canonical form - #[expect(deprecated)] let current_array_canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); let expected_result = - fill_null_canonical_array(current_array_canonical, &fill_value) + fill_null_canonical_array(current_array_canonical, &fill_value, &mut ctx) .vortex_expect("fill_null_canonical_array should succeed in fuzz test"); // Update current_array to the result for chaining current_array = expected_result.clone(); @@ -389,13 +391,14 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { .collect::>>()?; // Compute expected result on canonical form - #[expect(deprecated)] let current_array_canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); let expected_result = mask_canonical_array( current_array_canonical, &Mask::from_iter(mask.clone()), + &mut ctx, ) .vortex_expect("mask_canonical_array should succeed in fuzz test"); // Update current_array to the result for chaining @@ -424,11 +427,11 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction { let expected_scalars: Vec = indices_vec .iter() .map(|&idx| { - #[expect(deprecated)] let canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); - scalar_at_canonical_array(canonical, idx).vortex_expect( + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); + scalar_at_canonical_array(canonical, idx, &mut ctx).vortex_expect( "scalar_at_canonical_array should succeed in fuzz test", ) }) @@ -581,10 +584,10 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { debug!(id = i, action = ?action); match action { Action::Compress(strategy) => { - #[expect(deprecated)] let canonical = current_array - .to_canonical() - .vortex_expect("to_canonical should succeed in fuzz test"); + .clone() + .execute::(&mut ctx) + .vortex_expect("execute canonical should succeed in fuzz test"); current_array = compress_array(&canonical.into_array(), strategy); assert_array_eq(&expected.array(), ¤t_array, i)?; } @@ -604,7 +607,7 @@ pub fn run_fuzz_action(fuzz_action: FuzzArrayAction) -> VortexFuzzResult { assert_array_eq(&expected.array(), ¤t_array, i)?; } Action::SearchSorted(s, side) => { - let mut sorted = sort_canonical_array(¤t_array) + let mut sorted = sort_canonical_array(¤t_array, &mut ctx) .vortex_expect("sort_canonical_array should succeed in fuzz test"); if !current_array.is_canonical() { diff --git a/fuzz/src/array/scalar_at.rs b/fuzz/src/array/scalar_at.rs index fcc26c39b12..e13ad75343d 100644 --- a/fuzz/src/array/scalar_at.rs +++ b/fuzz/src/array/scalar_at.rs @@ -4,9 +4,8 @@ use std::sync::Arc; use vortex_array::Canonical; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -use vortex_array::VortexSessionExecute; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::arrays::extension::ExtensionArrayExt; use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt; @@ -24,9 +23,13 @@ use vortex_error::VortexResult; /// Baseline implementation of scalar_at that works on canonical arrays. /// This implementation manually extracts the scalar value from each canonical type /// without using the scalar_at method, to serve as an independent baseline for testing. -pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexResult { +pub fn scalar_at_canonical_array( + canonical: Canonical, + index: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult { let canonical_ref = canonical.clone().into_array(); - if canonical_ref.is_invalid(index, &mut LEGACY_SESSION.create_execution_ctx())? { + if canonical_ref.is_invalid(index, ctx)? { return Ok(Scalar::null(canonical_ref.dtype().clone())); } Ok(match canonical { @@ -54,11 +57,11 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let list = array.list_elements_at(index)?; let children: Vec = (0..list.len()) .map(|i| { - #[expect(deprecated)] let canonical = list - .to_canonical() + .clone() + .execute::(ctx) .vortex_expect("to_canonical should succeed in fuzz test"); - scalar_at_canonical_array(canonical, i) + scalar_at_canonical_array(canonical, i, ctx) .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); @@ -72,11 +75,11 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let list = array.fixed_size_list_elements_at(index)?; let children: Vec = (0..list.len()) .map(|i| { - #[expect(deprecated)] let canonical = list - .to_canonical() + .clone() + .execute::(ctx) .vortex_expect("to_canonical should succeed in fuzz test"); - scalar_at_canonical_array(canonical, i) + scalar_at_canonical_array(canonical, i, ctx) .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); @@ -86,20 +89,19 @@ pub fn scalar_at_canonical_array(canonical: Canonical, index: usize) -> VortexRe let field_scalars: Vec = array .iter_unmasked_fields() .map(|field| { - #[expect(deprecated)] let canonical = field - .to_canonical() + .clone() + .execute::(ctx) .vortex_expect("to_canonical should succeed in fuzz test"); - scalar_at_canonical_array(canonical, index) + scalar_at_canonical_array(canonical, index, ctx) .vortex_expect("scalar_at_canonical_array should succeed in fuzz test") }) .collect(); Scalar::struct_(array.dtype().clone(), field_scalars) } Canonical::Extension(array) => { - #[expect(deprecated)] - let storage_canonical = array.storage_array().to_canonical()?; - let storage_scalar = scalar_at_canonical_array(storage_canonical, index)?; + let storage_canonical = array.storage_array().clone().execute::(ctx)?; + let storage_scalar = scalar_at_canonical_array(storage_canonical, index, ctx)?; Scalar::extension_ref(array.ext_dtype().clone(), storage_scalar) } Canonical::Variant(_) => unreachable!("Variant arrays are not fuzzed"), diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index f0de94db90a..30b4d234b18 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -5,11 +5,12 @@ use std::cmp::Ordering; use std::fmt::Debug; use vortex_array::ArrayRef; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; +use vortex_array::ExecutionCtx; use vortex_array::accessor::ArrayAccessor; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::DecimalArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -63,18 +64,15 @@ pub fn search_sorted_canonical_array( array: &ArrayRef, scalar: &Scalar, side: SearchSortedSide, + ctx: &mut ExecutionCtx, ) -> VortexResult { match array.dtype() { DType::Bool(_) => { - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array.clone().execute::(ctx)?; let validity = bool_array .as_ref() .validity()? - .to_mask( - bool_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(bool_array.as_ref().len(), ctx)? .to_bit_buffer(); let opt_values = bool_array .to_bit_buffer() @@ -86,15 +84,11 @@ pub fn search_sorted_canonical_array( SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side) } DType::Primitive(p, _) => { - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array.clone().execute::(ctx)?; let validity = primitive_array .as_ref() .validity()? - .to_mask( - primitive_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(primitive_array.as_ref().len(), ctx)? .to_bit_buffer(); match_each_native_ptype!(p, |P| { let opt_values = primitive_array @@ -109,15 +103,11 @@ pub fn search_sorted_canonical_array( }) } DType::Decimal(d, _) => { - #[expect(deprecated)] - let decimal_array = array.to_decimal(); + let decimal_array = array.clone().execute::(ctx)?; let validity = decimal_array .as_ref() .validity()? - .to_mask( - decimal_array.as_ref().len(), - &mut LEGACY_SESSION.create_execution_ctx(), - )? + .to_mask(decimal_array.as_ref().len(), ctx)? .to_bit_buffer(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); @@ -142,8 +132,7 @@ pub fn search_sorted_canonical_array( }) } DType::Utf8(_) | DType::Binary(_) => { - #[expect(deprecated)] - let utf8 = array.to_varbinview(); + let utf8 = array.clone().execute::(ctx)?; let opt_values = utf8.with_iterator(|iter| iter.map(|v| v.map(|u| u.to_vec())).collect::>()); let to_find = if matches!(array.dtype(), DType::Utf8(_)) { @@ -154,9 +143,8 @@ pub fn search_sorted_canonical_array( SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side) } DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); let scalar_vals = (0..array.len()) - .map(|i| array.execute_scalar(i, &mut ctx)) + .map(|i| array.execute_scalar(i, ctx)) .collect::>>()?; scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side) } diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index c131f70e9c4..ce0cfe062b6 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -2,11 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; -use vortex_array::LEGACY_SESSION; -#[expect(deprecated)] -use vortex_array::ToCanonical; -use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -29,12 +26,10 @@ pub fn slice_canonical_array( array: &ArrayRef, start: usize, stop: usize, + ctx: &mut ExecutionCtx, ) -> VortexResult { let validity = if array.dtype().is_nullable() { - let bool_buff = array - .validity()? - .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? - .to_bit_buffer(); + let bool_buff = array.validity()?.to_mask(array.len(), ctx)?.to_bit_buffer(); Validity::from(bool_buff.slice(start..stop)) } else { Validity::NonNullable @@ -42,14 +37,12 @@ pub fn slice_canonical_array( match array.dtype() { DType::Bool(_) => { - #[expect(deprecated)] - let bool_array = array.to_bool(); + let bool_array = array.clone().execute::(ctx)?; let sliced_bools = bool_array.to_bit_buffer().slice(start..stop); Ok(BoolArray::new(sliced_bools, validity).into_array()) } DType::Primitive(p, _) => { - #[expect(deprecated)] - let primitive_array = array.to_primitive(); + let primitive_array = array.clone().execute::(ctx)?; match_each_native_ptype!(p, |P| { Ok(PrimitiveArray::new( primitive_array.to_buffer::