Skip to content

Commit 69ce351

Browse files
a10ya10y
committed
save
Signed-off-by: Andrew Duffy <andrew@a10y.dev>
1 parent fcbda04 commit 69ce351

File tree

9 files changed

+218
-180
lines changed

9 files changed

+218
-180
lines changed

‎encodings/fastlanes/src/bitpacking/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,8 +297,8 @@ impl VTable for BitPacked {
297297
})
298298
}
299299

300-
fn execute(array: &Self::Array, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
301-
Ok(ExecutionResult::done(unpack_array(array)?.into_array()))
300+
fn execute(array: Arc<Self::Array>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
301+
Ok(ExecutionResult::done(unpack_array(array.as_ref())?.into_array()))
302302
}
303303

304304
fn execute_parent(

‎encodings/fastlanes/src/delta/array/delta_compress.rs‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@ use std::mem::MaybeUninit;
77
use fastlanes::Delta;
88
use fastlanes::FastLanes;
99
use fastlanes::Transpose;
10-
use vortex_array::ExecutionCtx;
1110
use vortex_array::arrays::PrimitiveArray;
1211
use vortex_array::dtype::NativePType;
1312
use vortex_array::match_each_unsigned_integer_ptype;
1413
use vortex_array::vtable::ValidityHelper;
14+
use vortex_array::ExecutionCtx;
1515
use vortex_buffer::Buffer;
1616
use vortex_buffer::BufferMut;
1717
use vortex_error::VortexResult;
1818

19-
use crate::FL_CHUNK_SIZE;
2019
use crate::bit_transpose::transpose_validity;
2120
use crate::fill_forward_nulls;
21+
use crate::FL_CHUNK_SIZE;
2222

2323
pub fn delta_compress(
2424
array: &PrimitiveArray,
@@ -96,19 +96,19 @@ mod tests {
9696
use std::sync::LazyLock;
9797

9898
use rstest::rstest;
99-
use vortex_array::IntoArray;
100-
use vortex_array::ToCanonical;
101-
use vortex_array::VortexSessionExecute;
10299
use vortex_array::arrays::PrimitiveArray;
103100
use vortex_array::assert_arrays_eq;
104101
use vortex_array::session::ArraySession;
102+
use vortex_array::IntoArray;
103+
use vortex_array::ToCanonical;
104+
use vortex_array::VortexSessionExecute;
105105
use vortex_error::VortexResult;
106106
use vortex_session::VortexSession;
107107

108-
use crate::DeltaArray;
109-
use crate::bitpack_compress::bitpack_encode;
108+
use crate::bitpack_compress::BitPackedEncoder;
110109
use crate::delta::array::delta_decompress::delta_decompress;
111110
use crate::delta_compress;
111+
use crate::DeltaArray;
112112

113113
static SESSION: LazyLock<VortexSession> =
114114
LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
@@ -137,14 +137,14 @@ mod tests {
137137
(0u8..200).map(|i| (!(50..100).contains(&i)).then_some(i)),
138138
);
139139
let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap();
140-
let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap();
141-
let packed_delta = DeltaArray::try_new(
142-
bases.into_array(),
143-
bitpacked_deltas.into_array(),
144-
0,
145-
array.len(),
146-
)
147-
.unwrap();
140+
let bitpacked_deltas = BitPackedEncoder::new(&deltas)
141+
.with_bit_width(1)
142+
.pack()
143+
.unwrap()
144+
.into_array()
145+
.unwrap();
146+
let packed_delta =
147+
DeltaArray::try_new(bases.into_array(), bitpacked_deltas, 0, array.len()).unwrap();
148148
assert_arrays_eq!(packed_delta.to_primitive(), array);
149149
}
150150
}

‎vortex-array/src/arrays/patched/array.rs‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,19 @@ use crate::patches::Patches;
2727
use crate::stats::ArrayStats;
2828
use crate::validity::Validity;
2929

30+
/// Shredded components of the [`PatchedArray`].
31+
///
32+
/// This is created when you consume the arrary using [`PatchedArray::into_parts`].
33+
pub struct PatchedArrayParts {
34+
pub inner: ArrayRef,
35+
pub n_chunks: usize,
36+
pub n_lanes: usize,
37+
pub offset: usize,
38+
pub lane_offsets: BufferHandle,
39+
pub indices: BufferHandle,
40+
pub values: ArrayRef,
41+
}
42+
3043
/// An array that partially "patches" another array with new values.
3144
#[derive(Debug, Clone)]
3245
pub struct PatchedArray {
@@ -109,7 +122,20 @@ impl PatchedArray {
109122
})
110123
}
111124

112-
/// Get an accessor, which allows ranged access to patches by chunk/lane.
125+
/// Split the array into named components.
126+
pub fn into_parts(self) -> PatchedArrayParts {
127+
PatchedArrayParts {
128+
inner: self.inner,
129+
n_chunks: self.n_chunks,
130+
n_lanes: self.n_lanes,
131+
offset: self.offset,
132+
lane_offsets: self.lane_offsets,
133+
indices: self.indices,
134+
values: self.values,
135+
}
136+
}
137+
138+
/// Builds and returns an accessor for reading patch data.
113139
pub fn accessor(&self) -> PatchAccessor<'_> {
114140
PatchAccessor {
115141
n_lanes: self.n_lanes,

‎vortex-cuda/benches/dynamic_dispatch_cuda.rs‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,10 @@ use cudarc::driver::LaunchConfig;
1717
use cudarc::driver::PushKernelArg;
1818
use cudarc::driver::sys::CUevent_flags;
1919
use vortex::array::IntoArray;
20-
use vortex::array::ToCanonical;
21-
use vortex::array::arrays::DictArray;
2220
use vortex::array::arrays::PrimitiveArray;
23-
use vortex::array::scalar::Scalar;
2421
use vortex::array::validity::Validity::NonNullable;
2522
use vortex::buffer::Buffer;
2623
use vortex::dtype::PType;
27-
use vortex::encodings::alp::ALPArray;
28-
use vortex::encodings::alp::ALPFloat;
29-
use vortex::encodings::alp::Exponents;
30-
use vortex::encodings::alp::alp_encode;
31-
use vortex::encodings::fastlanes::BitPackedArray;
32-
use vortex::encodings::fastlanes::FoRArray;
3324
use vortex::encodings::runend::RunEndArray;
3425
use vortex::error::VortexExpect;
3526
use vortex::error::VortexResult;

‎vortex-cuda/src/dynamic_dispatch/mod.rs‎

Lines changed: 72 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,16 @@ impl DynamicDispatchPlan {
306306
mod tests {
307307
use std::sync::Arc;
308308

309+
use super::DynamicDispatchPlan;
310+
use super::SMEM_TILE_SIZE;
311+
use super::ScalarOp;
312+
use super::SourceOp;
313+
use super::Stage;
314+
use super::build_plan;
315+
use crate::CudaBufferExt;
316+
use crate::CudaDeviceBuffer;
317+
use crate::CudaExecutionCtx;
318+
use crate::session::CudaSession;
309319
use cudarc::driver::DevicePtr;
310320
use cudarc::driver::LaunchConfig;
311321
use cudarc::driver::PushKernelArg;
@@ -324,31 +334,26 @@ mod tests {
324334
use vortex::encodings::alp::alp_encode;
325335
use vortex::encodings::fastlanes::BitPackedArray;
326336
use vortex::encodings::fastlanes::FoRArray;
337+
use vortex::encodings::fastlanes::bitpack_compress::BitPackedEncoder;
327338
use vortex::encodings::runend::RunEndArray;
328339
use vortex::encodings::zigzag::ZigZagArray;
329340
use vortex::error::VortexExpect;
330341
use vortex::error::VortexResult;
331342
use vortex::session::VortexSession;
343+
use vortex_array::ArrayRef;
332344

333-
use super::DynamicDispatchPlan;
334-
use super::SMEM_TILE_SIZE;
335-
use super::ScalarOp;
336-
use super::SourceOp;
337-
use super::Stage;
338-
use super::build_plan;
339-
use crate::CudaBufferExt;
340-
use crate::CudaDeviceBuffer;
341-
use crate::CudaExecutionCtx;
342-
use crate::session::CudaSession;
343-
344-
fn make_bitpacked_array_u32(bit_width: u8, len: usize) -> BitPackedArray {
345+
fn make_bitpacked_array_u32(bit_width: u8, len: usize) -> ArrayRef {
345346
let max_val = (1u64 << bit_width).saturating_sub(1);
346347
let values: Vec<u32> = (0..len)
347348
.map(|i| ((i as u64) % (max_val + 1)) as u32)
348349
.collect();
349350
let primitive = PrimitiveArray::new(Buffer::from(values), NonNullable);
350-
BitPackedArray::encode(&primitive.into_array(), bit_width)
351-
.vortex_expect("failed to create BitPacked array")
351+
BitPackedEncoder::new(&primitive)
352+
.with_bit_width(bit_width)
353+
.pack()
354+
.unwrap()
355+
.into_array()
356+
.unwrap()
352357
}
353358

354359
#[crate::test]
@@ -548,7 +553,7 @@ mod tests {
548553

549554
let bp = make_bitpacked_array_u32(bit_width, len);
550555
let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?;
551-
let (plan, _bufs) = build_plan(&bp.into_array(), &cuda_ctx)?;
556+
let (plan, _bufs) = build_plan(&bp, &cuda_ctx)?;
552557

553558
let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan)?;
554559
assert_eq!(actual, expected);
@@ -569,7 +574,7 @@ mod tests {
569574
let expected: Vec<u32> = raw.iter().map(|&v| v + reference).collect();
570575

571576
let bp = make_bitpacked_array_u32(bit_width, len);
572-
let for_arr = FoRArray::try_new(bp.into_array(), Scalar::from(reference))?;
577+
let for_arr = FoRArray::try_new(bp, Scalar::from(reference))?;
573578

574579
let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?;
575580
let (plan, _bufs) = build_plan(&for_arr.into_array(), &cuda_ctx)?;
@@ -1063,57 +1068,57 @@ mod tests {
10631068
Ok(())
10641069
}
10651070

1066-
#[rstest]
1067-
#[case(0, 1024)]
1068-
#[case(0, 3000)]
1069-
#[case(0, 4096)]
1070-
#[case(500, 600)]
1071-
#[case(500, 1024)]
1072-
#[case(500, 2048)]
1073-
#[case(500, 4500)]
1074-
#[case(777, 3333)]
1075-
#[case(1024, 2048)]
1076-
#[case(1024, 4096)]
1077-
#[case(1500, 3500)]
1078-
#[case(2048, 4096)]
1079-
#[case(2500, 4500)]
1080-
#[case(3333, 4444)]
1081-
#[crate::test]
1082-
fn test_sliced_dict_for_bp_values_bp_codes(
1083-
#[case] slice_start: usize,
1084-
#[case] slice_end: usize,
1085-
) -> VortexResult<()> {
1086-
let dict_reference = 1_000_000u32;
1087-
let dict_residuals: Vec<u32> = (0..64).collect();
1088-
let dict_expected: Vec<u32> = dict_residuals.iter().map(|&r| r + dict_reference).collect();
1089-
let dict_size = dict_residuals.len();
1090-
1091-
let len = 5000;
1092-
let codes: Vec<u32> = (0..len).map(|i| (i % dict_size) as u32).collect();
1093-
let all_decoded: Vec<u32> = codes.iter().map(|&c| dict_expected[c as usize]).collect();
1094-
1095-
// BitPack+FoR the dict values
1096-
let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable);
1097-
let dict_bp = BitPackedArray::encode(&dict_prim.into_array(), 6)?;
1098-
let dict_for = FoRArray::try_new(dict_bp.into_array(), Scalar::from(dict_reference))?;
1099-
1100-
// BitPack the codes
1101-
let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable);
1102-
let codes_bp = BitPackedArray::encode(&codes_prim.into_array(), 6)?;
1103-
1104-
let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?;
1105-
1106-
let sliced = dict.into_array().slice(slice_start..slice_end)?;
1107-
let expected: Vec<u32> = all_decoded[slice_start..slice_end].to_vec();
1108-
1109-
let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?;
1110-
let (plan, _bufs) = build_plan(&sliced, &cuda_ctx)?;
1111-
1112-
let actual = run_dynamic_dispatch_plan(&cuda_ctx, expected.len(), &plan)?;
1113-
assert_eq!(actual, expected);
1114-
1115-
Ok(())
1116-
}
1071+
// #[rstest]
1072+
// #[case(0, 1024)]
1073+
// #[case(0, 3000)]
1074+
// #[case(0, 4096)]
1075+
// #[case(500, 600)]
1076+
// #[case(500, 1024)]
1077+
// #[case(500, 2048)]
1078+
// #[case(500, 4500)]
1079+
// #[case(777, 3333)]
1080+
// #[case(1024, 2048)]
1081+
// #[case(1024, 4096)]
1082+
// #[case(1500, 3500)]
1083+
// #[case(2048, 4096)]
1084+
// #[case(2500, 4500)]
1085+
// #[case(3333, 4444)]
1086+
// #[crate::test]
1087+
// fn test_sliced_dict_for_bp_values_bp_codes(
1088+
// #[case] slice_start: usize,
1089+
// #[case] slice_end: usize,
1090+
// ) -> VortexResult<()> {
1091+
// let dict_reference = 1_000_000u32;
1092+
// let dict_residuals: Vec<u32> = (0..64).collect();
1093+
// let dict_expected: Vec<u32> = dict_residuals.iter().map(|&r| r + dict_reference).collect();
1094+
// let dict_size = dict_residuals.len();
1095+
//
1096+
// let len = 5000;
1097+
// let codes: Vec<u32> = (0..len).map(|i| (i % dict_size) as u32).collect();
1098+
// let all_decoded: Vec<u32> = codes.iter().map(|&c| dict_expected[c as usize]).collect();
1099+
//
1100+
// // BitPack+FoR the dict values
1101+
// let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable);
1102+
// let dict_bp = BitPackedArray::encode(&dict_prim.into_array(), 6)?;
1103+
// let dict_for = FoRArray::try_new(dict_bp.into_array(), Scalar::from(dict_reference))?;
1104+
//
1105+
// // BitPack the codes
1106+
// let codes_prim = PrimitiveArray::new(Buffer::from(codes), NonNullable);
1107+
// let codes_bp = BitPackedArray::encode(&codes_prim.into_array(), 6)?;
1108+
//
1109+
// let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?;
1110+
//
1111+
// let sliced = dict.into_array().slice(slice_start..slice_end)?;
1112+
// let expected: Vec<u32> = all_decoded[slice_start..slice_end].to_vec();
1113+
//
1114+
// let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?;
1115+
// let (plan, _bufs) = build_plan(&sliced, &cuda_ctx)?;
1116+
//
1117+
// let actual = run_dynamic_dispatch_plan(&cuda_ctx, expected.len(), &plan)?;
1118+
// assert_eq!(actual, expected);
1119+
//
1120+
// Ok(())
1121+
// }
11171122

11181123
#[rstest]
11191124
#[case(0u32, 1u32, 100)]

‎vortex-cuda/src/dynamic_dispatch/plan_builder.rs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,10 +173,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool {
173173
return false;
174174
}
175175
if id == BitPacked::ID {
176-
if let Ok(a) = array.clone().try_into::<BitPacked>() {
177-
return a.patches().is_none();
178-
}
179-
return false;
176+
return true;
180177
}
181178
id == FoR::ID
182179
|| id == ZigZag::ID

0 commit comments

Comments
 (0)