Skip to content

Commit 468030a

Browse files
gatesngatesn
committed
Slots
Signed-off-by: Nicholas Gates <nick@nickgates.com>
1 parent dc12213 commit 468030a

7 files changed

Lines changed: 59 additions & 197 deletions

File tree

‎vortex-array/public-api.lock‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1438,31 +1438,27 @@ pub fn vortex_array::arrays::decimal::DecimalData::buffer_handle(&self) -> &vort
14381438

14391439
pub fn vortex_array::arrays::decimal::DecimalData::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType
14401440

1441-
pub fn vortex_array::arrays::decimal::DecimalData::from_iter<T: vortex_array::dtype::NativeDecimalType, I: core::iter::traits::collect::IntoIterator<Item = T>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
1442-
1443-
pub fn vortex_array::arrays::decimal::DecimalData::from_option_iter<T: vortex_array::dtype::NativeDecimalType, I: core::iter::traits::collect::IntoIterator<Item = core::option::Option<T>>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
1444-
14451441
pub fn vortex_array::arrays::decimal::DecimalData::is_empty(&self) -> bool
14461442

14471443
pub fn vortex_array::arrays::decimal::DecimalData::len(&self) -> usize
14481444

1449-
pub fn vortex_array::arrays::decimal::DecimalData::new<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self
1445+
pub fn vortex_array::arrays::decimal::DecimalData::new<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
14501446

1451-
pub fn vortex_array::arrays::decimal::DecimalData::new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self
1447+
pub fn vortex_array::arrays::decimal::DecimalData::new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
14521448

1453-
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self
1449+
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
14541450

1455-
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_from_byte_buffer(byte_buffer: vortex_buffer::ByteBuffer, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self
1451+
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_from_byte_buffer(byte_buffer: vortex_buffer::ByteBuffer, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
14561452

1457-
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> Self
1453+
pub unsafe fn vortex_array::arrays::decimal::DecimalData::new_unchecked_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self
14581454

14591455
pub fn vortex_array::arrays::decimal::DecimalData::precision(&self) -> u8
14601456

14611457
pub fn vortex_array::arrays::decimal::DecimalData::scale(&self) -> i8
14621458

1463-
pub fn vortex_array::arrays::decimal::DecimalData::try_new<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult<Self>
1459+
pub fn vortex_array::arrays::decimal::DecimalData::try_new<T: vortex_array::dtype::NativeDecimalType>(buffer: vortex_buffer::buffer::Buffer<T>, decimal_dtype: vortex_array::dtype::DecimalDType) -> vortex_error::VortexResult<Self>
14641460

1465-
pub fn vortex_array::arrays::decimal::DecimalData::try_new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult<Self>
1461+
pub fn vortex_array::arrays::decimal::DecimalData::try_new_handle(values: vortex_array::buffer::BufferHandle, values_type: vortex_array::dtype::DecimalType, decimal_dtype: vortex_array::dtype::DecimalDType) -> vortex_error::VortexResult<Self>
14661462

14671463
pub fn vortex_array::arrays::decimal::DecimalData::values_type(&self) -> vortex_array::dtype::DecimalType
14681464

@@ -2344,7 +2340,7 @@ impl vortex_array::arrays::fixed_size_list::FixedSizeListData
23442340

23452341
pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self
23462342

2347-
pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self
2343+
pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(list_size: u32, len: usize) -> Self
23482344

23492345
pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()>
23502346

@@ -2490,7 +2486,7 @@ impl vortex_array::arrays::list::ListData
24902486

24912487
pub fn vortex_array::arrays::list::ListData::new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> Self
24922488

2493-
pub unsafe fn vortex_array::arrays::list::ListData::new_unchecked(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, _validity: vortex_array::validity::Validity) -> Self
2489+
pub unsafe fn vortex_array::arrays::list::ListData::new_unchecked() -> Self
24942490

24952491
pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()>
24962492

@@ -4012,7 +4008,7 @@ pub fn vortex_array::arrays::struct_::StructData::new(names: vortex_array::dtype
40124008

40134009
pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: usize) -> Self
40144010

4015-
pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(fields: impl core::convert::Into<alloc::sync::Arc<[vortex_array::ArrayRef]>>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self
4011+
pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(dtype: vortex_array::dtype::StructFields, fieldless_len: core::option::Option<usize>) -> Self
40164012

40174013
pub fn vortex_array::arrays::struct_::StructData::validate(fields: &[vortex_array::ArrayRef], dtype: &vortex_array::dtype::StructFields, length: usize, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()>
40184014

@@ -4214,9 +4210,9 @@ pub fn vortex_array::arrays::varbin::VarBinData::new(offsets: vortex_array::Arra
42144210

42154211
pub fn vortex_array::arrays::varbin::VarBinData::new_from_handle(offset: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self
42164212

4217-
pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self
4213+
pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked(bytes: vortex_buffer::ByteBuffer) -> Self
42184214

4219-
pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked_from_handle(offsets: vortex_array::ArrayRef, bytes: vortex_array::buffer::BufferHandle, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> Self
4215+
pub unsafe fn vortex_array::arrays::varbin::VarBinData::new_unchecked_from_handle(bytes: vortex_array::buffer::BufferHandle) -> Self
42204216

42214217
pub fn vortex_array::arrays::varbin::VarBinData::try_new(offsets: vortex_array::ArrayRef, bytes: vortex_buffer::ByteBuffer, dtype: vortex_array::dtype::DType, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult<Self>
42224218

‎vortex-array/src/arrays/decimal/array.rs‎

Lines changed: 28 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use itertools::Itertools;
5+
use vortex_buffer::Alignment;
56
use vortex_buffer::BitBufferMut;
67
use vortex_buffer::Buffer;
78
use vortex_buffer::BufferMut;
@@ -169,13 +170,8 @@ impl DecimalData {
169170
///
170171
/// Panics if the provided components do not satisfy the invariants documented in
171172
/// [`DecimalArray::new_unchecked`].
172-
pub fn new<T: NativeDecimalType>(
173-
buffer: Buffer<T>,
174-
decimal_dtype: DecimalDType,
175-
validity: Validity,
176-
) -> Self {
177-
Self::try_new(buffer, decimal_dtype, validity)
178-
.vortex_expect("DecimalArray construction failed")
173+
pub fn new<T: NativeDecimalType>(buffer: Buffer<T>, decimal_dtype: DecimalDType) -> Self {
174+
Self::try_new(buffer, decimal_dtype).vortex_expect("DecimalArray construction failed")
179175
}
180176

181177
/// Creates a new [`DecimalArray`] from a [`BufferHandle`] of values that may live in
@@ -189,9 +185,8 @@ impl DecimalData {
189185
values: BufferHandle,
190186
values_type: DecimalType,
191187
decimal_dtype: DecimalDType,
192-
validity: Validity,
193188
) -> Self {
194-
Self::try_new_handle(values, values_type, decimal_dtype, validity)
189+
Self::try_new_handle(values, values_type, decimal_dtype)
195190
.vortex_expect("DecimalArray construction failed")
196191
}
197192

@@ -206,12 +201,11 @@ impl DecimalData {
206201
pub fn try_new<T: NativeDecimalType>(
207202
buffer: Buffer<T>,
208203
decimal_dtype: DecimalDType,
209-
validity: Validity,
210204
) -> VortexResult<Self> {
211205
let values = BufferHandle::new_host(buffer.into_byte_buffer());
212206
let values_type = T::DECIMAL_TYPE;
213207

214-
Self::try_new_handle(values, values_type, decimal_dtype, validity)
208+
Self::try_new_handle(values, values_type, decimal_dtype)
215209
}
216210

217211
/// Constructs a new `DecimalArray` with validation from a [`BufferHandle`].
@@ -225,12 +219,11 @@ impl DecimalData {
225219
values: BufferHandle,
226220
values_type: DecimalType,
227221
decimal_dtype: DecimalDType,
228-
validity: Validity,
229222
) -> VortexResult<Self> {
230-
Self::validate(&values, values_type, &validity)?;
223+
Self::validate(&values, values_type)?;
231224

232225
// SAFETY: validate ensures all invariants are met.
233-
Ok(unsafe { Self::new_unchecked_handle(values, values_type, decimal_dtype, validity) })
226+
Ok(unsafe { Self::new_unchecked_handle(values, values_type, decimal_dtype) })
234227
}
235228

236229
/// Creates a new [`DecimalArray`] without validation from these components:
@@ -249,15 +242,13 @@ impl DecimalData {
249242
pub unsafe fn new_unchecked<T: NativeDecimalType>(
250243
buffer: Buffer<T>,
251244
decimal_dtype: DecimalDType,
252-
validity: Validity,
253245
) -> Self {
254246
// SAFETY: new_unchecked_handle inherits the safety guarantees of new_unchecked
255247
unsafe {
256248
Self::new_unchecked_handle(
257249
BufferHandle::new_host(buffer.into_byte_buffer()),
258250
T::DECIMAL_TYPE,
259251
decimal_dtype,
260-
validity,
261252
)
262253
}
263254
}
@@ -275,14 +266,7 @@ impl DecimalData {
275266
values: BufferHandle,
276267
values_type: DecimalType,
277268
decimal_dtype: DecimalDType,
278-
validity: Validity,
279269
) -> Self {
280-
#[cfg(debug_assertions)]
281-
{
282-
Self::validate(&values, values_type, &validity)
283-
.vortex_expect("[Debug Assertion]: Invalid `DecimalArray` parameters");
284-
}
285-
286270
Self {
287271
decimal_dtype,
288272
values,
@@ -293,21 +277,23 @@ impl DecimalData {
293277
/// Validates the components that would be used to create a [`DecimalArray`] from a byte buffer.
294278
///
295279
/// This function checks all the invariants required by [`DecimalArray::new_unchecked`].
296-
fn validate(
297-
buffer: &BufferHandle,
298-
values_type: DecimalType,
299-
validity: &Validity,
300-
) -> VortexResult<()> {
301-
if let Some(validity_len) = validity.maybe_len() {
302-
let expected_len = values_type.byte_width() * validity_len;
280+
fn validate(buffer: &BufferHandle, values_type: DecimalType) -> VortexResult<()> {
281+
let byte_width = values_type.byte_width();
282+
vortex_ensure!(
283+
buffer.len().is_multiple_of(byte_width),
284+
InvalidArgument: "decimal buffer size {} is not divisible by element width {}",
285+
buffer.len(),
286+
byte_width,
287+
);
288+
match_each_decimal_value_type!(values_type, |D| {
303289
vortex_ensure!(
304-
buffer.len() == expected_len,
305-
InvalidArgument: "expected buffer of size {} bytes, was {} bytes",
306-
expected_len,
307-
buffer.len(),
290+
buffer.is_aligned_to(Alignment::of::<D>()),
291+
InvalidArgument: "decimal buffer alignment {:?} is invalid for values type {:?}",
292+
buffer.alignment(),
293+
D::DECIMAL_TYPE,
308294
);
309-
}
310-
295+
Ok::<(), vortex_error::VortexError>(())
296+
})?;
311297
Ok(())
312298
}
313299

@@ -324,15 +310,13 @@ impl DecimalData {
324310
byte_buffer: ByteBuffer,
325311
values_type: DecimalType,
326312
decimal_dtype: DecimalDType,
327-
validity: Validity,
328313
) -> Self {
329314
// SAFETY: inherits the same safety contract as `new_unchecked_from_byte_buffer`
330315
unsafe {
331316
Self::new_unchecked_handle(
332317
BufferHandle::new_host(byte_buffer),
333318
values_type,
334319
decimal_dtype,
335-
validity,
336320
)
337321
}
338322
}
@@ -380,46 +364,6 @@ impl DecimalData {
380364
pub fn scale(&self) -> i8 {
381365
self.decimal_dtype.scale()
382366
}
383-
384-
pub fn from_iter<T: NativeDecimalType, I: IntoIterator<Item = T>>(
385-
iter: I,
386-
decimal_dtype: DecimalDType,
387-
) -> Self {
388-
let iter = iter.into_iter();
389-
390-
Self::new(
391-
BufferMut::from_iter(iter).freeze(),
392-
decimal_dtype,
393-
Validity::NonNullable,
394-
)
395-
}
396-
397-
pub fn from_option_iter<T: NativeDecimalType, I: IntoIterator<Item = Option<T>>>(
398-
iter: I,
399-
decimal_dtype: DecimalDType,
400-
) -> Self {
401-
let iter = iter.into_iter();
402-
let mut values = BufferMut::with_capacity(iter.size_hint().0);
403-
let mut validity = BitBufferMut::with_capacity(values.capacity());
404-
405-
for i in iter {
406-
match i {
407-
None => {
408-
validity.append(false);
409-
values.push(T::default());
410-
}
411-
Some(e) => {
412-
validity.append(true);
413-
values.push(e);
414-
}
415-
}
416-
}
417-
Self::new(
418-
values.freeze(),
419-
decimal_dtype,
420-
Validity::from(validity.freeze()),
421-
)
422-
}
423367
}
424368

425369
impl Array<Decimal> {
@@ -446,7 +390,7 @@ impl Array<Decimal> {
446390
let dtype = DType::Decimal(decimal_dtype, validity.nullability());
447391
let len = buffer.len();
448392
let slots = DecimalData::make_slots(&validity, len);
449-
let data = DecimalData::new(buffer, decimal_dtype, validity);
393+
let data = DecimalData::new(buffer, decimal_dtype);
450394
unsafe {
451395
Array::from_parts_unchecked(
452396
ArrayParts::new(Decimal, dtype, len, data).with_slots(slots),
@@ -467,7 +411,7 @@ impl Array<Decimal> {
467411
let dtype = DType::Decimal(decimal_dtype, validity.nullability());
468412
let len = buffer.len();
469413
let slots = DecimalData::make_slots(&validity, len);
470-
let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) };
414+
let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype) };
471415
unsafe {
472416
Array::from_parts_unchecked(
473417
ArrayParts::new(Decimal, dtype, len, data).with_slots(slots),
@@ -484,7 +428,7 @@ impl Array<Decimal> {
484428
let dtype = DType::Decimal(decimal_dtype, validity.nullability());
485429
let len = buffer.len();
486430
let slots = DecimalData::make_slots(&validity, len);
487-
let data = DecimalData::try_new(buffer, decimal_dtype, validity)?;
431+
let data = DecimalData::try_new(buffer, decimal_dtype)?;
488432
Ok(unsafe {
489433
Array::from_parts_unchecked(
490434
ArrayParts::new(Decimal, dtype, len, data).with_slots(slots),
@@ -547,7 +491,7 @@ impl Array<Decimal> {
547491
let dtype = DType::Decimal(decimal_dtype, validity.nullability());
548492
let len = values.len() / values_type.byte_width();
549493
let slots = DecimalData::make_slots(&validity, len);
550-
let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity);
494+
let data = DecimalData::new_handle(values, values_type, decimal_dtype);
551495
unsafe {
552496
Array::from_parts_unchecked(
553497
ArrayParts::new(Decimal, dtype, len, data).with_slots(slots),
@@ -569,9 +513,7 @@ impl Array<Decimal> {
569513
let dtype = DType::Decimal(decimal_dtype, validity.nullability());
570514
let len = values.len() / values_type.byte_width();
571515
let slots = DecimalData::make_slots(&validity, len);
572-
let data = unsafe {
573-
DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity)
574-
};
516+
let data = unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype) };
575517
unsafe {
576518
Array::from_parts_unchecked(
577519
ArrayParts::new(Decimal, dtype, len, data).with_slots(slots),
@@ -613,7 +555,6 @@ impl Array<Decimal> {
613555
patch_indices,
614556
offset,
615557
patch_values,
616-
patched_validity.clone(),
617558
)
618559
})
619560
})
@@ -683,7 +624,6 @@ fn patch_typed<I, ValuesDVT, PatchDVT>(
683624
patch_indices: &[I],
684625
patch_indices_offset: usize,
685626
patch_values: Buffer<PatchDVT>,
686-
patched_validity: Validity,
687627
) -> DecimalData
688628
where
689629
I: IntegerPType,
@@ -704,5 +644,5 @@ where
704644
);
705645
}
706646

707-
DecimalData::new(buffer.freeze(), decimal_dtype, patched_validity)
647+
DecimalData::new(buffer.freeze(), decimal_dtype)
708648
}

‎vortex-array/src/arrays/decimal/vtable/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ impl VTable for Decimal {
164164
"DecimalArray buffer not aligned for values type {:?}",
165165
D::DECIMAL_TYPE
166166
);
167-
DecimalData::try_new_handle(values, metadata.values_type(), *decimal_dtype, validity)
167+
DecimalData::try_new_handle(values, metadata.values_type(), *decimal_dtype)
168168
})?;
169169
Ok(crate::array::ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots))
170170
}

0 commit comments

Comments
 (0)