Skip to content

Commit 3ba3236

Browse files
connortsui20connortsui20
committed
Unify primitive and decimal physical storage
Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
1 parent b7b01d3 commit 3ba3236

99 files changed

Lines changed: 1096 additions & 941 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ fn to_canonical_decimal(
285285
// SAFETY: The primitive array's buffer is already validated with correct type.
286286
// The decimal dtype matches the array's dtype, and validity is preserved.
287287
unsafe {
288-
DecimalArray::new_unchecked(
288+
DecimalArray::new_decimal_unchecked(
289289
prim.to_buffer::<P>(),
290290
*array
291291
.dtype()

‎encodings/sparse/src/canonical.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -535,7 +535,7 @@ fn execute_sparse_decimal<D: NativeDecimalType>(
535535
}
536536
}
537537
let filled_array = builder.finish_into_decimal();
538-
Ok(filled_array.patch(patches, ctx)?.into_array())
538+
Ok(filled_array.patch_decimal(patches, ctx)?.into_array())
539539
}
540540

541541
fn execute_varbin(
@@ -856,7 +856,7 @@ mod test {
856856
let mut ctx = SESSION.create_execution_ctx();
857857
let indices = buffer![0u32, 1u32, 7u32, 8u32].into_array();
858858
let decimal_dtype = DecimalDType::new(3, 2);
859-
let patch_values = DecimalArray::new(
859+
let patch_values = DecimalArray::new_decimal(
860860
buffer![100i128, 200i128, 300i128, 4000i128],
861861
decimal_dtype,
862862
Validity::from_iter([true, true, true, false]),
@@ -869,7 +869,7 @@ mod test {
869869
let expected = SESSION
870870
.arrow()
871871
.execute_arrow(
872-
DecimalArray::new(
872+
DecimalArray::new_decimal(
873873
buffer![100i128, 200, 123, 123, 123, 123, 123, 300, 4000, 123],
874874
decimal_dtype,
875875
// NB: patch indices: [0, 1, 7, 8]; patch validity: [Valid, Valid, Valid, Invalid]; ergo 0, 1, 7 are valid.

‎fuzz/src/array/fill_null.rs‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use vortex_array::arrays::DecimalArray;
1313
use vortex_array::arrays::PrimitiveArray;
1414
use vortex_array::arrays::VarBinViewArray;
1515
use vortex_array::arrays::bool::BoolArrayExt;
16+
use vortex_array::arrays::decimal::DecimalArrayExt;
1617
use vortex_array::builtins::ArrayBuiltins;
1718
use vortex_array::dtype::DType;
1819
use vortex_array::dtype::Nullability;
@@ -161,7 +162,7 @@ fn fill_decimal_array(
161162
.validity()
162163
.vortex_expect("decimal validity should be derivable in fuzz baseline")
163164
{
164-
Validity::NonNullable | Validity::AllValid => DecimalArray::new(
165+
Validity::NonNullable | Validity::AllValid => DecimalArray::new_decimal(
165166
array.buffer::<D>(),
166167
decimal_dtype,
167168
result_nullability.into(),
@@ -186,9 +187,13 @@ fn fill_decimal_array(
186187
}
187188
}
188189

189-
DecimalArray::try_new(new_data.freeze(), decimal_dtype, result_nullability.into())
190-
.vortex_expect("DecimalArray creation should succeed in fuzz test")
191-
.into_array()
190+
DecimalArray::try_new_decimal(
191+
new_data.freeze(),
192+
decimal_dtype,
193+
result_nullability.into(),
194+
)
195+
.vortex_expect("DecimalArray creation should succeed in fuzz test")
196+
.into_array()
192197
}
193198
}
194199
})
@@ -412,7 +417,7 @@ mod tests {
412417
#[test]
413418
fn test_fill_null_decimal_i32() {
414419
let mut ctx = array_session().create_execution_ctx();
415-
let array = DecimalArray::from_option_iter(
420+
let array = DecimalArray::from_decimal_option_iter(
416421
[Some(100i32), None, Some(300i32), None, Some(500i32)],
417422
DecimalDType::new(10, 2),
418423
);
@@ -424,7 +429,7 @@ mod tests {
424429

425430
let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap();
426431

427-
let expected = DecimalArray::from_iter(
432+
let expected = DecimalArray::from_decimal_iter(
428433
[100i32, 999i32, 300i32, 999i32, 500i32],
429434
DecimalDType::new(10, 2),
430435
);
@@ -434,7 +439,7 @@ mod tests {
434439
#[test]
435440
fn test_fill_null_decimal_i64() {
436441
let mut ctx = array_session().create_execution_ctx();
437-
let array = DecimalArray::from_option_iter(
442+
let array = DecimalArray::from_decimal_option_iter(
438443
[Some(1000i64), None, Some(3000i64)],
439444
DecimalDType::new(15, 3),
440445
);
@@ -447,14 +452,14 @@ mod tests {
447452
let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap();
448453

449454
let expected =
450-
DecimalArray::from_iter([1000i64, 9999i64, 3000i64], DecimalDType::new(15, 3));
455+
DecimalArray::from_decimal_iter([1000i64, 9999i64, 3000i64], DecimalDType::new(15, 3));
451456
assert_arrays_eq!(expected, result, &mut ctx);
452457
}
453458

454459
#[test]
455460
fn test_fill_null_decimal_i128() {
456461
let mut ctx = array_session().create_execution_ctx();
457-
let array = DecimalArray::from_option_iter(
462+
let array = DecimalArray::from_decimal_option_iter(
458463
[Some(10000i128), None, Some(30000i128), None],
459464
DecimalDType::new(20, 4),
460465
);
@@ -466,7 +471,7 @@ mod tests {
466471

467472
let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap();
468473

469-
let expected = DecimalArray::from_iter(
474+
let expected = DecimalArray::from_decimal_iter(
470475
[10000i128, 99999i128, 30000i128, 99999i128],
471476
DecimalDType::new(20, 4),
472477
);
@@ -476,8 +481,10 @@ mod tests {
476481
#[test]
477482
fn test_fill_null_decimal_all_invalid() {
478483
let mut ctx = array_session().create_execution_ctx();
479-
let array =
480-
DecimalArray::from_option_iter([None::<i64>, None, None], DecimalDType::new(10, 2));
484+
let array = DecimalArray::from_decimal_option_iter(
485+
[None::<i64>, None, None],
486+
DecimalDType::new(10, 2),
487+
);
481488
let fill_value = Scalar::decimal(
482489
DecimalValue::I64(777i64),
483490
DecimalDType::new(10, 2),
@@ -486,7 +493,7 @@ mod tests {
486493

487494
let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap();
488495

489-
let expected = DecimalArray::from_option_iter(
496+
let expected = DecimalArray::from_decimal_option_iter(
490497
[Some(777i64), Some(777i64), Some(777i64)],
491498
DecimalDType::new(10, 2),
492499
)
@@ -499,7 +506,7 @@ mod tests {
499506
#[test]
500507
fn test_fill_null_decimal_no_nulls() {
501508
let mut ctx = array_session().create_execution_ctx();
502-
let array = DecimalArray::from_option_iter(
509+
let array = DecimalArray::from_decimal_option_iter(
503510
[Some(100i32), Some(200i32), Some(300i32)],
504511
DecimalDType::new(10, 2),
505512
);
@@ -511,7 +518,7 @@ mod tests {
511518

512519
let result = fill_null_canonical_array(canonical(array), &fill_value, &mut ctx).unwrap();
513520

514-
let expected = DecimalArray::from_option_iter(
521+
let expected = DecimalArray::from_decimal_option_iter(
515522
[Some(100i32), Some(200i32), Some(300i32)],
516523
DecimalDType::new(10, 2),
517524
)

‎fuzz/src/array/filter.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ pub fn filter_canonical_array(
7474
let decimal_array = array.clone().execute::<DecimalArray>(ctx)?;
7575
match_each_decimal_value_type!(decimal_array.values_type(), |D| {
7676
let buf = decimal_array.buffer::<D>();
77-
Ok(DecimalArray::new(
77+
Ok(DecimalArray::new_decimal(
7878
filter
7979
.iter()
8080
.zip(buf.as_slice().iter().copied())

‎fuzz/src/array/mask.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use vortex_array::arrays::PrimitiveArray;
1616
use vortex_array::arrays::StructArray;
1717
use vortex_array::arrays::VarBinViewArray;
1818
use vortex_array::arrays::bool::BoolArrayExt;
19+
use vortex_array::arrays::decimal::DecimalArrayExt;
1920
use vortex_array::arrays::extension::ExtensionArrayExt;
2021
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
2122
use vortex_array::arrays::listview::ListViewArrayExt;
@@ -235,16 +236,18 @@ mod tests {
235236
fn test_mask_decimal_array() {
236237
let mut ctx = array_session().create_execution_ctx();
237238
let dtype = DecimalDType::new(10, 2);
238-
let array = DecimalArray::from_option_iter(
239+
let array = DecimalArray::from_decimal_option_iter(
239240
[Some(1i128), Some(2), Some(3), Some(4), Some(5)],
240241
dtype,
241242
);
242243
let mask = Mask::from_iter([true, true, false, true, true]);
243244

244245
let result = mask_canonical_array(canonical(array, &mut ctx), &mask, &mut ctx).unwrap();
245246

246-
let expected =
247-
DecimalArray::from_option_iter([Some(1i128), Some(2), None, Some(4), Some(5)], dtype);
247+
let expected = DecimalArray::from_decimal_option_iter(
248+
[Some(1i128), Some(2), None, Some(4), Some(5)],
249+
dtype,
250+
);
248251
assert_arrays_eq!(result, expected, &mut ctx);
249252
}
250253

‎fuzz/src/array/scalar_at.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use vortex_array::Canonical;
77
use vortex_array::ExecutionCtx;
88
use vortex_array::IntoArray;
99
use vortex_array::arrays::bool::BoolArrayExt;
10+
use vortex_array::arrays::decimal::DecimalArrayExt;
1011
use vortex_array::arrays::extension::ExtensionArrayExt;
1112
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
1213
use vortex_array::arrays::listview::ListViewArrayExt;

‎fuzz/src/array/slice.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ pub fn slice_canonical_array(
5757
let decimal_array = array.clone().execute::<DecimalArray>(ctx)?;
5858
Ok(
5959
match_each_decimal_value_type!(decimal_array.values_type(), |D| {
60-
DecimalArray::new(
60+
DecimalArray::new_decimal(
6161
decimal_array.buffer::<D>().slice(start..stop),
6262
*decimal_dtype,
6363
validity,

‎fuzz/src/array/sort.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR
8080
.map(|(p, v)| v.then_some(p))
8181
.collect::<Vec<_>>();
8282
opt_values.sort();
83-
Ok(DecimalArray::from_option_iter(opt_values, *d).into_array())
83+
Ok(DecimalArray::from_decimal_option_iter(opt_values, *d).into_array())
8484
})
8585
}
8686
DType::Utf8(_) | DType::Binary(_) => {

‎fuzz/src/array/take.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ fn take_decimal<D: NativeDecimalType>(
178178
) -> ArrayRef {
179179
let buf = array.buffer::<D>();
180180
let vec_values = buf.as_slice();
181-
DecimalArray::new(
181+
DecimalArray::new_decimal(
182182
indices
183183
.iter()
184184
.map(|i| vec_values[*i])

‎vortex-array/benches/cast_decimal.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ fn decimal_array(n: usize, precision: u8, scale: i8, nullable: bool) -> ArrayRef
4949
} else {
5050
Validity::NonNullable
5151
};
52-
DecimalArray::new(
52+
DecimalArray::new_decimal(
5353
values.freeze(),
5454
DecimalDType::new(precision, scale),
5555
validity,

0 commit comments

Comments
 (0)