Skip to content

Commit d0c58b8

Browse files
joseph-isaacsjoseph-isaacs
committed
fix
Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
1 parent 4b707d9 commit d0c58b8

11 files changed

Lines changed: 198 additions & 32 deletions

File tree

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/bool.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,18 @@ impl FlatLayoutFixture for BooleansFixture {
3232
let bools = BoolArray::from_iter([true, false, true, true, false]);
3333
let nullable_bools =
3434
BoolArray::from_iter([Some(true), None, Some(false), None, Some(true)]);
35+
let all_true = BoolArray::from_iter([true, true, true, true, true]);
36+
let all_false = BoolArray::from_iter([false, false, false, false, false]);
37+
let all_null: BoolArray = BoolArray::from_iter([None::<bool>, None, None, None, None]);
3538
let arr = StructArray::try_new(
36-
FieldNames::from(["flag", "nullable_flag"]),
37-
vec![bools.into_array(), nullable_bools.into_array()],
39+
FieldNames::from(["flag", "nullable_flag", "all_true", "all_false", "all_null"]),
40+
vec![
41+
bools.into_array(),
42+
nullable_bools.into_array(),
43+
all_true.into_array(),
44+
all_false.into_array(),
45+
all_null.into_array(),
46+
],
3847
5,
3948
Validity::NonNullable,
4049
)?;

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/chunked.rs‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,30 +22,44 @@ impl FlatLayoutFixture for ChunkedFixture {
2222
}
2323

2424
fn description(&self) -> &str {
25-
"ChunkedArray with 3 chunks of 1000 rows each containing deterministic u32 values"
25+
"ChunkedArray with variable-size chunks containing nullable data"
2626
}
2727

2828
fn expected_encodings(&self) -> Vec<ArrayId> {
2929
vec![Primitive::ID]
3030
}
3131

3232
fn build(&self) -> VortexResult<ArrayRef> {
33-
let value_gen = |chunk_idx| {
34-
let values: Vec<u32> = (0u32..1000).map(|i| chunk_idx * 1000 + i).collect();
35-
let primitives =
36-
PrimitiveArray::new(vortex_buffer::Buffer::from(values), Validity::NonNullable);
37-
Ok(StructArray::try_new(
38-
FieldNames::from(["id"]),
39-
vec![primitives.into_array()],
40-
1000,
41-
Validity::NonNullable,
42-
)?
43-
.into_array())
44-
};
45-
46-
Ok(
47-
ChunkedArray::from_iter((0u32..3).map(value_gen).collect::<VortexResult<Vec<_>>>()?)
48-
.into_array(),
49-
)
33+
// Variable chunk sizes: 500, 1000, 250
34+
let chunk_sizes: [u32; 3] = [500, 1000, 250];
35+
let mut offset = 0u32;
36+
37+
let chunks = chunk_sizes
38+
.iter()
39+
.map(|&size| {
40+
let ids: Vec<u32> = (offset..offset + size).collect();
41+
let nullable_vals = PrimitiveArray::from_option_iter(
42+
(offset..offset + size)
43+
.map(|i| if i % 7 == 0 { None } else { Some(i as i64 * 3) }),
44+
);
45+
offset += size;
46+
Ok(StructArray::try_new(
47+
FieldNames::from(["id", "nullable_val"]),
48+
vec![
49+
PrimitiveArray::new(
50+
vortex_buffer::Buffer::from(ids),
51+
Validity::NonNullable,
52+
)
53+
.into_array(),
54+
nullable_vals.into_array(),
55+
],
56+
size as usize,
57+
Validity::NonNullable,
58+
)?
59+
.into_array())
60+
})
61+
.collect::<VortexResult<Vec<_>>>()?;
62+
63+
Ok(ChunkedArray::from_iter(chunks).into_array())
5064
}
5165
}

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,20 +81,50 @@ impl FlatLayoutFixture for DateTimeFixture {
8181
TimeUnit::Seconds,
8282
);
8383

84+
// Timestamps in nanoseconds (i64)
85+
let ts_nanos = TemporalArray::new_timestamp(
86+
PrimitiveArray::new(
87+
buffer![
88+
1704067200000000000i64,
89+
1718451000000000000,
90+
1735689599000000000
91+
],
92+
Validity::NonNullable,
93+
)
94+
.into_array(),
95+
TimeUnit::Nanoseconds,
96+
None,
97+
);
98+
99+
// Timestamps with non-UTC timezone
100+
let ts_eastern = TemporalArray::new_timestamp(
101+
PrimitiveArray::new(
102+
buffer![1704067200i64, 1718451000, 1735689599],
103+
Validity::NonNullable,
104+
)
105+
.into_array(),
106+
TimeUnit::Seconds,
107+
Some(Arc::from("America/New_York")),
108+
);
109+
84110
let arr = StructArray::try_new(
85111
FieldNames::from([
86112
"ts_seconds",
87113
"ts_millis_tz",
88114
"ts_nullable_micros",
89115
"date_days",
90116
"time_seconds",
117+
"ts_nanos",
118+
"ts_eastern",
91119
]),
92120
vec![
93121
ts_seconds.into_array(),
94122
ts_millis_tz.into_array(),
95123
ts_nullable.into_array(),
96124
date_days.into_array(),
97125
time_secs.into_array(),
126+
ts_nanos.into_array(),
127+
ts_eastern.into_array(),
98128
],
99129
3,
100130
Validity::NonNullable,

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/fixed_size_list.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,21 @@ impl FlatLayoutFixture for FixedSizeListFixture {
4040
);
4141
let fsl = FixedSizeListArray::try_new(elements.into_array(), 3, Validity::NonNullable, 4)?;
4242

43+
// Nullable FSL: 4 vectors of 2 i32, second entry is null
44+
let nullable_elements = PrimitiveArray::new(
45+
buffer![10i32, 20, 0, 0, 50, 60, 70, 80],
46+
Validity::NonNullable,
47+
);
48+
let nullable_fsl = FixedSizeListArray::try_new(
49+
nullable_elements.into_array(),
50+
2,
51+
Validity::from_iter([true, false, true, true]),
52+
4,
53+
)?;
54+
4355
let arr = StructArray::try_new(
44-
FieldNames::from(["vectors"]),
45-
vec![fsl.into_array()],
56+
FieldNames::from(["vectors", "nullable_vectors"]),
57+
vec![fsl.into_array(), nullable_fsl.into_array()],
4658
4,
4759
Validity::NonNullable,
4860
)?;

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/list.rs‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,24 @@ impl FlatLayoutFixture for ListFixture {
5353
Validity::NonNullable,
5454
)?;
5555

56+
// Nullable list of i32: [[100,200], null, [], [300]]
57+
let nullable_elements =
58+
PrimitiveArray::new(buffer![100i32, 200, 300], Validity::NonNullable);
59+
let nullable_offsets =
60+
PrimitiveArray::new(buffer![0i64, 2, 2, 2, 3], Validity::NonNullable);
61+
let nullable_int_list = ListArray::try_new(
62+
nullable_elements.into_array(),
63+
nullable_offsets.into_array(),
64+
Validity::from_iter([true, false, true, true]),
65+
)?;
66+
5667
let arr = StructArray::try_new(
57-
FieldNames::from(["int_list", "str_list"]),
58-
vec![int_list.into_array(), str_list.into_array()],
68+
FieldNames::from(["int_list", "str_list", "nullable_int_list"]),
69+
vec![
70+
int_list.into_array(),
71+
str_list.into_array(),
72+
nullable_int_list.into_array(),
73+
],
5974
4,
6075
Validity::NonNullable,
6176
)?;

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,25 @@ impl FlatLayoutFixture for ListViewFixture {
5757
Validity::NonNullable,
5858
)?;
5959

60+
// Nullable ListView of i32: [[10,20], null, [30], [40,50,60]]
61+
let nullable_elements =
62+
PrimitiveArray::new(buffer![10i32, 20, 30, 40, 50, 60], Validity::NonNullable);
63+
let nullable_offsets = PrimitiveArray::new(buffer![0u32, 2, 2, 3], Validity::NonNullable);
64+
let nullable_sizes = PrimitiveArray::new(buffer![2u32, 0, 1, 3], Validity::NonNullable);
65+
let nullable_listview = ListViewArray::try_new(
66+
nullable_elements.into_array(),
67+
nullable_offsets.into_array(),
68+
nullable_sizes.into_array(),
69+
Validity::from_iter([true, false, true, true]),
70+
)?;
71+
6072
let arr = StructArray::try_new(
61-
FieldNames::from(["int_listview", "str_listview"]),
62-
vec![int_listview.into_array(), str_listview.into_array()],
73+
FieldNames::from(["int_listview", "str_listview", "nullable_int_listview"]),
74+
vec![
75+
int_listview.into_array(),
76+
str_listview.into_array(),
77+
nullable_listview.into_array(),
78+
],
6379
4,
6480
Validity::NonNullable,
6581
)?;

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/primitive.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ impl FlatLayoutFixture for PrimitivesFixture {
4141
"f32",
4242
"f64",
4343
"nullable_i32",
44+
"f32_special",
45+
"f64_special",
4446
]),
4547
vec![
4648
PrimitiveArray::new(buffer![0u8, 128, 255], Validity::NonNullable).into_array(),
@@ -65,6 +67,17 @@ impl FlatLayoutFixture for PrimitivesFixture {
6567
PrimitiveArray::new(buffer![f64::MIN, 0.0f64, f64::MAX], Validity::NonNullable)
6668
.into_array(),
6769
PrimitiveArray::from_option_iter([Some(1i32), None, Some(42)]).into_array(),
70+
// Special float values: NaN, infinities, negative zero, subnormal
71+
PrimitiveArray::new(
72+
buffer![f32::NAN, f32::INFINITY, f32::NEG_INFINITY],
73+
Validity::NonNullable,
74+
)
75+
.into_array(),
76+
PrimitiveArray::new(
77+
buffer![f64::NAN, -0.0f64, f64::MIN_POSITIVE / 2.0],
78+
Validity::NonNullable,
79+
)
80+
.into_array(),
6881
],
6982
3,
7083
Validity::NonNullable,

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/struct_nested.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,22 @@ impl FlatLayoutFixture for StructNestedFixture {
4141
Validity::NonNullable,
4242
)?;
4343

44+
// Nullable inner struct: second row is null at the struct level.
45+
let nullable_inner = StructArray::try_new(
46+
FieldNames::from(["c", "d"]),
47+
vec![
48+
PrimitiveArray::new(buffer![100i64, 0, 300], Validity::NonNullable).into_array(),
49+
PrimitiveArray::new(buffer![1.0f32, 0.0, 3.0], Validity::NonNullable).into_array(),
50+
],
51+
3,
52+
Validity::from_iter([true, false, true]),
53+
)?;
54+
4455
let arr = StructArray::try_new(
45-
FieldNames::from(["inner", "value"]),
56+
FieldNames::from(["inner", "nullable_inner", "value"]),
4657
vec![
4758
inner.into_array(),
59+
nullable_inner.into_array(),
4860
PrimitiveArray::new(buffer![1.1f64, 2.2, 3.3], Validity::NonNullable).into_array(),
4961
],
5062
3,

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/varbinview.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ impl FlatLayoutFixture for VarBinViewFixture {
2121
}
2222

2323
fn description(&self) -> &str {
24-
"VarBinView-encoded strings including empty, unicode, emoji, and a nullable column"
24+
"VarBinView-encoded strings including empty, unicode, emoji, long (>12 byte) strings, and a nullable column"
2525
}
2626

2727
fn expected_encodings(&self) -> Vec<ArrayId> {
@@ -36,9 +36,20 @@ impl FlatLayoutFixture for VarBinViewFixture {
3636
Some("world"),
3737
Some(""),
3838
]);
39+
// Strings >12 bytes exercise VarBinView's buffer-reference mechanism (out-of-line storage).
40+
let long_strings = VarBinViewArray::from_iter_str(vec![
41+
"short",
42+
"this string is definitely longer than twelve bytes",
43+
"another-long-string-that-exceeds-inline-limit",
44+
"ABCDEFGHIJKLMNOPQRSTUVWXYZ-0123456789",
45+
]);
3946
let arr = StructArray::try_new(
40-
FieldNames::from(["text", "nullable_text"]),
41-
vec![strings.into_array(), nullable_strings.into_array()],
47+
FieldNames::from(["text", "nullable_text", "long_text"]),
48+
vec![
49+
strings.into_array(),
50+
nullable_strings.into_array(),
51+
long_strings.into_array(),
52+
],
4253
4,
4354
Validity::NonNullable,
4455
)?;

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sequence.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ impl FlatLayoutFixture for SequenceFixture {
2323
}
2424

2525
fn description(&self) -> &str {
26-
"Arithmetic sequences (0,1,2,... and stepped) for Sequence encoding"
26+
"Arithmetic sequences (0,1,2,... and stepped) for Sequence encoding, including nullable"
2727
}
2828

2929
fn expected_encodings(&self) -> Vec<ArrayId> {
@@ -48,6 +48,8 @@ impl FlatLayoutFixture for SequenceFixture {
4848
)?;
4949
let small_negative_i16 =
5050
SequenceArray::try_new_typed::<i16>(1200, -2, Nullability::NonNullable, N)?;
51+
let nullable_i64 = SequenceArray::try_new_typed::<i64>(0, 2, Nullability::Nullable, N)?;
52+
let nullable_u32 = SequenceArray::try_new_typed::<u32>(100, 7, Nullability::Nullable, N)?;
5153

5254
let arr = StructArray::try_new(
5355
FieldNames::from([
@@ -60,6 +62,8 @@ impl FlatLayoutFixture for SequenceFixture {
6062
"zero_crossing",
6163
"near_overflow",
6264
"small_negative_i16",
65+
"nullable_i64",
66+
"nullable_u32",
6367
]),
6468
vec![
6569
row_ids.into_array(),
@@ -71,6 +75,8 @@ impl FlatLayoutFixture for SequenceFixture {
7175
zero_crossing.into_array(),
7276
near_overflow.into_array(),
7377
small_negative_i16.into_array(),
78+
nullable_i64.into_array(),
79+
nullable_u32.into_array(),
7480
],
7581
N,
7682
Validity::NonNullable,

0 commit comments

Comments
 (0)