Skip to content

Commit 533ce6a

Browse files
jiaqizhojiaqizho
committed
Add FlatLayout range read for sub-segment IO
When a FlatLayout has its array_tree metadata inlined in the footer, we can figure out exactly which bytes of the segment are needed for a given row range without any IO. This lets us issue a single small read instead of fetching the entire segment, which is a big win for point lookups and narrow scans on wide tables. The range read planner walks the encoding tree via `VTable::plan_range_read`, where each encoding (Primitive, Bool, BitPacked, Delta, FoR, ZigZag, ALP, ALPRD, Dict, FixedSizeList, Constant, Null, Sequence, ByteBool, DateTimeParts, DecimalByteParts) declares its own buffer sub-ranges and child recursion strategy. If the resulting byte range is less than 50% of the full segment, we issue the targeted read; otherwise we fall back to reading the whole segment. To make Delta work with sub-ranged buffers, Delta::build() now derives child array lengths from `len + offset` instead of metadata.deltas_len. On disk, offset is always 0 so this is a no-op for the normal decode path, but it lets the range read pass a smaller decode_len without the decoder panicking on buffer size mismatch. Also adds `request_range()` to the SegmentSource trait with a default fallback implementation, efficient overrides in FileSegmentSource and BufferSegmentSource, a `RangeReadEnabled` session flag, and `ScanBuilder::with_split_row_indices` to generate per-index tight ranges for point lookups. Signed-off-by: jiaqizho <jiaqi.zhou@zilliz.com>
1 parent 2a0d1b7 commit 533ce6a

30 files changed

Lines changed: 1857 additions & 24 deletions

File tree

‎Cargo.lock‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎encodings/alp/src/alp/array.rs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -25,6 +26,9 @@ use vortex_array::stats::ArrayStats;
2526
use vortex_array::stats::StatsSetRef;
2627
use vortex_array::vtable;
2728
use vortex_array::vtable::ArrayId;
29+
use vortex_array::vtable::ChildRangeRead;
30+
use vortex_array::vtable::EncodingRangeRead;
31+
use vortex_array::vtable::RangeDecodeInfo;
2832
use vortex_array::vtable::VTable;
2933
use vortex_array::vtable::ValidityChild;
3034
use vortex_array::vtable::ValidityVTableFromChild;
@@ -258,6 +262,38 @@ impl VTable for ALP {
258262
) -> VortexResult<Option<ArrayRef>> {
259263
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
260264
}
265+
266+
fn plan_range_read(
267+
metadata: &ProstMetadata<ALPMetadata>,
268+
row_range: Range<usize>,
269+
row_count: usize,
270+
dtype: &DType,
271+
) -> Option<EncodingRangeRead> {
272+
// Patches cannot be safely sub-ranged (global indices).
273+
if metadata.0.patches.is_some() {
274+
return None;
275+
}
276+
277+
// Child 0 = encoded values (f32→i32 or f64→i64).
278+
let child_dtype = match dtype {
279+
DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n),
280+
DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n),
281+
_ => return None,
282+
};
283+
284+
Some(EncodingRangeRead {
285+
buffer_sub_ranges: vec![],
286+
children: vec![ChildRangeRead::Recurse {
287+
row_range,
288+
row_count,
289+
dtype: child_dtype,
290+
}],
291+
decode_info: RangeDecodeInfo::FromChild {
292+
child_idx: 0,
293+
divisor: 1,
294+
},
295+
})
296+
}
261297
}
262298

263299
#[derive(Clone, Debug)]

‎encodings/alp/src/alp_rd/array.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use itertools::Itertools;
89
use vortex_array::ArrayEq;
@@ -29,6 +30,9 @@ use vortex_array::stats::StatsSetRef;
2930
use vortex_array::validity::Validity;
3031
use vortex_array::vtable;
3132
use vortex_array::vtable::ArrayId;
33+
use vortex_array::vtable::ChildRangeRead;
34+
use vortex_array::vtable::EncodingRangeRead;
35+
use vortex_array::vtable::RangeDecodeInfo;
3236
use vortex_array::vtable::VTable;
3337
use vortex_array::vtable::ValidityChild;
3438
use vortex_array::vtable::ValidityVTableFromChild;
@@ -354,6 +358,53 @@ impl VTable for ALPRD {
354358
) -> VortexResult<Option<ArrayRef>> {
355359
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
356360
}
361+
362+
fn plan_range_read(
363+
metadata: &ProstMetadata<ALPRDMetadata>,
364+
row_range: Range<usize>,
365+
row_count: usize,
366+
dtype: &DType,
367+
) -> Option<EncodingRangeRead> {
368+
// Patches cannot be safely sub-ranged (global indices).
369+
if metadata.0.patches.is_some() {
370+
return None;
371+
}
372+
373+
let left_parts_ptype = PType::try_from(metadata.0.left_parts_ptype).ok()?;
374+
let left_parts_dtype = DType::Primitive(left_parts_ptype, dtype.nullability());
375+
376+
let right_parts_dtype = match dtype {
377+
DType::Primitive(PType::F32, _) => {
378+
DType::Primitive(PType::U32, Nullability::NonNullable)
379+
}
380+
DType::Primitive(PType::F64, _) => {
381+
DType::Primitive(PType::U64, Nullability::NonNullable)
382+
}
383+
_ => return None,
384+
};
385+
386+
Some(EncodingRangeRead {
387+
buffer_sub_ranges: vec![],
388+
children: vec![
389+
// Child 0 = left_parts.
390+
ChildRangeRead::Recurse {
391+
row_range: row_range.clone(),
392+
row_count,
393+
dtype: left_parts_dtype,
394+
},
395+
// Child 1 = right_parts.
396+
ChildRangeRead::Recurse {
397+
row_range,
398+
row_count,
399+
dtype: right_parts_dtype,
400+
},
401+
],
402+
decode_info: RangeDecodeInfo::FromChild {
403+
child_idx: 0,
404+
divisor: 1,
405+
},
406+
})
407+
}
357408
}
358409

359410
#[derive(Clone, Debug)]

‎encodings/bytebool/src/array.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -22,7 +23,10 @@ use vortex_array::stats::StatsSetRef;
2223
use vortex_array::validity::Validity;
2324
use vortex_array::vtable;
2425
use vortex_array::vtable::ArrayId;
26+
use vortex_array::vtable::BufferSubRange;
27+
use vortex_array::vtable::EncodingRangeRead;
2528
use vortex_array::vtable::OperationsVTable;
29+
use vortex_array::vtable::RangeDecodeInfo;
2630
use vortex_array::vtable::VTable;
2731
use vortex_array::vtable::ValidityHelper;
2832
use vortex_array::vtable::ValidityVTableFromValidityHelper;
@@ -199,6 +203,23 @@ impl VTable for ByteBool {
199203
) -> VortexResult<Option<ArrayRef>> {
200204
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
201205
}
206+
207+
fn plan_range_read(
208+
_metadata: &EmptyMetadata,
209+
row_range: Range<usize>,
210+
_row_count: usize,
211+
_dtype: &DType,
212+
) -> Option<EncodingRangeRead> {
213+
// 1 byte per boolean value
214+
Some(EncodingRangeRead {
215+
buffer_sub_ranges: vec![BufferSubRange::Range(row_range.start..row_range.end)],
216+
children: vec![],
217+
decode_info: RangeDecodeInfo::Leaf {
218+
decode_len: row_range.len(),
219+
post_slice: None,
220+
},
221+
})
222+
}
202223
}
203224

204225
#[derive(Clone, Debug)]

‎encodings/datetime-parts/src/array.rs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use std::fmt::Debug;
55
use std::hash::Hash;
6+
use std::ops::Range;
67

78
use vortex_array::ArrayEq;
89
use vortex_array::ArrayHash;
@@ -24,6 +25,9 @@ use vortex_array::stats::ArrayStats;
2425
use vortex_array::stats::StatsSetRef;
2526
use vortex_array::vtable;
2627
use vortex_array::vtable::ArrayId;
28+
use vortex_array::vtable::ChildRangeRead;
29+
use vortex_array::vtable::EncodingRangeRead;
30+
use vortex_array::vtable::RangeDecodeInfo;
2731
use vortex_array::vtable::VTable;
2832
use vortex_array::vtable::ValidityChild;
2933
use vortex_array::vtable::ValidityVTableFromChild;
@@ -244,6 +248,45 @@ impl VTable for DateTimeParts {
244248
) -> VortexResult<Option<ArrayRef>> {
245249
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
246250
}
251+
252+
fn plan_range_read(
253+
metadata: &ProstMetadata<DateTimePartsMetadata>,
254+
row_range: Range<usize>,
255+
row_count: usize,
256+
dtype: &DType,
257+
) -> Option<EncodingRangeRead> {
258+
let days_ptype = metadata.0.get_days_ptype().ok()?;
259+
let seconds_ptype = metadata.0.get_seconds_ptype().ok()?;
260+
let subseconds_ptype = metadata.0.get_subseconds_ptype().ok()?;
261+
262+
Some(EncodingRangeRead {
263+
buffer_sub_ranges: vec![],
264+
children: vec![
265+
// Child 0 = days (carries validity from parent dtype).
266+
ChildRangeRead::Recurse {
267+
row_range: row_range.clone(),
268+
row_count,
269+
dtype: DType::Primitive(days_ptype, dtype.nullability()),
270+
},
271+
// Child 1 = seconds (always non-nullable).
272+
ChildRangeRead::Recurse {
273+
row_range: row_range.clone(),
274+
row_count,
275+
dtype: DType::Primitive(seconds_ptype, Nullability::NonNullable),
276+
},
277+
// Child 2 = subseconds (always non-nullable).
278+
ChildRangeRead::Recurse {
279+
row_range,
280+
row_count,
281+
dtype: DType::Primitive(subseconds_ptype, Nullability::NonNullable),
282+
},
283+
],
284+
decode_info: RangeDecodeInfo::FromChild {
285+
child_idx: 0,
286+
divisor: 1,
287+
},
288+
})
289+
}
247290
}
248291

249292
#[derive(Clone, Debug)]

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ mod rules;
66
mod slice;
77

88
use std::hash::Hash;
9+
use std::ops::Range;
910

1011
use prost::Message as _;
1112
use vortex_array::ArrayEq;
@@ -33,7 +34,10 @@ use vortex_array::stats::ArrayStats;
3334
use vortex_array::stats::StatsSetRef;
3435
use vortex_array::vtable;
3536
use vortex_array::vtable::ArrayId;
37+
use vortex_array::vtable::ChildRangeRead;
38+
use vortex_array::vtable::EncodingRangeRead;
3639
use vortex_array::vtable::OperationsVTable;
40+
use vortex_array::vtable::RangeDecodeInfo;
3741
use vortex_array::vtable::VTable;
3842
use vortex_array::vtable::ValidityChild;
3943
use vortex_array::vtable::ValidityHelper;
@@ -202,6 +206,27 @@ impl VTable for DecimalByteParts {
202206
) -> VortexResult<Option<ArrayRef>> {
203207
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
204208
}
209+
210+
fn plan_range_read(
211+
metadata: &ProstMetadata<DecimalBytesPartsMetadata>,
212+
row_range: Range<usize>,
213+
row_count: usize,
214+
dtype: &DType,
215+
) -> Option<EncodingRangeRead> {
216+
let child_dtype = DType::Primitive(metadata.zeroth_child_ptype(), dtype.nullability());
217+
Some(EncodingRangeRead {
218+
buffer_sub_ranges: vec![],
219+
children: vec![ChildRangeRead::Recurse {
220+
row_range,
221+
row_count,
222+
dtype: child_dtype,
223+
}],
224+
decode_info: RangeDecodeInfo::FromChild {
225+
child_idx: 0,
226+
divisor: 1,
227+
},
228+
})
229+
}
205230
}
206231

207232
/// This array encodes decimals as between 1-4 columns of primitive typed children.

‎encodings/fastlanes/src/bitpacking/vtable/mod.rs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::hash::Hash;
5+
use std::ops::Range;
56

67
use vortex_array::ArrayEq;
78
use vortex_array::ArrayHash;
@@ -25,6 +26,9 @@ use vortex_array::stats::StatsSetRef;
2526
use vortex_array::validity::Validity;
2627
use vortex_array::vtable;
2728
use vortex_array::vtable::ArrayId;
29+
use vortex_array::vtable::BufferSubRange;
30+
use vortex_array::vtable::EncodingRangeRead;
31+
use vortex_array::vtable::RangeDecodeInfo;
2832
use vortex_array::vtable::VTable;
2933
use vortex_array::vtable::ValidityVTableFromValidityHelper;
3034
use vortex_array::vtable::patches_child;
@@ -366,6 +370,44 @@ impl VTable for BitPacked {
366370
) -> VortexResult<Option<ArrayRef>> {
367371
PARENT_KERNELS.execute(array, parent, child_idx, ctx)
368372
}
373+
374+
fn plan_range_read(
375+
metadata: &ProstMetadata<BitPackedMetadata>,
376+
row_range: Range<usize>,
377+
_row_count: usize,
378+
_dtype: &DType,
379+
) -> Option<EncodingRangeRead> {
380+
let bit_width = metadata.0.bit_width as usize;
381+
382+
if bit_width == 0 || metadata.0.offset != 0 {
383+
return None;
384+
}
385+
386+
// Patches and validity children not supported for range read.
387+
if metadata.0.patches.is_some() {
388+
return None;
389+
}
390+
391+
let first_block = row_range.start / 1024;
392+
let last_block = row_range.end.saturating_sub(1) / 1024;
393+
let bytes_per_block = 128 * bit_width;
394+
let byte_start = first_block * bytes_per_block;
395+
let byte_end = (last_block + 1) * bytes_per_block;
396+
397+
let block_start_row = first_block * 1024;
398+
let decode_len = row_range.end - block_start_row;
399+
let intra_block_offset = row_range.start - block_start_row;
400+
let post_slice = (intra_block_offset > 0).then_some(intra_block_offset..decode_len);
401+
402+
Some(EncodingRangeRead {
403+
buffer_sub_ranges: vec![BufferSubRange::Range(byte_start..byte_end)],
404+
children: vec![],
405+
decode_info: RangeDecodeInfo::Leaf {
406+
decode_len,
407+
post_slice,
408+
},
409+
})
410+
}
369411
}
370412

371413
#[derive(Debug)]

0 commit comments

Comments
 (0)