Skip to content

Commit 95befa7

Browse files
feat: fsst inline varbin child (#7316)
The FSST arrays used to have a children that MUST be a VarBin. However this doesn't work with slots. Instead the FSST array has a offsets child and a bytes buffer. This is not a break its backwards compat --------- Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent b9c47cf commit 95befa7

File tree

8 files changed

+261
-123
lines changed

8 files changed

+261
-123
lines changed

encodings/fsst/public-api.lock

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@ pub struct vortex_fsst::FSSTData
8484

8585
impl vortex_fsst::FSSTData
8686

87-
pub fn vortex_fsst::FSSTData::codes(&self) -> &vortex_array::arrays::varbin::vtable::VarBinArray
87+
pub fn vortex_fsst::FSSTData::codes_bytes(&self) -> &vortex_buffer::ByteBuffer
8888

89-
pub fn vortex_fsst::FSSTData::codes_dtype(&self) -> &vortex_array::dtype::DType
89+
pub fn vortex_fsst::FSSTData::codes_bytes_handle(&self) -> &vortex_array::buffer::BufferHandle
9090

9191
pub fn vortex_fsst::FSSTData::compressor(&self) -> &fsst::Compressor
9292

@@ -100,7 +100,7 @@ pub fn vortex_fsst::FSSTData::symbol_lengths(&self) -> &vortex_buffer::buffer::B
100100

101101
pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer<fsst::Symbol>
102102

103-
pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer<fsst::Symbol>, symbol_lengths: vortex_buffer::buffer::Buffer<u8>, codes: vortex_array::arrays::varbin::vtable::VarBinArray, _dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
103+
pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer<fsst::Symbol>, symbol_lengths: vortex_buffer::buffer::Buffer<u8>, codes_bytes: vortex_array::buffer::BufferHandle, len: usize) -> vortex_error::VortexResult<Self>
104104

105105
pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize, slots: &[core::option::Option<vortex_array::array::erased::ArrayRef>]) -> vortex_error::VortexResult<()>
106106

@@ -156,12 +156,20 @@ pub fn vortex_fsst::FSSTMetadata::encoded_len(&self) -> usize
156156

157157
pub trait vortex_fsst::FSSTArrayExt: vortex_array::array::typed::TypedArrayRef<vortex_fsst::FSST>
158158

159+
pub fn vortex_fsst::FSSTArrayExt::codes(&self) -> vortex_array::arrays::varbin::vtable::VarBinArray
160+
161+
pub fn vortex_fsst::FSSTArrayExt::codes_dtype(&self) -> vortex_array::dtype::DType
162+
159163
pub fn vortex_fsst::FSSTArrayExt::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef
160164

161165
pub fn vortex_fsst::FSSTArrayExt::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType
162166

163167
impl<T: vortex_array::array::typed::TypedArrayRef<vortex_fsst::FSST>> vortex_fsst::FSSTArrayExt for T
164168

169+
pub fn T::codes(&self) -> vortex_array::arrays::varbin::vtable::VarBinArray
170+
171+
pub fn T::codes_dtype(&self) -> vortex_array::dtype::DType
172+
165173
pub fn T::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef
166174

167175
pub fn T::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType

0 commit comments

Comments
 (0)