Skip to content

Commit 7408855

Browse files
a10ya10y
committed
add a LazyPatchedArray
this lets us deserialize BPArray with Patches without eagerly transposing Signed-off-by: Andrew Duffy <andrew@a10y.dev> update VTable::build to return ArrayRef This lets us return something other than the original array encoding at read time. Currently we'll want this so that BitPacked::build returns a LazyPatched, but this is applicable for pretty much any back-compat preserving encoding rewrites. Signed-off-by: Andrew Duffy <andrew@a10y.dev> Remove patches from BitPackedArray removes both the patches field as well as all code for handling patches. this is safe to do now that we have updated the VTable build function to always read methods. note that we need to leave the metadata as-is. Signed-off-by: Andrew Duffy <andrew@a10y.dev> more Signed-off-by: Andrew Duffy <andrew@a10y.dev>
1 parent bf97ba2 commit 7408855

File tree

92 files changed

+8120
-4132
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+8120
-4132
lines changed

‎encodings/alp/public-api.lock‎

Lines changed: 83 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ impl vortex_alp::ALP
1212

1313
pub fn vortex_alp::ALP::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option<vortex_array::patches::Patches>) -> vortex_alp::ALPArray
1414

15-
pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option<vortex_array::patches::Patches>) -> vortex_alp::ALPArray
15+
pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option<vortex_array::patches::Patches>, dtype: vortex_array::dtype::DType) -> vortex_alp::ALPArray
1616

1717
pub fn vortex_alp::ALP::try_new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option<vortex_array::patches::Patches>) -> vortex_error::VortexResult<vortex_alp::ALPArray>
1818

@@ -28,6 +28,8 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALP
2828

2929
pub type vortex_alp::ALP::ArrayData = vortex_alp::ALPData
3030

31+
pub type vortex_alp::ALP::Metadata = vortex_array::metadata::ProstMetadata<vortex_alp::ALPMetadata>
32+
3133
pub type vortex_alp::ALP::OperationsVTable = vortex_alp::ALP
3234

3335
pub type vortex_alp::ALP::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild
@@ -40,25 +42,35 @@ pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_,
4042

4143
pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option<alloc::string::String>
4244

43-
pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<vortex_alp::ALPData>
45+
pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>
46+
47+
pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
48+
49+
pub fn vortex_alp::ALP::dtype(array: &vortex_alp::ALPData) -> &vortex_array::dtype::DType
4450

4551
pub fn vortex_alp::ALP::execute(array: vortex_array::array::typed::Array<Self>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionResult>
4652

4753
pub fn vortex_alp::ALP::execute_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
4854

4955
pub fn vortex_alp::ALP::id(&self) -> vortex_array::array::ArrayId
5056

57+
pub fn vortex_alp::ALP::len(array: &vortex_alp::ALPData) -> usize
58+
59+
pub fn vortex_alp::ALP::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<Self::Metadata>
60+
5161
pub fn vortex_alp::ALP::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize
5262

5363
pub fn vortex_alp::ALP::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
5464

55-
pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
65+
pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
5666

5767
pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
5868

5969
pub fn vortex_alp::ALP::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option<vortex_array::array::erased::ArrayRef>]
6070

61-
pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()>
71+
pub fn vortex_alp::ALP::stats(array: &vortex_alp::ALPData) -> &vortex_array::stats::array::ArrayStats
72+
73+
pub fn vortex_alp::ALP::vtable(_array: &Self::ArrayData) -> &Self
6274

6375
pub fn vortex_alp::ALP::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec<core::option::Option<vortex_array::array::erased::ArrayRef>>) -> vortex_error::VortexResult<()>
6476

@@ -106,14 +118,22 @@ pub struct vortex_alp::ALPData
106118

107119
impl vortex_alp::ALPData
108120

121+
pub fn vortex_alp::ALPData::dtype(&self) -> &vortex_array::dtype::DType
122+
109123
pub fn vortex_alp::ALPData::encoded(&self) -> &vortex_array::array::erased::ArrayRef
110124

111125
pub fn vortex_alp::ALPData::exponents(&self) -> vortex_alp::Exponents
112126

113-
pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option<vortex_array::patches::Patches>)
127+
pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option<vortex_array::patches::Patches>, vortex_array::dtype::DType)
128+
129+
pub fn vortex_alp::ALPData::is_empty(&self) -> bool
130+
131+
pub fn vortex_alp::ALPData::len(&self) -> usize
114132

115133
pub fn vortex_alp::ALPData::patches(&self) -> core::option::Option<vortex_array::patches::Patches>
116134

135+
pub fn vortex_alp::ALPData::ptype(&self) -> vortex_array::dtype::ptype::PType
136+
117137
impl vortex_alp::ALPData
118138

119139
pub fn vortex_alp::ALPData::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option<vortex_array::patches::Patches>) -> Self
@@ -124,10 +144,18 @@ impl core::clone::Clone for vortex_alp::ALPData
124144

125145
pub fn vortex_alp::ALPData::clone(&self) -> vortex_alp::ALPData
126146

147+
impl core::convert::From<vortex_alp::ALPData> for vortex_array::array::erased::ArrayRef
148+
149+
pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPData) -> vortex_array::array::erased::ArrayRef
150+
127151
impl core::fmt::Debug for vortex_alp::ALPData
128152

129153
pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
130154

155+
impl vortex_array::array::IntoArray for vortex_alp::ALPData
156+
157+
pub fn vortex_alp::ALPData::into_array(self) -> vortex_array::array::erased::ArrayRef
158+
131159
pub struct vortex_alp::ALPMetadata
132160

133161
impl core::clone::Clone for vortex_alp::ALPMetadata
@@ -170,6 +198,8 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALPRD
170198

171199
pub type vortex_alp::ALPRD::ArrayData = vortex_alp::ALPRDData
172200

201+
pub type vortex_alp::ALPRD::Metadata = vortex_array::metadata::ProstMetadata<vortex_alp::ALPRDMetadata>
202+
173203
pub type vortex_alp::ALPRD::OperationsVTable = vortex_alp::ALPRD
174204

175205
pub type vortex_alp::ALPRD::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild
@@ -182,25 +212,35 @@ pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_
182212

183213
pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option<alloc::string::String>
184214

185-
pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<vortex_alp::ALPRDData>
215+
pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult<vortex_array::array::erased::ArrayRef>
216+
217+
pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
218+
219+
pub fn vortex_alp::ALPRD::dtype(array: &vortex_alp::ALPRDData) -> &vortex_array::dtype::DType
186220

187221
pub fn vortex_alp::ALPRD::execute(array: vortex_array::array::typed::Array<Self>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::executor::ExecutionResult>
188222

189223
pub fn vortex_alp::ALPRD::execute_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
190224

191225
pub fn vortex_alp::ALPRD::id(&self) -> vortex_array::array::ArrayId
192226

227+
pub fn vortex_alp::ALPRD::len(array: &vortex_alp::ALPRDData) -> usize
228+
229+
pub fn vortex_alp::ALPRD::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<Self::Metadata>
230+
193231
pub fn vortex_alp::ALPRD::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize
194232

195233
pub fn vortex_alp::ALPRD::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
196234

197-
pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
235+
pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
198236

199237
pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
200238

201239
pub fn vortex_alp::ALPRD::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option<vortex_array::array::erased::ArrayRef>]
202240

203-
pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()>
241+
pub fn vortex_alp::ALPRD::stats(array: &vortex_alp::ALPRDData) -> &vortex_array::stats::array::ArrayStats
242+
243+
pub fn vortex_alp::ALPRD::vtable(_array: &Self::ArrayData) -> &Self
204244

205245
pub fn vortex_alp::ALPRD::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec<core::option::Option<vortex_array::array::erased::ArrayRef>>) -> vortex_error::VortexResult<()>
206246

@@ -232,18 +272,46 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_alp::ALPR
232272

233273
pub fn vortex_alp::ALPRD::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
234274

275+
pub struct vortex_alp::ALPRDArrayParts
276+
277+
pub vortex_alp::ALPRDArrayParts::dtype: vortex_array::dtype::DType
278+
279+
pub vortex_alp::ALPRDArrayParts::left_parts: vortex_array::array::erased::ArrayRef
280+
281+
pub vortex_alp::ALPRDArrayParts::left_parts_dictionary: vortex_buffer::buffer::Buffer<u16>
282+
283+
pub vortex_alp::ALPRDArrayParts::left_parts_patches: core::option::Option<vortex_array::patches::Patches>
284+
285+
pub vortex_alp::ALPRDArrayParts::right_parts: vortex_array::array::erased::ArrayRef
286+
287+
impl core::clone::Clone for vortex_alp::ALPRDArrayParts
288+
289+
pub fn vortex_alp::ALPRDArrayParts::clone(&self) -> vortex_alp::ALPRDArrayParts
290+
291+
impl core::fmt::Debug for vortex_alp::ALPRDArrayParts
292+
293+
pub fn vortex_alp::ALPRDArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
294+
235295
pub struct vortex_alp::ALPRDData
236296

237297
impl vortex_alp::ALPRDData
238298

239-
pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDDataParts
299+
pub fn vortex_alp::ALPRDData::dtype(&self) -> &vortex_array::dtype::DType
300+
301+
pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDArrayParts
302+
303+
pub fn vortex_alp::ALPRDData::is_empty(&self) -> bool
304+
305+
pub fn vortex_alp::ALPRDData::is_f32(&self) -> bool
240306

241307
pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef
242308

243309
pub fn vortex_alp::ALPRDData::left_parts_dictionary(&self) -> &vortex_buffer::buffer::Buffer<u16>
244310

245311
pub fn vortex_alp::ALPRDData::left_parts_patches(&self) -> core::option::Option<vortex_array::patches::Patches>
246312

313+
pub fn vortex_alp::ALPRDData::len(&self) -> usize
314+
247315
pub fn vortex_alp::ALPRDData::replace_left_parts_patches(&mut self, patches: core::option::Option<vortex_array::patches::Patches>)
248316

249317
pub fn vortex_alp::ALPRDData::right_bit_width(&self) -> u8
@@ -256,27 +324,17 @@ impl core::clone::Clone for vortex_alp::ALPRDData
256324

257325
pub fn vortex_alp::ALPRDData::clone(&self) -> vortex_alp::ALPRDData
258326

259-
impl core::fmt::Debug for vortex_alp::ALPRDData
260-
261-
pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
262-
263-
pub struct vortex_alp::ALPRDDataParts
327+
impl core::convert::From<vortex_alp::ALPRDData> for vortex_array::array::erased::ArrayRef
264328

265-
pub vortex_alp::ALPRDDataParts::left_parts: vortex_array::array::erased::ArrayRef
329+
pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPRDData) -> vortex_array::array::erased::ArrayRef
266330

267-
pub vortex_alp::ALPRDDataParts::left_parts_dictionary: vortex_buffer::buffer::Buffer<u16>
268-
269-
pub vortex_alp::ALPRDDataParts::left_parts_patches: core::option::Option<vortex_array::patches::Patches>
270-
271-
pub vortex_alp::ALPRDDataParts::right_parts: vortex_array::array::erased::ArrayRef
272-
273-
impl core::clone::Clone for vortex_alp::ALPRDDataParts
331+
impl core::fmt::Debug for vortex_alp::ALPRDData
274332

275-
pub fn vortex_alp::ALPRDDataParts::clone(&self) -> vortex_alp::ALPRDDataParts
333+
pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
276334

277-
impl core::fmt::Debug for vortex_alp::ALPRDDataParts
335+
impl vortex_array::array::IntoArray for vortex_alp::ALPRDData
278336

279-
pub fn vortex_alp::ALPRDDataParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
337+
pub fn vortex_alp::ALPRDData::into_array(self) -> vortex_array::array::erased::ArrayRef
280338

281339
pub struct vortex_alp::ALPRDMetadata
282340

0 commit comments

Comments
 (0)