Skip to content

Commit 1a56560

Browse files
authored
remove check for ArrayPlugin encoding ID match (#7427)
## Summary When deserializing an array encoded using the experimental patched encoding `ArrayPlugin`, reads can fail b/c of this assertion. `ArrayPlugin`s can/should be able to support multiple codecs The alternative to this is to change ArrayPlugin to return a set of ArrayId's that it covers, rather than a single ID. --------- Signed-off-by: Andrew Duffy <andrew@a10y.dev>
1 parent d87dfed commit 1a56560

4 files changed

Lines changed: 30 additions & 3 deletions

File tree

encodings/fastlanes/src/bitpacking/plugin.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
8282

8383
Ok(patched.into_array())
8484
}
85+
86+
fn is_supported_encoding(&self, id: &ArrayId) -> bool {
87+
id == &BitPacked::ID || id == &Patched.id()
88+
}
8589
}
8690

8791
#[cfg(test)]

vortex-array/public-api.lock

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19412,6 +19412,8 @@ pub fn vortex_array::vtable::ArrayPlugin::deserialize(&self, dtype: &vortex_arra
1941219412

1941319413
pub fn vortex_array::vtable::ArrayPlugin::id(&self) -> vortex_array::ArrayId
1941419414

19415+
pub fn vortex_array::vtable::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
19416+
1941519417
pub fn vortex_array::vtable::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
1941619418

1941719419
impl<V: vortex_array::VTable> vortex_array::ArrayPlugin for V
@@ -19420,6 +19422,8 @@ pub fn V::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, met
1942019422

1942119423
pub fn V::id(&self) -> arcref::ArcRef<str>
1942219424

19425+
pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
19426+
1942319427
pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result<core::option::Option<alloc::vec::Vec<u8>>, vortex_error::VortexError>
1942419428

1942519429
pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug
@@ -23056,6 +23060,10 @@ pub fn vortex_array::ArrayPlugin::id(&self) -> vortex_array::ArrayId
2305623060

2305723061
pub fn vortex_array::ArrayPlugin::id(&self) -> vortex_array::ArrayId
2305823062

23063+
pub fn vortex_array::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
23064+
23065+
pub fn vortex_array::ArrayPlugin::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
23066+
2305923067
pub fn vortex_array::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
2306023068

2306123069
pub fn vortex_array::ArrayPlugin::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
@@ -23072,6 +23080,10 @@ pub fn V::id(&self) -> arcref::ArcRef<str>
2307223080

2307323081
pub fn V::id(&self) -> arcref::ArcRef<str>
2307423082

23083+
pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
23084+
23085+
pub fn V::is_supported_encoding(&self, id: &vortex_array::ArrayId) -> bool
23086+
2307523087
pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result<core::option::Option<alloc::vec::Vec<u8>>, vortex_error::VortexError>
2307623088

2307723089
pub fn V::serialize(&self, array: &vortex_array::ArrayRef, session: &vortex_session::VortexSession) -> core::result::Result<core::option::Option<alloc::vec::Vec<u8>>, vortex_error::VortexError>

vortex-array/src/array/plugin.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ pub type ArrayPluginRef = Arc<dyn ArrayPlugin>;
2727
/// [`deserialize`]: ArrayPlugin::deserialize
2828
pub trait ArrayPlugin: 'static + Send + Sync {
2929
/// Returns the ID for this array encoding.
30+
///
31+
/// During serde, this is the key the registry uses to find
32+
/// this plugin instance and call the appropriate method on it.
3033
fn id(&self) -> ArrayId;
3134

3235
/// Serialize the array metadata.
@@ -49,6 +52,14 @@ pub trait ArrayPlugin: 'static + Send + Sync {
4952
children: &dyn ArrayChildren,
5053
session: &VortexSession,
5154
) -> VortexResult<ArrayRef>;
55+
56+
/// Can this plugin emit an array with the given encoding.
57+
///
58+
/// By default, this is just the [ID][Self::id] of the plugin, but
59+
/// can be overridden if this plugin instance supports reading/writing multiple arrays.
60+
fn is_supported_encoding(&self, id: &ArrayId) -> bool {
61+
self.id() == *id
62+
}
5263
}
5364

5465
impl std::fmt::Debug for dyn ArrayPlugin {

vortex-array/src/serde.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -367,9 +367,9 @@ impl SerializedArray {
367367
decoded.dtype(),
368368
dtype,
369369
);
370-
assert_eq!(
371-
decoded.encoding_id(),
372-
encoding_id,
370+
371+
assert!(
372+
plugin.is_supported_encoding(&decoded.encoding_id()),
373373
"Array decoded from {} has incorrect encoding {}",
374374
encoding_id,
375375
decoded.encoding_id(),

0 commit comments

Comments
 (0)