feat(rust): support skipping fields bytes when deserializing in compatible mode (apache#2545)

urlyy · web-flow · commit 707de6029dd8 · 2025-08-29T15:22:09.000+08:00
## What does this PR do? Support skipping unneeded field bytes when deserializing in compatible mode. ## Related issues apache#2531
diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs
@@ -90,7 +90,7 @@ impl Fory {
         &self.type_resolver
     }
 
-    pub fn register<T: 'static + StructSerializer>(&mut self, id: u32) {
+    pub fn register<T: 'static + StructSerializer>(&mut self, id: i16) {
         let type_info = TypeInfo::new::<T>(self, id);
         self.type_resolver.register::<T>(type_info, id);
     }
diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs
@@ -39,10 +39,10 @@ static ENCODING_OPTIONS: &[Encoding] = &[
     Encoding::LowerUpperDigitSpecial,
 ];
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub struct FieldType {
     pub type_id: i16,
-    generics: Vec<FieldType>,
+    pub generics: Vec<FieldType>,
 }
 
 impl FieldType {
@@ -128,7 +128,7 @@ impl FieldType {
     }
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub struct FieldInfo {
     pub field_name: String,
     pub field_type: FieldType,
@@ -213,19 +213,19 @@ impl FieldInfo {
 
 #[derive(Debug)]
 pub struct TypeMetaLayer {
-    type_id: u32,
+    type_id: i16,
     field_infos: Vec<FieldInfo>,
 }
 
 impl TypeMetaLayer {
-    pub fn new(type_id: u32, field_infos: Vec<FieldInfo>) -> TypeMetaLayer {
+    pub fn new(type_id: i16, field_infos: Vec<FieldInfo>) -> TypeMetaLayer {
         TypeMetaLayer {
             type_id,
             field_infos,
         }
     }
 
-    pub fn get_type_id(&self) -> u32 {
+    pub fn get_type_id(&self) -> i16 {
         self.type_id
     }
 
@@ -271,7 +271,7 @@ impl TypeMetaLayer {
         if is_register_by_name {
             todo!()
         } else {
-            type_id = reader.var_int32() as u32;
+            type_id = reader.var_int32() as i16;
         }
         let mut field_infos = Vec::with_capacity(num_fields);
         for _ in 0..num_fields {
@@ -293,11 +293,11 @@ impl TypeMeta {
         self.layers.first().unwrap().get_field_infos()
     }
 
-    pub fn get_type_id(&self) -> u32 {
+    pub fn get_type_id(&self) -> i16 {
         self.layers.first().unwrap().get_type_id()
     }
 
-    pub fn from_fields(type_id: u32, field_infos: Vec<FieldInfo>) -> TypeMeta {
+    pub fn from_fields(type_id: i16, field_infos: Vec<FieldInfo>) -> TypeMeta {
         TypeMeta {
             // hash: 0,
             layers: vec![TypeMetaLayer::new(type_id, field_infos)],
diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs
@@ -24,7 +24,7 @@ use std::rc::Rc;
 
 #[derive(Default)]
 pub struct MetaReaderResolver {
-    reading_type_defs: Vec<Rc<TypeMeta>>,
+    pub reading_type_defs: Vec<Rc<TypeMeta>>,
 }
 
 impl MetaReaderResolver {
diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs
@@ -18,9 +18,7 @@
 use super::context::{ReadContext, WriteContext};
 use crate::error::Error;
 use crate::fory::Fory;
-use crate::serializer::{Serializer, StructSerializer};
-use crate::types::TypeId;
-use chrono::{NaiveDate, NaiveDateTime};
+use crate::serializer::StructSerializer;
 use std::{any::Any, collections::HashMap};
 
 pub struct Harness {
@@ -50,18 +48,18 @@ impl Harness {
 
 pub struct TypeInfo {
     type_def: Vec<u8>,
-    type_id: u32,
+    type_id: i16,
 }
 
 impl TypeInfo {
-    pub fn new<T: StructSerializer>(fory: &Fory, type_id: u32) -> TypeInfo {
+    pub fn new<T: StructSerializer>(fory: &Fory, type_id: i16) -> TypeInfo {
         TypeInfo {
-            type_def: T::type_def(fory),
+            type_def: T::type_def(fory, type_id),
             type_id,
         }
     }
 
-    pub fn get_type_id(&self) -> u32 {
+    pub fn get_type_id(&self) -> i16 {
         self.type_id
     }
 
@@ -70,56 +68,12 @@ impl TypeInfo {
     }
 }
 
+#[derive(Default)]
 pub struct TypeResolver {
-    serialize_map: HashMap<u32, Harness>,
-    type_id_map: HashMap<std::any::TypeId, u32>,
+    serialize_map: HashMap<i16, Harness>,
+    type_id_map: HashMap<std::any::TypeId, i16>,
     type_info_map: HashMap<std::any::TypeId, TypeInfo>,
 }
-macro_rules! register_harness {
-    ($ty:ty, $id:expr, $map:expr) => {{
-        fn serializer(this: &dyn std::any::Any, context: &mut WriteContext) {
-            let this = this.downcast_ref::<$ty>();
-            match this {
-                Some(v) => <$ty>::serialize(v, context),
-                None => todo!(""),
-            }
-        }
-
-        fn deserializer(context: &mut ReadContext) -> Result<Box<dyn std::any::Any>, Error> {
-            match <$ty>::deserialize(context) {
-                Ok(v) => Ok(Box::new(v)),
-                Err(e) => Err(e),
-            }
-        }
-
-        $map.insert($id as u32, Harness::new(serializer, deserializer));
-    }};
-}
-
-impl Default for TypeResolver {
-    fn default() -> Self {
-        let mut serialize_map = HashMap::new();
-
-        register_harness!(bool, TypeId::BOOL, serialize_map);
-        register_harness!(i8, TypeId::INT8, serialize_map);
-        register_harness!(i16, TypeId::INT16, serialize_map);
-        register_harness!(i32, TypeId::INT32, serialize_map);
-        register_harness!(i64, TypeId::INT64, serialize_map);
-        register_harness!(f32, TypeId::FLOAT32, serialize_map);
-        register_harness!(f64, TypeId::FLOAT64, serialize_map);
-
-        register_harness!(String, TypeId::STRING, serialize_map);
-
-        register_harness!(NaiveDate, TypeId::LOCAL_DATE, serialize_map);
-        register_harness!(NaiveDateTime, TypeId::TIMESTAMP, serialize_map);
-
-        TypeResolver {
-            serialize_map,
-            type_id_map: HashMap::new(),
-            type_info_map: HashMap::new(),
-        }
-    }
-}
 
 impl TypeResolver {
     pub fn get_type_info(&self, type_id: std::any::TypeId) -> &TypeInfo {
@@ -131,7 +85,7 @@ impl TypeResolver {
         })
     }
 
-    pub fn register<T: StructSerializer>(&mut self, type_info: TypeInfo, id: u32) {
+    pub fn register<T: StructSerializer>(&mut self, type_info: TypeInfo, id: i16) {
         fn serializer<T2: 'static + StructSerializer>(this: &dyn Any, context: &mut WriteContext) {
             let this = this.downcast_ref::<T2>();
             match this {
@@ -161,7 +115,7 @@ impl TypeResolver {
         self.get_harness(*self.type_id_map.get(&type_id).unwrap())
     }
 
-    pub fn get_harness(&self, id: u32) -> Option<&Harness> {
+    pub fn get_harness(&self, id: i16) -> Option<&Harness> {
         self.serialize_map.get(&id)
     }
 }
diff --git a/rust/fory-core/src/serializer/any.rs b/rust/fory-core/src/serializer/any.rs
@@ -71,7 +71,7 @@ impl Serializer for Box<dyn Any> {
                 context
                     .get_fory()
                     .get_type_resolver()
-                    .get_harness(type_id as u32)
+                    .get_harness(type_id)
                     .unwrap()
                     .get_deserializer()(context)
             }
diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs
@@ -27,6 +27,7 @@ mod bool;
 mod datetime;
 mod list;
 mod map;
+pub mod nonexistent;
 mod number;
 mod option;
 mod primitive_list;
@@ -92,5 +93,5 @@ where
 }
 
 pub trait StructSerializer: Serializer + 'static {
-    fn type_def(fory: &Fory) -> Vec<u8>;
+    fn type_def(fory: &Fory, type_id: i16) -> Vec<u8>;
 }
diff --git a/rust/fory-core/src/serializer/nonexistent.rs b/rust/fory-core/src/serializer/nonexistent.rs
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::ensure;
+use crate::error::Error;
+use crate::meta::FieldType;
+use crate::resolver::context::ReadContext;
+use crate::serializer::Serializer;
+use crate::types::{RefFlag, TypeId, BASIC_TYPES, COLLECTION_TYPES};
+use anyhow::anyhow;
+use chrono::{NaiveDate, NaiveDateTime};
+
+macro_rules! basic_type_deserialize {
+    ($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => {
+        $(
+            if $tid == TypeId::$id {
+                <$ty>::deserialize($context)?;
+                return Ok(());
+            }
+        )+else {
+            unreachable!()
+        }
+    };
+}
+
+pub fn skip_field_value(context: &mut ReadContext, field_type: &FieldType) -> Result<(), Error> {
+    match TypeId::try_from(field_type.type_id) {
+        Ok(type_id) => {
+            if BASIC_TYPES.contains(&type_id) {
+                basic_type_deserialize!(type_id, context;
+                    (bool, BOOL),
+                    (i8, INT8),
+                    (i16, INT16),
+                    (i32, INT32),
+                    (i64, INT64),
+                    (f32, FLOAT32),
+                    (f64, FLOAT64),
+                    (String, STRING),
+                    (NaiveDate, LOCAL_DATE),
+                    (NaiveDateTime, TIMESTAMP),
+                );
+            } else if COLLECTION_TYPES.contains(&type_id) {
+                let ref_flag = context.reader.i8();
+                let actual_type_id = context.reader.i16();
+                let type_id_num = type_id.into();
+                ensure!(
+                    actual_type_id == type_id_num,
+                    anyhow!("Invalid field type, expected:{type_id_num}, actual:{actual_type_id}")
+                );
+                if ref_flag == (RefFlag::NotNullValue as i8)
+                    || ref_flag == (RefFlag::RefValue as i8)
+                {
+                    if type_id == TypeId::ARRAY || type_id == TypeId::SET {
+                        let length = context.reader.var_int32() as usize;
+                        println!("skipping array with length {}", length);
+                        for _ in 0..length {
+                            skip_field_value(context, field_type.generics.first().unwrap())?;
+                        }
+                    } else if type_id == TypeId::MAP {
+                        let length = context.reader.var_int32() as usize;
+                        for _ in 0..length {
+                            skip_field_value(context, field_type.generics.first().unwrap())?;
+                            skip_field_value(context, field_type.generics.get(1).unwrap())?;
+                        }
+                    }
+                    Ok(())
+                } else if ref_flag == (RefFlag::Null as i8) {
+                    Err(anyhow!("Try to deserialize non-option type to null"))?
+                } else if ref_flag == (RefFlag::Ref as i8) {
+                    Err(Error::Ref)
+                } else {
+                    Err(anyhow!("Unknown ref flag, value:{ref_flag}"))?
+                }
+            } else {
+                unreachable!()
+            }
+        }
+        Err(_) => {
+            // skip ref_flag and meta_index
+            context.reader.i8();
+            context.reader.i16();
+            let type_defs: Vec<_> = context.meta_resolver.reading_type_defs.to_vec();
+            for type_def in type_defs.iter() {
+                if type_def.get_type_id() == field_type.type_id {
+                    let field_infos: Vec<_> = type_def.get_field_infos().to_vec();
+                    for field_info in field_infos.iter() {
+                        skip_field_value(context, &field_info.field_type)?;
+                    }
+                }
+            }
+            Ok(())
+        }
+    }
+}
diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs
@@ -112,19 +112,20 @@ pub fn compute_string_hash(s: &str) -> u32 {
     hash as u32
 }
 
-// const BASIC_TYPES: [FieldType; 11] = [
-//     FieldType::BOOL,
-//     FieldType::INT8,
-//     FieldType::INT16,
-//     FieldType::INT32,
-//     FieldType::INT64,
-//     FieldType::FLOAT,
-//     FieldType::DOUBLE,
-//     FieldType::STRING,
-//     FieldType::BINARY,
-//     FieldType::DATE,
-//     FieldType::TIMESTAMP,
-// ];
+pub const BASIC_TYPES: [TypeId; 10] = [
+    TypeId::BOOL,
+    TypeId::INT8,
+    TypeId::INT16,
+    TypeId::INT32,
+    TypeId::INT64,
+    TypeId::FLOAT32,
+    TypeId::FLOAT64,
+    TypeId::STRING,
+    TypeId::LOCAL_DATE,
+    TypeId::TIMESTAMP,
+];
+
+pub const COLLECTION_TYPES: [TypeId; 3] = [TypeId::ARRAY, TypeId::SET, TypeId::MAP];
 
 pub fn compute_field_hash(hash: u32, id: i16) -> u32 {
     let mut new_hash: u64 = (hash as u64) * 31 + (id as u64);
diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs
@@ -21,7 +21,7 @@ use syn::DataEnum;
 
 pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream {
     quote! {
-        fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {
+        fn type_def(fory: &fory_core::fory::Fory, type_id: i16) -> Vec<u8> {
             Vec::new()
         }
     }
diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs
@@ -56,9 +56,9 @@ fn type_def(fields: &[&Field]) -> TokenStream {
         }
     });
     quote! {
-        fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {
+        fn type_def(fory: &fory_core::fory::Fory, layer_id: i16) -> Vec<u8> {
             fory_core::meta::TypeMeta::from_fields(
-                0,
+                layer_id,
                 vec![#(#field_infos),*]
             ).to_bytes().unwrap()
         }
diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs
diff --git a/rust/tests/tests/test_compatible.rs b/rust/tests/tests/test_compatible.rs

Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ impl Fory {`
`90`	`90`	`&self.type_resolver`
`91`	`91`	`}`
`92`	`92`
`93`		`- pub fn register<T: 'static + StructSerializer>(&mut self, id: u32) {`
	`93`	`+ pub fn register<T: 'static + StructSerializer>(&mut self, id: i16) {`
`94`	`94`	`let type_info = TypeInfo::new::<T>(self, id);`
`95`	`95`	`self.type_resolver.register::<T>(type_info, id);`
`96`	`96`	`}`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ use std::rc::Rc;`
`24`	`24`
`25`	`25`	`#[derive(Default)]`
`26`	`26`	`pub struct MetaReaderResolver {`
`27`		`- reading_type_defs: Vec<Rc<TypeMeta>>,`
	`27`	`+ pub reading_type_defs: Vec<Rc<TypeMeta>>,`
`28`	`28`	`}`
`29`	`29`
`30`	`30`	`impl MetaReaderResolver {`
Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ impl Serializer for Box<dyn Any> {`
`71`	`71`	`context`
`72`	`72`	`.get_fory()`
`73`	`73`	`.get_type_resolver()`
`74`		`- .get_harness(type_id as u32)`
	`74`	`+ .get_harness(type_id)`
`75`	`75`	`.unwrap()`
`76`	`76`	`.get_deserializer()(context)`
`77`	`77`	`}`
Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,7 @@ mod bool;`
`27`	`27`	`mod datetime;`
`28`	`28`	`mod list;`
`29`	`29`	`mod map;`
	`30`	`+pub mod nonexistent;`
`30`	`31`	`mod number;`
`31`	`32`	`mod option;`
`32`	`33`	`mod primitive_list;`
`@@ -92,5 +93,5 @@ where`
`92`	`93`	`}`
`93`	`94`
`94`	`95`	`pub trait StructSerializer: Serializer + 'static {`
`95`		`- fn type_def(fory: &Fory) -> Vec<u8>;`
	`96`	`+ fn type_def(fory: &Fory, type_id: i16) -> Vec<u8>;`
`96`	`97`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@ use syn::DataEnum;`
`21`	`21`
`22`	`22`	`pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream {`
`23`	`23`	`quote! {`
`24`		`- fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {`
	`24`	`+ fn type_def(fory: &fory_core::fory::Fory, type_id: i16) -> Vec<u8> {`
`25`	`25`	`Vec::new()`
`26`	`26`	`}`
`27`	`27`	`}`
Original file line number	Diff line number	Diff line change
`@@ -56,9 +56,9 @@ fn type_def(fields: &[&Field]) -> TokenStream {`
`56`	`56`	`}`
`57`	`57`	`});`
`58`	`58`	`quote! {`
`59`		`- fn type_def(fory: &fory_core::fory::Fory) -> Vec<u8> {`
	`59`	`+ fn type_def(fory: &fory_core::fory::Fory, layer_id: i16) -> Vec<u8> {`
`60`	`60`	`fory_core::meta::TypeMeta::from_fields(`
`61`		`- 0,`
	`61`	`+ layer_id,`
`62`	`62`	`vec![#(#field_infos),*]`
`63`	`63`	`).to_bytes().unwrap()`
`64`	`64`	`}`