feat(Rust): Implementing Type Compatible (apache#2492)

## What does this PR do?
- Modify according to the latest `xlang` specification. (Implemented but
not tested with `Java`)
- Improved the implementation of compatible mode by comparing the
`field_name` and `field_type` of the sender and receiver to determine
whether to assign or skip.

## TODO
- Haven’t figured out how to serialize and deserialize `Box` yet. 
- Handling of built-in unsigned types. (I will add type_id like
`125=>u8` to implement se/deserialize for `rust to rust`)
- Special handling of Option field types when extracting the generic
structure of field types in macros.
- In deserialization, for fields that do not exist locally and need to
be skipped, automatically create a deserializer for the corresponding
type and consume the byte data.

## Related issues
- apache#2145


## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
-->

- [ ] Does this PR introduce any public API change?
- [x] Does this PR introduce any binary protocol compatibility change?

## Benchmark

<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->

Author: urlyy

rust/fory-core/src/buffer.rs

@@ -93,24 +93,24 @@ impl Writer {
         } else if value >> 14 == 0 {
             let u1 = (value & 0x7F) | 0x80;
             let u2 = value >> 7;
-            self.u16(((u1 << 8) | u2) as u16);
+            self.u16(((u2 << 8) | u1) as u16);
         } else if value >> 21 == 0 {
             let u1 = (value & 0x7F) | 0x80;
-            let u2 = (value >> 7) | 0x80;
-            self.u16(((u1 << 8) | u2) as u16);
+            let u2 = ((value >> 7) & 0x7F) | 0x80;
+            self.u16(((u2 << 8) | u1) as u16);
             self.u8((value >> 14) as u8);
         } else if value >> 28 == 0 {
             let u1 = (value & 0x7F) | 0x80;
-            let u2 = (value >> 7) | 0x80;
-            let u3 = (value >> 14) | 0x80;
-            let u4 = (value >> 21) | 0x80;
-            self.u32(((u1 << 24) | (u2 << 16) | (u3 << 8) | u4) as u32);
+            let u2 = ((value >> 7) & 0x7F) | 0x80;
+            let u3 = ((value >> 14) & 0x7F) | 0x80;
+            let u4 = value >> 21;
+            self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
         } else {
             let u1 = (value & 0x7F) | 0x80;
-            let u2 = (value >> 7) | 0x80;
-            let u3 = (value >> 14) | 0x80;
-            let u4 = (value >> 21) | 0x80;
-            self.u32(((u1 << 24) | (u2 << 16) | (u3 << 8) | u4) as u32);
+            let u2 = ((value >> 7) & 0x7F) | 0x80;
+            let u3 = ((value >> 14) & 0x7F) | 0x80;
+            let u4 = ((value >> 21) & 0x7F) | 0x80;
+            self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
             self.u8((value >> 28) as u8);
         }
     }

rust/fory-core/src/fory.rs

@@ -17,22 +17,22 @@
 
 use crate::buffer::{Reader, Writer};
 use crate::error::Error;
-use crate::resolver::class_resolver::{ClassInfo, ClassResolver};
 use crate::resolver::context::ReadContext;
 use crate::resolver::context::WriteContext;
+use crate::resolver::type_resolver::{TypeInfo, TypeResolver};
 use crate::serializer::{Serializer, StructSerializer};
 use crate::types::{config_flags, Language, Mode, SIZE_OF_REF_AND_TYPE};
 
 pub struct Fory {
     mode: Mode,
-    class_resolver: ClassResolver,
+    type_resolver: TypeResolver,
 }
 
 impl Default for Fory {
     fn default() -> Self {
         Fory {
             mode: Mode::SchemaConsistent,
-            class_resolver: ClassResolver::default(),
+            type_resolver: TypeResolver::default(),
         }
     }
 }
@@ -78,20 +78,20 @@ impl Fory {
     pub fn serialize<T: Serializer>(&self, record: &T) -> Vec<u8> {
         let mut writer = Writer::default();
         let meta_offset = self.write_head::<T>(&mut writer);
-        let mut context = WriteContext::new(self, &mut writer);
+        let mut context: WriteContext<'_> = WriteContext::new(self, &mut writer);
         <T as Serializer>::serialize(record, &mut context);
         if Mode::Compatible == self.mode {
             context.write_meta(meta_offset);
         }
         writer.dump()
     }
 
-    pub fn get_class_resolver(&self) -> &ClassResolver {
-        &self.class_resolver
+    pub fn get_type_resolver(&self) -> &TypeResolver {
+        &self.type_resolver
     }
 
     pub fn register<T: 'static + StructSerializer>(&mut self, id: u32) {
-        let class_info = ClassInfo::new::<T>(self, id);
-        self.class_resolver.register::<T>(class_info, id);
+        let type_info = TypeInfo::new::<T>(self, id);
+        self.type_resolver.register::<T>(type_info, id);
     }
 }

rust/fory-core/src/meta/meta_string.rs

@@ -65,11 +65,9 @@ impl Default for MetaStringDecoder {
     }
 }
 
-pub struct MetaStringEncoder {}
-impl Default for MetaStringEncoder {
-    fn default() -> Self {
-        Self::new()
-    }
+#[derive(Default)]
+pub struct MetaStringEncoder<'a> {
+    encoding_options: Option<&'a [Encoding]>,
 }
 
 #[derive(Debug)]
@@ -80,9 +78,13 @@ struct StringStatistics {
     can_lower_special_encoded: bool,
 }
 
-impl MetaStringEncoder {
+impl<'a> MetaStringEncoder<'a> {
     pub fn new() -> Self {
-        MetaStringEncoder {}
+        Self::default()
+    }
+    pub fn set_options(mut self, encoding_options: Option<&'a [Encoding]>) -> Self {
+        self.encoding_options = encoding_options;
+        self
     }
 
     fn is_latin(&self, s: &str) -> bool {
@@ -108,22 +110,30 @@ impl MetaStringEncoder {
     }
 
     fn compute_encoding(&self, input: &str) -> Encoding {
+        let allow = |e: Encoding| self.encoding_options.map_or(true, |opts| opts.contains(&e));
         let statistics = self.compute_statistics(input);
-        if statistics.can_lower_special_encoded {
+        if statistics.can_lower_special_encoded && allow(Encoding::LowerSpecial) {
             return Encoding::LowerSpecial;
         }
         if statistics.can_lower_upper_digit_special_encoded {
-            if statistics.digit_count != 0 {
+            if statistics.digit_count != 0 && allow(Encoding::LowerUpperDigitSpecial) {
                 return Encoding::LowerUpperDigitSpecial;
             }
             let upper_count: usize = statistics.upper_count;
-            if upper_count == 1 && input.chars().next().unwrap().is_uppercase() {
+            if upper_count == 1
+                && input.chars().next().unwrap().is_uppercase()
+                && allow(Encoding::FirstToLowerSpecial)
+            {
                 return Encoding::FirstToLowerSpecial;
             }
-            if ((input.len() + upper_count) * 5) < (input.len() * 6) {
+            if ((input.len() + upper_count) * 5) < (input.len() * 6)
+                && allow(Encoding::AllToLowerSpecial)
+            {
                 return Encoding::AllToLowerSpecial;
             }
-            return Encoding::LowerUpperDigitSpecial;
+            if allow(Encoding::LowerUpperDigitSpecial) {
+                return Encoding::LowerUpperDigitSpecial;
+            }
         }
         Encoding::Utf8
     }

rust/fory-core/src/meta/mod.rs

@@ -20,4 +20,5 @@ mod string_util;
 mod type_meta;
 
 pub use meta_string::{Encoding, MetaString, MetaStringDecoder, MetaStringEncoder};
-pub use type_meta::{FieldInfo, TypeMeta};
+pub use string_util::murmurhash3_x64_128;
+pub use type_meta::{FieldInfo, FieldType, TypeMeta, TypeMetaLayer};

rust/fory-core/src/meta/string_util.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::mem;
+
 #[cfg(target_feature = "neon")]
 use std::arch::aarch64::*;
 
@@ -187,3 +189,198 @@ mod tests {
         assert!(!is_latin_standard(&not_latin_str));
     }
 }
+
+fn fmix64(mut k: u64) -> u64 {
+    k ^= k >> 33;
+    k = k.wrapping_mul(0xff51afd7ed558ccdu64);
+    k ^= k >> 33;
+    k = k.wrapping_mul(0xc4ceb9fe1a85ec53u64);
+    k ^= k >> 33;
+
+    k
+}
+
+pub fn murmurhash3_x64_128(bytes: &[u8], seed: u64) -> (u64, u64) {
+    let c1 = 0x87c37b91114253d5u64;
+    let c2 = 0x4cf5ad432745937fu64;
+    let read_size = 16;
+    let len = bytes.len() as u64;
+    let block_count = len / read_size;
+
+    let (mut h1, mut h2) = (seed, seed);
+
+    for i in 0..block_count as usize {
+        let b64: &[u64] = unsafe { mem::transmute(bytes) };
+        let (mut k1, mut k2) = (b64[i * 2], b64[i * 2 + 1]);
+
+        k1 = k1.wrapping_mul(c1);
+        k1 = k1.rotate_left(31);
+        k1 = k1.wrapping_mul(c2);
+        h1 ^= k1;
+
+        h1 = h1.rotate_left(27);
+        h1 = h1.wrapping_add(h2);
+        h1 = h1.wrapping_mul(5);
+        h1 = h1.wrapping_add(0x52dce729);
+
+        k2 = k2.wrapping_mul(c2);
+        k2 = k2.rotate_left(33);
+        k2 = k2.wrapping_mul(c1);
+        h2 ^= k2;
+
+        h2 = h2.rotate_left(31);
+        h2 = h2.wrapping_add(h1);
+        h2 = h2.wrapping_mul(5);
+        h2 = h2.wrapping_add(0x38495ab5);
+    }
+    let (mut k1, mut k2) = (0u64, 0u64);
+
+    if len & 15 == 15 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 14] as u64) << 48;
+    }
+    if len & 15 >= 14 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 13] as u64) << 40;
+    }
+    if len & 15 >= 13 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 12] as u64) << 32;
+    }
+    if len & 15 >= 12 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 11] as u64) << 24;
+    }
+    if len & 15 >= 11 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 10] as u64) << 16;
+    }
+    if len & 15 >= 10 {
+        k2 ^= (bytes[(block_count * read_size) as usize + 9] as u64) << 8;
+    }
+    if len & 15 >= 9 {
+        k2 ^= bytes[(block_count * read_size) as usize + 8] as u64;
+        k2 = k2.wrapping_mul(c2);
+        k2 = k2.rotate_left(33);
+        k2 = k2.wrapping_mul(c1);
+        h2 ^= k2;
+    }
+
+    if len & 15 >= 8 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 7] as u64) << 56;
+    }
+    if len & 15 >= 7 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 6] as u64) << 48;
+    }
+    if len & 15 >= 6 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 5] as u64) << 40;
+    }
+    if len & 15 >= 5 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 4] as u64) << 32;
+    }
+    if len & 15 >= 4 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 3] as u64) << 24;
+    }
+    if len & 15 >= 3 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 2] as u64) << 16;
+    }
+    if len & 15 >= 2 {
+        k1 ^= (bytes[(block_count * read_size) as usize + 1] as u64) << 8;
+    }
+    if len & 15 >= 1 {
+        k1 ^= bytes[(block_count * read_size) as usize] as u64;
+        k1 = k1.wrapping_mul(c1);
+        k1 = k1.rotate_left(31);
+        k1 = k1.wrapping_mul(c2);
+        h1 ^= k1;
+    }
+
+    h1 ^= bytes.len() as u64;
+    h2 ^= bytes.len() as u64;
+
+    h1 = h1.wrapping_add(h2);
+    h2 = h2.wrapping_add(h1);
+
+    h1 = fmix64(h1);
+    h2 = fmix64(h2);
+
+    h1 = h1.wrapping_add(h2);
+    h2 = h2.wrapping_add(h1);
+
+    (h1, h2)
+}
+
+#[cfg(test)]
+mod test_hash {
+    use super::murmurhash3_x64_128;
+
+    #[test]
+    fn test_empty_string() {
+        assert!(murmurhash3_x64_128("".as_bytes(), 0) == (0, 0));
+    }
+
+    #[test]
+    fn test_tail_lengths() {
+        assert!(
+            murmurhash3_x64_128("1".as_bytes(), 0) == (8213365047359667313, 10676604921780958775)
+        );
+        assert!(
+            murmurhash3_x64_128("12".as_bytes(), 0) == (5355690773644049813, 9855895140584599837)
+        );
+        assert!(
+            murmurhash3_x64_128("123".as_bytes(), 0) == (10978418110857903978, 4791445053355511657)
+        );
+        assert!(
+            murmurhash3_x64_128("1234".as_bytes(), 0) == (619023178690193332, 3755592904005385637)
+        );
+        assert!(
+            murmurhash3_x64_128("12345".as_bytes(), 0)
+                == (2375712675693977547, 17382870096830835188)
+        );
+        assert!(
+            murmurhash3_x64_128("123456".as_bytes(), 0)
+                == (16435832985690558678, 5882968373513761278)
+        );
+        assert!(
+            murmurhash3_x64_128("1234567".as_bytes(), 0)
+                == (3232113351312417698, 4025181827808483669)
+        );
+        assert!(
+            murmurhash3_x64_128("12345678".as_bytes(), 0)
+                == (4272337174398058908, 10464973996478965079)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789".as_bytes(), 0)
+                == (4360720697772133540, 11094893415607738629)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789a".as_bytes(), 0)
+                == (12594836289594257748, 2662019112679848245)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789ab".as_bytes(), 0)
+                == (6978636991469537545, 12243090730442643750)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789abc".as_bytes(), 0)
+                == (211890993682310078, 16480638721813329343)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789abcd".as_bytes(), 0)
+                == (12459781455342427559, 3193214493011213179)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789abcde".as_bytes(), 0)
+                == (12538342858731408721, 9820739847336455216)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789abcdef".as_bytes(), 0)
+                == (9165946068217512774, 2451472574052603025)
+        );
+        assert!(
+            murmurhash3_x64_128("123456789abcdef1".as_bytes(), 0)
+                == (9259082041050667785, 12459473952842597282)
+        );
+    }
+
+    #[test]
+    fn test_large_data() {
+        assert!(murmurhash3_x64_128("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam at consequat massa. Cras eleifend pellentesque ex, at dignissim libero maximus ut. Sed eget nulla felis".as_bytes(), 0)
+            == (9455322759164802692, 17863277201603478371));
+    }
+}