feat(python): type meta encoding for python (#2509)

## Why?

type forward/backward compatible serialization is critical for online
service which different service update their data schema and deploy at
different time. The schema is in an inconsistent state.

meta shared encoding can address this :
https://fory.apache.org/docs/specification/fory_xlang_serialization_spec#type-def

## What does this PR do?

Add type meta encoding for python to support type forward/backward
compatible serialization.

Things not finished in this PR:
- not null field support
- generate serializer from type meta
- meta share mode

Those feature will be implemented in follow-up PRs.

## Related issues

#1938
#2160
#2278

## Does this PR introduce any user-facing change?

<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.

Delete section if not applicable.
-->

- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?

## Benchmark

<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.

Delete section if not applicable.
-->

Author: chaokunyang

python/pyfory/__init__.py

@@ -21,6 +21,9 @@
     Language,
 )
 
+PYTHON = Language.PYTHON
+XLANG = Language.XLANG
+
 try:
     from pyfory._serialization import ENABLE_FORY_CYTHON_SERIALIZATION
 except ImportError:

python/pyfory/_registry.py

@@ -66,6 +66,7 @@
     ObjectSerializer,
 )
 from pyfory.meta.metastring import MetaStringEncoder, MetaStringDecoder
+from pyfory.meta.meta_compressor import DeflaterMetaCompressor
 from pyfory.type import (
     TypeId,
     Int8Type,
@@ -154,6 +155,7 @@ class TypeResolver:
         "namespace_decoder",
         "typename_encoder",
         "typename_decoder",
+        "meta_compressor",
         "require_registration",
         "metastring_resolver",
         "language",
@@ -182,6 +184,7 @@ def __init__(self, fory):
         self.namespace_decoder = MetaStringDecoder(".", "_")
         self.typename_encoder = MetaStringEncoder("$", "_")
         self.typename_decoder = MetaStringDecoder("$", "_")
+        self.meta_compressor = DeflaterMetaCompressor()
 
     def initialize(self):
         self._initialize_xlang()
@@ -576,6 +579,17 @@ def read_typeinfo(self, buffer):
         else:
             return self._type_id_to_typeinfo[type_id]
 
+    def get_typeinfo_by_id(self, type_id):
+        """Get typeinfo by type_id."""
+        return self._type_id_to_typeinfo[type_id]
+
+    def get_typeinfo_by_name(self, namespace, typename):
+        """Get typeinfo by namespace and typename."""
+        return self._named_type_to_typeinfo.get((namespace, typename))
+
+    def get_meta_compressor(self):
+        return self.meta_compressor
+
     def reset(self):
         pass
 

python/pyfory/_serialization.pyx

@@ -570,6 +570,15 @@ cdef class TypeResolver:
             raise ValueError(f"Unexpected type_id {type_id}")
         typeinfo = <TypeInfo> typeinfo_ptr
         return typeinfo
+    
+    def get_typeinfo_by_id(self, type_id):
+        return self._resolver.get_typeinfo_by_id(type_id=type_id)
+
+    def get_typeinfo_by_name(self, namespace, typename):
+        return self._resolver.get_typeinfo_by_name(namespace=namespace, typename=typename)
+
+    def get_meta_compressor(self):
+        return self._resolver.get_meta_compressor()
 
     cpdef inline reset(self):
         pass

python/pyfory/_struct.py

@@ -18,6 +18,7 @@
 import datetime
 import enum
 import logging
+import typing
 
 from pyfory.type import (
     TypeVisitor,
@@ -166,7 +167,7 @@ def numeric_sorter(item):
     map_types = sorted(map_types, key=sorter)
     other_types = sorted(other_types, key=sorter)
     all_types = boxed_types + final_types + other_types + collection_types + map_types
-    return [t[1] for t in all_types], [t[2] for t in all_types]
+    return [t[2] for t in all_types], [t[1] for t in all_types]
 
 
 class StructHashVisitor(TypeVisitor):
@@ -221,3 +222,49 @@ def _compute_field_hash(hash_, id_):
 
     def get_hash(self):
         return self._hash
+
+
+class StructTypeIdVisitor(TypeVisitor):
+    def __init__(
+        self,
+        fory,
+    ):
+        self.fory = fory
+
+    def visit_list(self, field_name, elem_type, types_path=None):
+        # Infer type recursively for type such as List[Dict[str, str]]
+        elem_ids = infer_field("item", elem_type, self, types_path=types_path)
+        return TypeId.LIST, elem_ids
+
+    def visit_dict(self, field_name, key_type, value_type, types_path=None):
+        # Infer type recursively for type such as Dict[str, Dict[str, str]]
+        key_ids = infer_field("key", key_type, self, types_path=types_path)
+        value_ids = infer_field("value", value_type, self, types_path=types_path)
+        return TypeId.MAP, key_ids, value_ids
+
+    def visit_customized(self, field_name, type_, types_path=None):
+        return None, None
+
+    def visit_other(self, field_name, type_, types_path=None):
+        from pyfory.serializer import PickleSerializer  # Local import
+
+        if is_subclass(type_, enum.Enum):
+            return self.fory.type_resolver.get_typeinfo(type_).type_id
+        if type_ not in basic_types and not is_py_array_type(type_):
+            return None, None
+        typeinfo = self.fory.type_resolver.get_typeinfo(type_)
+        assert not isinstance(typeinfo.serializer, (PickleSerializer,))
+        return [typeinfo.type_id]
+
+
+def get_field_names(clz, type_hints=None):
+    if hasattr(clz, "__dict__"):
+        # Regular object with __dict__
+        # We can't know the fields without an instance, so we rely on type hints
+        if type_hints is None:
+            type_hints = typing.get_type_hints(clz)
+        return sorted(type_hints.keys())
+    elif hasattr(clz, "__slots__"):
+        # Object with __slots__
+        return sorted(clz.__slots__)
+    return []

python/pyfory/_util.pxd

@@ -94,6 +94,8 @@ cdef class Buffer:
 
     cpdef inline write_bool(self, c_bool value)
 
+    cpdef inline write_uint8(self, uint8_t value)
+
     cpdef inline write_int8(self, int8_t value)
 
     cpdef inline write_int16(self, int16_t value)

python/pyfory/_util.pyx

@@ -167,6 +167,11 @@ cdef class Buffer:
         self.grow(<int32_t>1)
         (<c_bool *>(self._c_address + self.writer_index))[0] = value
         self.writer_index += <int32_t>1
+    
+    cpdef inline write_uint8(self, uint8_t value):
+        self.grow(<int32_t>1)
+        (<uint8_t *>(self._c_address + self.writer_index))[0] = value
+        self.writer_index += <int32_t>1
 
     cpdef inline write_int8(self, int8_t value):
         self.grow(<int32_t>1)

python/pyfory/meta/metastring.py

@@ -372,7 +372,7 @@ def encode_with_encoding(self, input_string: str, encoding: Encoding) -> MetaStr
                 self.special_char2,
             )
 
-    def compute_encoding(self, input_string: str) -> Encoding:
+    def compute_encoding(self, input_string: str, encodings: List[Encoding] = None) -> Encoding:
         """
         Determines the encoding type of the input string.
 
@@ -384,23 +384,28 @@ def compute_encoding(self, input_string: str) -> Encoding:
         """
         if not input_string:
             return Encoding.LOWER_SPECIAL
+        if encodings is None:
+            encodings = list(Encoding.__members__.values())
 
         chars = list(input_string)
         statistics = self._compute_statistics(chars)
-        if statistics.can_lower_special_encoded:
+        if statistics.can_lower_special_encoded and Encoding.LOWER_SPECIAL in encodings:
             return Encoding.LOWER_SPECIAL
-        elif statistics.can_lower_upper_digit_special_encoded:
+        elif statistics.can_lower_upper_digit_special_encoded and Encoding.LOWER_UPPER_DIGIT_SPECIAL in encodings:
             if statistics.digit_count != 0:
                 return Encoding.LOWER_UPPER_DIGIT_SPECIAL
             else:
                 upper_count = statistics.upper_count
                 if upper_count == 1 and chars[0].isupper():
                     return Encoding.FIRST_TO_LOWER_SPECIAL
-                if (len(chars) + upper_count) * 5 < len(chars) * 6:
+                if (len(chars) + upper_count) * 5 < len(chars) * 6 and Encoding.ALL_TO_LOWER_SPECIAL in encodings:
                     return Encoding.ALL_TO_LOWER_SPECIAL
                 else:
-                    return Encoding.LOWER_UPPER_DIGIT_SPECIAL
-        return Encoding.UTF_8
+                    if Encoding.LOWER_UPPER_DIGIT_SPECIAL in encodings:
+                        return Encoding.LOWER_UPPER_DIGIT_SPECIAL
+        if Encoding.UTF_8 in encodings:
+            return Encoding.UTF_8
+        raise ValueError(f"No encoding found for string: {input_string}, encodings: {encodings}")
 
     def _compute_statistics(self, chars: List[str]) -> Statistics:
         """