Skip to content

Commit 2199d4a

Browse files
chaokunyangchaokunyang
authored
feat(java/python): align java and python compatible mode serialization (#2602)
## Why? align java and python compatible mode serialization ## What does this PR do? <!-- Describe the details of this PR. --> ## Related issues #2509 #1938 #2160 #2278 #2593 ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/fory/issues/new/choose) describing the need to do so and update the document if necessary. Delete section if not applicable. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark <!-- When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here. Delete section if not applicable. -->
1 parent 6c3f4c1 commit 2199d4a

17 files changed

Lines changed: 165 additions & 89 deletions

File tree

‎cpp/fory/type/type.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,4 +119,16 @@ inline bool IsNamespacedType(int32_t type_id) {
119119
}
120120
}
121121

122+
inline bool IsTypeShareMeta(int32_t type_id) {
123+
switch (static_cast<TypeId>(type_id)) {
124+
case TypeId::NAMED_ENUM:
125+
case TypeId::NAMED_STRUCT:
126+
case TypeId::NAMED_EXT:
127+
case TypeId::COMPATIBLE_STRUCT:
128+
case TypeId::NAMED_COMPATIBLE_STRUCT:
129+
return true;
130+
default:
131+
return false;
132+
}
133+
}
122134
} // namespace fory

‎java/fory-core/src/main/java/org/apache/fory/Fory.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1703,6 +1703,10 @@ public boolean isCrossLanguage() {
17031703
return crossLanguage;
17041704
}
17051705

1706+
public boolean isCompatible() {
1707+
return config.getCompatibleMode() == CompatibleMode.COMPATIBLE;
1708+
}
1709+
17061710
public boolean trackingRef() {
17071711
return refTracking;
17081712
}

‎java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1725,16 +1725,11 @@ public void writeClassDefs(MemoryBuffer buffer) {
17251725

17261726
private void writeClassDefs(
17271727
MemoryBuffer buffer, ObjectArray<ClassDef> writingClassDefs, int size) {
1728-
int writerIndex = buffer.writerIndex();
17291728
for (int i = 0; i < size; i++) {
1730-
byte[] encoded = writingClassDefs.get(i).getEncoded();
1731-
int bytesLen = encoded.length;
1732-
buffer.ensure(writerIndex + bytesLen);
1733-
final byte[] targetArray = buffer.getHeapMemory();
1734-
System.arraycopy(encoded, 0, targetArray, writerIndex, bytesLen);
1735-
writerIndex += bytesLen;
1736-
}
1737-
buffer.writerIndex(writerIndex);
1729+
buffer.writeBytes(writingClassDefs.get(i).getEncoded());
1730+
MemoryBuffer memoryBuffer = MemoryBuffer.fromByteArray(writingClassDefs.get(i).getEncoded());
1731+
ClassDef.readClassDef(fory, memoryBuffer, memoryBuffer.readInt64());
1732+
}
17381733
}
17391734

17401735
/**

‎java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,13 @@
3838
import java.time.LocalDate;
3939
import java.time.LocalDateTime;
4040
import java.util.ArrayList;
41+
import java.util.Collection;
4142
import java.util.Date;
4243
import java.util.HashMap;
4344
import java.util.HashSet;
4445
import java.util.LinkedHashMap;
4546
import java.util.LinkedHashSet;
47+
import java.util.List;
4648
import java.util.Map;
4749
import java.util.Set;
4850
import java.util.concurrent.atomic.AtomicBoolean;
@@ -210,7 +212,8 @@ public void register(Class<?> type, String namespace, String typeName) {
210212
short xtypeId;
211213
if (serializer != null) {
212214
if (isStructType(serializer)) {
213-
xtypeId = Types.NAMED_STRUCT;
215+
xtypeId =
216+
(short) (fory.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT);
214217
} else if (serializer instanceof EnumSerializer) {
215218
xtypeId = Types.NAMED_ENUM;
216219
} else {
@@ -220,7 +223,8 @@ public void register(Class<?> type, String namespace, String typeName) {
220223
if (type.isEnum()) {
221224
xtypeId = Types.NAMED_ENUM;
222225
} else {
223-
xtypeId = Types.NAMED_STRUCT;
226+
xtypeId =
227+
(short) (fory.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT);
224228
}
225229
}
226230
register(type, serializer, namespace, typeName, xtypeId);
@@ -530,9 +534,9 @@ private void registerDefaultTypes() {
530534
registerDefaultTypes(Types.INT64_ARRAY, long[].class);
531535
registerDefaultTypes(Types.FLOAT32_ARRAY, float[].class);
532536
registerDefaultTypes(Types.FLOAT64_ARRAY, double[].class);
533-
registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class);
534-
registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class);
535-
registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class);
537+
registerDefaultTypes(Types.LIST, ArrayList.class, Object[].class, List.class, Collection.class);
538+
registerDefaultTypes(Types.SET, HashSet.class, LinkedHashSet.class, Set.class);
539+
registerDefaultTypes(Types.MAP, HashMap.class, LinkedHashMap.class, Map.class);
536540
registerDefaultTypes(Types.LOCAL_DATE, LocalDate.class);
537541
}
538542

@@ -542,8 +546,12 @@ private void registerDefaultTypes(int xtypeId, Class<?> defaultType, Class<?>...
542546
classInfoMap.put(defaultType, classInfo);
543547
xtypeIdToClassMap.put(xtypeId, classInfo);
544548
for (Class<?> otherType : otherTypes) {
545-
classInfo = newClassInfo(otherType, classResolver.getSerializer(otherType), (short) xtypeId);
546-
classInfoMap.put(otherType, classInfo);
549+
Serializer<?> serializer =
550+
ReflectionUtils.isAbstract(otherType)
551+
? classInfo.serializer
552+
: classResolver.getSerializer(otherType);
553+
ClassInfo info = newClassInfo(otherType, serializer, (short) xtypeId);
554+
classInfoMap.put(otherType, info);
547555
}
548556
}
549557

@@ -561,7 +569,6 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) {
561569
switch (internalTypeId) {
562570
case Types.NAMED_ENUM:
563571
case Types.NAMED_STRUCT:
564-
case Types.NAMED_COMPATIBLE_STRUCT:
565572
case Types.NAMED_EXT:
566573
if (shareMeta) {
567574
writeSharedClassMeta(buffer, classInfo);
@@ -572,6 +579,11 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) {
572579
assert classInfo.typeNameBytes != null;
573580
metaStringResolver.writeMetaStringBytes(buffer, classInfo.typeNameBytes);
574581
break;
582+
case Types.NAMED_COMPATIBLE_STRUCT:
583+
case Types.COMPATIBLE_STRUCT:
584+
assert shareMeta : "Meta share must be enabled for compatible mode";
585+
writeSharedClassMeta(buffer, classInfo);
586+
break;
575587
default:
576588
break;
577589
}
@@ -634,14 +646,17 @@ public ClassInfo readClassInfo(MemoryBuffer buffer) {
634646
switch (internalTypeId) {
635647
case Types.NAMED_ENUM:
636648
case Types.NAMED_STRUCT:
637-
case Types.NAMED_COMPATIBLE_STRUCT:
638649
case Types.NAMED_EXT:
639650
if (shareMeta) {
640651
return readSharedClassMeta(buffer);
641652
}
642653
MetaStringBytes packageBytes = metaStringResolver.readMetaStringBytes(buffer);
643654
MetaStringBytes simpleClassNameBytes = metaStringResolver.readMetaStringBytes(buffer);
644655
return loadBytesToClassInfo(internalTypeId, packageBytes, simpleClassNameBytes);
656+
case Types.NAMED_COMPATIBLE_STRUCT:
657+
case Types.COMPATIBLE_STRUCT:
658+
assert shareMeta : "Meta share must be enabled for compatible mode";
659+
return readSharedClassMeta(buffer);
645660
case Types.LIST:
646661
return getListClassInfo();
647662
case Types.TIMESTAMP:

‎java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -483,19 +483,21 @@ public void testStructHash() throws Exception {
483483
roundBytes("test_struct_hash", buffer.getBytes(0, 4));
484484
}
485485

486-
@Test
487-
public void testSerializeSimpleStruct() throws Exception {
486+
@Test(dataProvider = "compatible")
487+
public void testSerializeSimpleStruct(boolean compatible) throws Exception {
488488
Fory fory =
489489
Fory.builder()
490490
.withLanguage(Language.XLANG)
491+
.withCompatibleMode(
492+
compatible ? CompatibleMode.COMPATIBLE : CompatibleMode.SCHEMA_CONSISTENT)
491493
.withRefTracking(true)
492494
.requireClassRegistration(false)
493495
.build();
494496
fory.register(ComplexObject2.class, "test.ComplexObject2");
495497
ComplexObject2 obj2 = new ComplexObject2();
496498
obj2.f1 = true;
497499
obj2.f2 = new HashMap<>(ImmutableMap.of((byte) -1, 2));
498-
structRoundBack(fory, obj2, "test_serialize_simple_struct");
500+
structRoundBack(fory, obj2, "test_serialize_simple_struct" + (compatible ? "_compatible" : ""));
499501
}
500502

501503
@Test
@@ -530,10 +532,13 @@ public void testRegisterByIdMetaShare() throws Exception {
530532
Assert.assertEquals(fory.deserialize(serialized), obj);
531533
}
532534

533-
public void testSerializeComplexStruct() throws Exception {
535+
@Test(dataProvider = "compatible")
536+
public void testSerializeComplexStruct(boolean compatible) throws Exception {
534537
Fory fory =
535538
Fory.builder()
536539
.withLanguage(Language.XLANG)
540+
.withCompatibleMode(
541+
compatible ? CompatibleMode.COMPATIBLE : CompatibleMode.SCHEMA_CONSISTENT)
537542
.withRefTracking(true)
538543
.requireClassRegistration(false)
539544
.build();
@@ -556,7 +561,7 @@ public void testSerializeComplexStruct() throws Exception {
556561
obj.f11 = new short[] {(short) 1, (short) 2};
557562
obj.f12 = ImmutableList.of((short) -1, (short) 4);
558563

559-
structRoundBack(fory, obj, "test_serialize_complex_struct");
564+
structRoundBack(fory, obj, "test_serialize_complex_struct" + (compatible ? "_compatible" : ""));
560565
}
561566

562567
private void structRoundBack(Fory fory, Object obj, String testName) throws IOException {

‎java/fory-core/src/test/java/org/apache/fory/ForyTestBase.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ public static Object[][] enableCodegen() {
131131
return new Object[][] {{false}, {true}};
132132
}
133133

134+
@DataProvider
135+
public static Object[][] compatible() {
136+
return new Object[][] {{false}, {true}};
137+
}
138+
134139
@DataProvider
135140
public static Object[][] compressNumber() {
136141
return new Object[][] {{false}, {true}};

‎python/pyfory/_fory.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,15 +144,14 @@ def __init__(
144144
self.ref_resolver = MapRefResolver()
145145
else:
146146
self.ref_resolver = NoRefResolver()
147-
from pyfory._serialization import MetaStringResolver
147+
from pyfory._serialization import MetaStringResolver, SerializationContext
148148
from pyfory._registry import TypeResolver
149149

150150
self.metastring_resolver = MetaStringResolver()
151151
self.type_resolver = TypeResolver(self, meta_share=compatible)
152+
self.serialization_context = SerializationContext(fory=self, scoped_meta_share_enabled=compatible)
152153
self.type_resolver.initialize()
153-
from pyfory._serialization import SerializationContext
154154

155-
self.serialization_context = SerializationContext(scoped_meta_share_enabled=compatible)
156155
self.buffer = Buffer.allocate(32)
157156
if not require_type_registration:
158157
warnings.warn(

‎python/pyfory/_registry.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ class TypeResolver:
169169
"_type_id_to_typeinfo",
170170
"_meta_shared_typeinfo",
171171
"meta_share",
172+
"serialization_context",
172173
)
173174

174175
def __init__(self, fory, meta_share=False):
@@ -202,6 +203,7 @@ def initialize(self):
202203
self._initialize_xlang()
203204
if self.fory.language == Language.PYTHON:
204205
self._initialize_py()
206+
self.serialization_context = self.fory.serialization_context
205207

206208
def _initialize_py(self):
207209
register = functools.partial(self._register_type, internal=True)
@@ -369,7 +371,10 @@ def _register_xtype(
369371
type_id = TypeId.NAMED_EXT if type_id is None else ((type_id << 8) + TypeId.EXT)
370372
else:
371373
serializer = None
372-
type_id = TypeId.NAMED_STRUCT if type_id is None else ((type_id << 8) + TypeId.STRUCT)
374+
if self.meta_share:
375+
type_id = TypeId.NAMED_COMPATIBLE_STRUCT if type_id is None else ((type_id << 8) + TypeId.COMPATIBLE_STRUCT)
376+
else:
377+
type_id = TypeId.NAMED_STRUCT if type_id is None else ((type_id << 8) + TypeId.STRUCT)
373378
elif not internal:
374379
type_id = TypeId.NAMED_EXT if type_id is None else ((type_id << 8) + TypeId.EXT)
375380
return self.__register_type(
@@ -607,14 +612,12 @@ def _load_metabytes_to_typeinfo(self, ns_metabytes, type_metabytes):
607612
def write_typeinfo(self, buffer, typeinfo):
608613
if typeinfo.dynamic_type:
609614
return
610-
type_id = typeinfo.type_id
611-
internal_type_id = type_id & 0xFF
612-
613615
# Check if meta share is enabled first
614616
if self.meta_share:
615617
self.write_shared_type_meta(buffer, typeinfo)
616618
return
617-
619+
type_id = typeinfo.type_id
620+
internal_type_id = type_id & 0xFF
618621
buffer.write_varuint32(type_id)
619622
if TypeId.is_namespaced_type(internal_type_id):
620623
self.metastring_resolver.write_meta_string_bytes(buffer, typeinfo.namespace_bytes)
@@ -659,17 +662,14 @@ def get_meta_compressor(self):
659662

660663
def write_shared_type_meta(self, buffer, typeinfo):
661664
"""Write shared type meta information."""
662-
assert typeinfo.type_def is not None, "Type info must be set when meta share is enabled"
663665
meta_context = self.fory.serialization_context.meta_context
664-
meta_context.write_typeinfo(buffer, typeinfo)
666+
meta_context.write_shared_typeinfo(buffer, typeinfo)
665667

666668
def read_shared_type_meta(self, buffer):
667669
"""Read shared type meta information."""
668-
meta_context = self.fory.serialization_context.meta_context
670+
meta_context = self.serialization_context.meta_context
669671
assert meta_context is not None, "Meta context must be set when meta share is enabled"
670-
type_id = buffer.read_varuint32()
671-
typeinfo = meta_context.get_read_type_info(type_id)
672-
assert typeinfo is not None, f"Type info not found for ID {type_id}"
672+
typeinfo = meta_context.read_shared_typeinfo(buffer)
673673
return typeinfo
674674

675675
def write_type_defs(self, buffer):
@@ -704,7 +704,7 @@ def read_type_defs(self, buffer):
704704
type_info = self._build_type_info_from_typedef(type_def)
705705
# Cache the tuple for future use
706706
self._meta_shared_typeinfo[header] = type_info
707-
meta_context.add_read_type_info(type_info)
707+
meta_context.add_read_typeinfo(type_info)
708708

709709
def _build_type_info_from_typedef(self, type_def):
710710
"""Build TypeInfo from TypeDef using TypeDef's create_serializer method."""

0 commit comments

Comments
 (0)