Skip to content

Commit 3b28265

Browse files
authored
feat(python): meta compression for python (#2504)
## What does this PR do? <!-- Describe the purpose of this PR. --> ## Related issues - #1938 - #2160 ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/fory/issues/new/choose) describing the need to do so and update the document if necessary. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark <!-- When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here. -->
1 parent 5673703 commit 3b28265

2 files changed

Lines changed: 329 additions & 0 deletions

File tree

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import zlib
19+
from abc import ABC, abstractmethod
20+
from typing import Optional
21+
22+
23+
class MetaCompressor(ABC):
24+
"""
25+
An interface used to compress class metadata such as field names and types.
26+
The implementation of this interface should be thread safe.
27+
"""
28+
29+
@abstractmethod
30+
def compress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
31+
"""
32+
Compress the given data.
33+
34+
Args:
35+
data: The data to compress
36+
offset: Starting offset in the data
37+
size: Size of data to compress (if None, uses len(data) - offset)
38+
39+
Returns:
40+
Compressed data as bytes
41+
"""
42+
pass
43+
44+
@abstractmethod
45+
def decompress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
46+
"""
47+
Decompress the given data.
48+
49+
Args:
50+
data: The compressed data to decompress
51+
offset: Starting offset in the data
52+
size: Size of data to decompress (if None, uses len(data) - offset)
53+
54+
Returns:
55+
Decompressed data as bytes
56+
"""
57+
pass
58+
59+
60+
class DeflaterMetaCompressor(MetaCompressor):
61+
"""
62+
A meta compressor based on zlib compression algorithm (equivalent to Java's Deflater).
63+
This implementation is thread safe.
64+
"""
65+
66+
def compress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
67+
"""
68+
Compress the given data using zlib.
69+
70+
Args:
71+
data: The data to compress
72+
offset: Starting offset in the data
73+
size: Size of data to compress (if None, uses len(data) - offset)
74+
75+
Returns:
76+
Compressed data as bytes
77+
"""
78+
if size is None:
79+
size = len(data) - offset
80+
81+
if size <= 0:
82+
return b""
83+
84+
# Use zlib.compress which is equivalent to Java's Deflater
85+
return zlib.compress(data[offset : offset + size])
86+
87+
def decompress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
88+
"""
89+
Decompress the given data using zlib.
90+
91+
Args:
92+
data: The compressed data to decompress
93+
offset: Starting offset in the data
94+
size: Size of data to decompress (if None, uses len(data) - offset)
95+
96+
Returns:
97+
Decompressed data as bytes
98+
"""
99+
if size is None:
100+
size = len(data) - offset
101+
102+
if size <= 0:
103+
return b""
104+
105+
# Use zlib.decompress which is equivalent to Java's Inflater
106+
return zlib.decompress(data[offset : offset + size])
107+
108+
def __hash__(self) -> int:
109+
"""Return hash code based on class type."""
110+
return hash(DeflaterMetaCompressor)
111+
112+
def __eq__(self, other) -> bool:
113+
"""Check equality based on class type."""
114+
if self is other:
115+
return True
116+
return other is not None and isinstance(other, DeflaterMetaCompressor)
117+
118+
119+
def check_meta_compressor(compressor: MetaCompressor) -> MetaCompressor:
120+
"""
121+
Check whether MetaCompressor implements `__eq__/__hash__` method. If not implemented,
122+
return TypeEqualMetaCompressor instead which compare equality by the compressor type
123+
for better serializer compile cache.
124+
125+
Args:
126+
compressor: The compressor to check
127+
128+
Returns:
129+
The compressor or a TypeEqualMetaCompressor wrapper
130+
"""
131+
# Check if the compressor has custom __eq__ and __hash__ methods
132+
# by comparing with the default object methods
133+
if compressor.__class__.__eq__ == object.__eq__ or compressor.__class__.__hash__ == object.__hash__:
134+
return TypeEqualMetaCompressor(compressor)
135+
return compressor
136+
137+
138+
class TypeEqualMetaCompressor(MetaCompressor):
139+
"""
140+
A MetaCompressor wrapper which compare equality by the compressor type for better
141+
serializer compile cache.
142+
"""
143+
144+
def __init__(self, compressor: MetaCompressor):
145+
"""
146+
Initialize with the wrapped compressor.
147+
148+
Args:
149+
compressor: The compressor to wrap
150+
"""
151+
self.compressor = compressor
152+
153+
def compress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
154+
"""Delegate compression to the wrapped compressor."""
155+
return self.compressor.compress(data, offset, size)
156+
157+
def decompress(self, data: bytes, offset: int = 0, size: Optional[int] = None) -> bytes:
158+
"""Delegate decompression to the wrapped compressor."""
159+
return self.compressor.decompress(data, offset, size)
160+
161+
def __eq__(self, other) -> bool:
162+
"""Check equality based on compressor class type."""
163+
if other is None or not isinstance(other, TypeEqualMetaCompressor):
164+
return False
165+
return self.compressor.__class__ == other.compressor.__class__
166+
167+
def __hash__(self) -> int:
168+
"""Return hash code based on compressor class type."""
169+
return hash(self.compressor.__class__)
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import pytest
19+
import threading
20+
from pyfory.meta.meta_compressor import MetaCompressor, DeflaterMetaCompressor, TypeEqualMetaCompressor, check_meta_compressor
21+
22+
23+
@pytest.fixture
24+
def compressor():
25+
"""Fixture providing a DeflaterMetaCompressor instance."""
26+
return DeflaterMetaCompressor()
27+
28+
29+
@pytest.fixture
30+
def test_data():
31+
"""Fixture providing test data for compression tests."""
32+
return b"This is some test data that should be compressed and decompressed correctly"
33+
34+
35+
class TestDeflaterMetaCompressor:
36+
def test_compress_decompress(self, compressor, test_data):
37+
"""Test that compression and decompression work correctly."""
38+
compressed = compressor.compress(test_data)
39+
decompressed = compressor.decompress(compressed)
40+
41+
assert decompressed == test_data
42+
assert len(compressed) < len(test_data)
43+
44+
def test_compress_decompress_with_offset(self, compressor, test_data):
45+
"""Test compression and decompression with offset."""
46+
offset = 5
47+
size = 20
48+
compressed = compressor.compress(test_data, offset, size)
49+
decompressed = compressor.decompress(compressed)
50+
51+
assert decompressed == test_data[offset : offset + size]
52+
53+
def test_compress_decompress_empty_data(self, compressor):
54+
"""Test compression and decompression with empty data."""
55+
empty_data = b""
56+
compressed = compressor.compress(empty_data)
57+
decompressed = compressor.decompress(compressed)
58+
59+
assert decompressed == empty_data
60+
61+
def test_compress_decompress_small_data(self, compressor):
62+
"""Test compression and decompression with small data."""
63+
small_data = b"abc"
64+
compressed = compressor.compress(small_data)
65+
decompressed = compressor.decompress(compressed)
66+
67+
assert decompressed == small_data
68+
69+
def test_equality_and_hash(self, compressor):
70+
"""Test equality and hash methods."""
71+
compressor1 = DeflaterMetaCompressor()
72+
compressor2 = DeflaterMetaCompressor()
73+
74+
# Test equality
75+
assert compressor1 == compressor2
76+
assert compressor1 == compressor1 # Same instance
77+
78+
# Test hash
79+
assert hash(compressor1) == hash(compressor2)
80+
assert hash(compressor1) == hash(DeflaterMetaCompressor)
81+
82+
def test_thread_safety(self):
83+
"""Test that the compressor is thread safe (basic test)."""
84+
results = []
85+
errors = []
86+
87+
def compress_worker():
88+
try:
89+
compressor = DeflaterMetaCompressor()
90+
for i in range(100):
91+
data = f"Test data {i}".encode("utf-8")
92+
compressed = compressor.compress(data)
93+
decompressed = compressor.decompress(compressed)
94+
if decompressed != data:
95+
errors.append(f"Mismatch at iteration {i}")
96+
results.append("success")
97+
except Exception as e:
98+
errors.append(str(e))
99+
100+
threads = []
101+
for _ in range(4):
102+
thread = threading.Thread(target=compress_worker)
103+
threads.append(thread)
104+
thread.start()
105+
106+
for thread in threads:
107+
thread.join()
108+
109+
assert len(results) == 4
110+
assert len(errors) == 0
111+
112+
113+
class TestTypeEqualMetaCompressor:
114+
def test_wrapper_functionality(self):
115+
"""Test that TypeEqualMetaCompressor correctly wraps another compressor."""
116+
original_compressor = DeflaterMetaCompressor()
117+
wrapped_compressor = TypeEqualMetaCompressor(original_compressor)
118+
119+
test_data = b"Test data for wrapper"
120+
compressed = wrapped_compressor.compress(test_data)
121+
decompressed = wrapped_compressor.decompress(compressed)
122+
123+
assert decompressed == test_data
124+
125+
def test_equality_by_type(self):
126+
"""Test that TypeEqualMetaCompressor compares equality by type."""
127+
compressor1 = DeflaterMetaCompressor()
128+
wrapped1 = TypeEqualMetaCompressor(compressor1)
129+
wrapped2 = TypeEqualMetaCompressor(compressor1)
130+
131+
# Should be equal because they wrap the same type
132+
assert wrapped1 == wrapped2
133+
assert hash(wrapped1) == hash(wrapped2)
134+
135+
136+
class TestCheckMetaCompressor:
137+
def test_check_meta_compressor_with_proper_implementation(self):
138+
"""Test check_meta_compressor with a compressor that has proper __eq__/__hash__."""
139+
compressor = DeflaterMetaCompressor()
140+
result = check_meta_compressor(compressor)
141+
142+
# Should return the original compressor since it has proper __eq__/__hash__
143+
assert result is compressor
144+
145+
def test_check_meta_compressor_without_proper_implementation(self):
146+
"""Test check_meta_compressor with a compressor that lacks proper __eq__/__hash__."""
147+
148+
class SimpleCompressor(MetaCompressor):
149+
def compress(self, data, offset=0, size=None):
150+
return data
151+
152+
def decompress(self, data, offset=0, size=None):
153+
return data
154+
155+
compressor = SimpleCompressor()
156+
result = check_meta_compressor(compressor)
157+
158+
# Should return a TypeEqualMetaCompressor wrapper
159+
assert isinstance(result, TypeEqualMetaCompressor)
160+
assert result.compressor is compressor

0 commit comments

Comments
 (0)