Skip to content

Commit c6e13b6

Browse files
authored
Revert "Fix Python VarIntCoder OverflowError on uint64 values (#39047)" (#39218)
This reverts commit e87f50e.
1 parent bdc2667 commit c6e13b6

3 files changed

Lines changed: 9 additions & 44 deletions

File tree

sdks/python/apache_beam/coders/coder_impl.pxd

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ cdef class TimestampCoderImpl(StreamCoderImpl):
135135

136136
cdef list small_ints
137137
cdef class VarIntCoderImpl(StreamCoderImpl):
138+
@cython.locals(ivalue=libc.stdint.int64_t)
138139
cpdef bytes encode(self, value)
139140

140141

sdks/python/apache_beam/coders/coder_impl.py

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -90,20 +90,6 @@
9090
is_compiled = False
9191
fits_in_64_bits = lambda x: -(1 << 63) <= x <= (1 << 63) - 1
9292

93-
94-
def _as_signed_int64(value):
95-
# type: (int) -> int
96-
97-
"""Folds a uint64 to the signed int64 with the same bits (and VarInt
98-
encoding), which the Cython int64_t stream params accept. Values outside the
99-
64-bit range raise here so the pure-Python path matches Cython's overflow."""
100-
if (1 << 63) <= value < (1 << 64):
101-
return int(value) - (1 << 64)
102-
if not fits_in_64_bits(value):
103-
raise OverflowError("%d is out of range for a 64-bit integer." % value)
104-
return value
105-
106-
10793
if TYPE_CHECKING or SLOW_STREAM:
10894
from .slow_stream import ByteCountingOutputStream
10995
from .slow_stream import InputStream as create_InputStream
@@ -1058,26 +1044,22 @@ class VarIntCoderImpl(StreamCoderImpl):
10581044
def encode_to_stream(self, value, out, nested):
10591045
# type: (int, create_OutputStream, bool) -> None
10601046
try:
1061-
# Fold uint64 values into signed int64 so Cython doesn't overflow.
1062-
out.write_var_int64(_as_signed_int64(value))
1047+
out.write_var_int64(value)
10631048
except OverflowError as e:
10641049
raise OverflowError(
10651050
f"Integer value '{value}' is out of the encodable range for "
1066-
f"VarIntCoder. This coder is limited to 64-bit integers: the "
1067-
f"signed range -(2**63) to 2**63 - 1, plus unsigned values up to "
1068-
f"2**64 - 1 which share the same wire encoding. "
1051+
f"VarIntCoder. This coder is limited to values that fit "
1052+
f"within a 64-bit signed integer (-(2**63) to 2**63 - 1). "
10691053
f"Original error: {e}") from e
10701054

10711055
def decode_from_stream(self, in_stream, nested):
10721056
# type: (create_InputStream, bool) -> int
10731057
return in_stream.read_var_int64()
10741058

10751059
def encode(self, value):
1076-
# Compare as a Python object: a uint64 value overflows the int64_t cast
1077-
# the compiled fast path used to do here. Non-small values (including
1078-
# uint64) fall through to encode_to_stream, which folds them.
1079-
if 0 <= value < len(small_ints):
1080-
return small_ints[value]
1060+
ivalue = value # type cast
1061+
if 0 <= ivalue < len(small_ints):
1062+
return small_ints[ivalue]
10811063
return StreamCoderImpl.encode(self, value)
10821064

10831065
def decode(self, encoded):
@@ -1091,11 +1073,11 @@ def estimate_size(self, value, nested=False):
10911073
# type: (Any, bool) -> int
10921074
# Note that VarInts are encoded the same way regardless of nesting.
10931075
try:
1094-
return get_varint_size(_as_signed_int64(value))
1076+
return get_varint_size(value)
10951077
except OverflowError as e:
10961078
raise OverflowError(
10971079
f"Cannot estimate size for integer value '{value}'. "
1098-
f"Value is out of the range for VarIntCoder (64-bit integer). "
1080+
f"Value is out of the range for VarIntCoder (64-bit signed integer). "
10991081
f"Original error: {e}") from e
11001082

11011083

sdks/python/apache_beam/coders/coders_test_common.py

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -437,24 +437,6 @@ def test_varint_coder(self):
437437
for k in range(0, int(math.log(MAX_64_BIT_INT)))
438438
])
439439

440-
def test_varint_coder_uint64(self):
441-
# uint64 values [2**63, 2**64) must encode like the signed int64 with the
442-
# same bits instead of overflowing Cython's int64_t. Decoding is signed,
443-
# matching Java's VarIntCoder.
444-
coder = coders.VarIntCoder()
445-
impl = coder.get_impl()
446-
for v in [1 << 63, (1 << 63) + 12345, (1 << 64) - 1]:
447-
signed_twin = v - (1 << 64)
448-
encoded = coder.encode(v)
449-
self.assertEqual(encoded, coder.encode(signed_twin))
450-
self.assertEqual(impl.estimate_size(v), len(encoded))
451-
self.assertEqual(coder.decode(encoded), signed_twin)
452-
453-
# Values outside the 64-bit range stay out of range on both paths.
454-
for v in [1 << 64, (1 << 70), -(1 << 63) - 1]:
455-
with self.assertRaises(OverflowError):
456-
coder.encode(v)
457-
458440
def test_varint32_coder(self):
459441
# Small ints.
460442
self.check_coder(coders.VarInt32Coder(), *range(-10, 10))

0 commit comments

Comments
 (0)