Skip to content

Commit 58d30b2

Browse files
committed
[python] Probe one null row so decimal-precision casts are validated
can_execute_cast probed an empty array, which only resolves the PyArrow cast kernel. Some kernels still reject the target type parameters on non-empty input: INT -> DECIMAL(10, 2) has a kernel but needs precision >= 12 to hold an int's range at scale 2 (BIGINT needs >= 21), so the empty probe accepted it, the alter succeeded, and reading an old file failed with ArrowInvalid "Precision is not great enough". Probe a single null row instead. It triggers PyArrow's static type-parameter validation (decimal precision sufficiency) while a null value avoids per-value parse/overflow errors that safe=False -- matching the read path -- tolerates.
1 parent ba2eb84 commit 58d30b2

3 files changed

Lines changed: 39 additions & 8 deletions

File tree

paimon-python/pypaimon/casting/data_type_casts.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,11 +239,16 @@ def _pyarrow_cast_supported(source_type, target_type) -> bool:
239239
cached = _EXECUTABLE_CAST_CACHE.get(cache_key)
240240
if cached is not None:
241241
return cached
242-
# An empty-array cast exercises only kernel resolution (not per-value
243-
# conversion), so it reports whether PyArrow has a cast kernel for the pair
244-
# without depending on any data. ``safe=False`` matches the read path.
242+
# Probe a one-row (null-valued) array rather than an empty one. An empty
243+
# array only resolves the cast kernel; some kernels additionally reject the
244+
# target type parameters on any non-empty input -- e.g. INT -> DECIMAL(10,2)
245+
# has a kernel but needs precision >= 12 to hold an int's range at scale 2,
246+
# so an empty probe passes yet the read later fails with ArrowInvalid. A
247+
# single null row triggers that static type-parameter validation while
248+
# avoiding per-value parse/overflow errors (which ``safe=False`` -- matching
249+
# the read path -- tolerates anyway).
245250
try:
246-
pa.array([], type=source_pa).cast(target_pa, safe=False)
251+
pa.nulls(1, type=source_pa).cast(target_pa, safe=False)
247252
ok = True
248253
except (pa.lib.ArrowNotImplementedError, pa.lib.ArrowInvalid,
249254
pa.lib.ArrowTypeError):

paimon-python/pypaimon/tests/filesystem_catalog_test.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -283,13 +283,24 @@ def test_update_column_type_rejects_non_executable_cast(self):
283283
False)
284284
self.assertIn("no executable cast", str(ctx.exception))
285285

286-
# An executable widening still succeeds.
286+
# INT -> DECIMAL(10, 2) has a PyArrow kernel but the target precision is
287+
# too small to hold an int's range at scale 2 (needs >= 12); the read
288+
# path would fail with ArrowInvalid, so reject it at alter time too.
289+
with self.assertRaises(RuntimeError) as ctx:
290+
catalog.alter_table(
291+
identifier,
292+
[SchemaChange.update_column_type(
293+
"k", AtomicType("DECIMAL(10, 2)"))],
294+
False)
295+
self.assertIn("no executable cast", str(ctx.exception))
296+
297+
# A wide-enough DECIMAL is executable and succeeds.
287298
catalog.alter_table(
288299
identifier,
289-
[SchemaChange.update_column_type("k", AtomicType("BIGINT"))],
300+
[SchemaChange.update_column_type("k", AtomicType("DECIMAL(12, 2)"))],
290301
False)
291302
table = catalog.get_table(identifier)
292-
self.assertEqual(table.fields[0].type.type, "BIGINT")
303+
self.assertEqual(table.fields[0].type.type, "DECIMAL(12, 2)")
293304

294305
def test_add_column_before_partition(self):
295306
catalog = CatalogFactory.create({

paimon-python/pypaimon/tests/schema_evolution_nested_read_test.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import pyarrow as pa
3737

3838
from pypaimon import CatalogFactory, Schema
39-
from pypaimon.casting.data_type_casts import supports_cast
39+
from pypaimon.casting.data_type_casts import can_execute_cast, supports_cast
4040
from pypaimon.schema.data_types import (ArrayType, AtomicInteger, AtomicType,
4141
DataField, MapType, MultisetType,
4242
PyarrowFieldParser, RowType,
@@ -728,6 +728,21 @@ def test_unsupported_casts(self):
728728
self.assertFalse(supports_cast(AtomicType(src), AtomicType(dst)),
729729
'{} -> {}'.format(src, dst))
730730

731+
def test_can_execute_cast_decimal_precision(self):
732+
# A numeric -> decimal cast has a PyArrow kernel but is only executable
733+
# when the target precision can hold the source's range at the target
734+
# scale (INT needs >= 12 at scale 2, BIGINT >= 21). An empty-array probe
735+
# misses this; can_execute_cast must reject the too-small targets so the
736+
# read path does not later fail with ArrowInvalid.
737+
for src, dst in [('INT', 'DECIMAL(10, 2)'), ('BIGINT', 'DECIMAL(10, 2)'),
738+
('BIGINT', 'DECIMAL(20, 2)')]:
739+
self.assertFalse(can_execute_cast(AtomicType(src), AtomicType(dst)),
740+
'{} -> {}'.format(src, dst))
741+
for src, dst in [('INT', 'DECIMAL(12, 2)'), ('BIGINT', 'DECIMAL(21, 2)'),
742+
('INT', 'BIGINT'), ('DOUBLE', 'INT'), ('INT', 'STRING')]:
743+
self.assertTrue(can_execute_cast(AtomicType(src), AtomicType(dst)),
744+
'{} -> {}'.format(src, dst))
745+
731746
def test_constructed_to_string(self):
732747
# ROW/ARRAY/MAP have a read-time string rendering; vector and
733748
# multiset do not, so their type change must be rejected.

0 commit comments

Comments
 (0)