From d81702ba82b1436d73bd7e03b49c150ce77813bf Mon Sep 17 00:00:00 2001 From: Alexis Placet Date: Wed, 10 Jun 2026 15:55:09 +0200 Subject: [PATCH 1/2] Support fixed_size_binary and list pyarrow types in features dtype conversion - Map fixed_size_binary to "fixed_size_binary[n]" and list to "list[]" in _arrow_to_datasets_dtype - Parse fixed_size_binary[...] in string_to_arrow and return pa.binary(byte_width) - Add test coverage for fixed_size_binary roundtrip --- src/datasets/features/features.py | 9 +++++++++ tests/features/test_features.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py index 17b3d441960..453fb4b748b 100644 --- a/src/datasets/features/features.py +++ b/src/datasets/features/features.py @@ -117,6 +117,10 @@ def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: return "large_string" elif pyarrow.types.is_string_view(arrow_type): return "string_view" + elif pyarrow.types.is_fixed_size_binary(arrow_type): + return f"fixed_size_binary[{arrow_type.byte_width}]" + elif pyarrow.types.is_list(arrow_type): + return f"list[{_arrow_to_datasets_dtype(arrow_type.value_type)}]" elif pyarrow.types.is_dictionary(arrow_type): return _arrow_to_datasets_dtype(arrow_type.value_type) else: @@ -266,6 +270,11 @@ def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None): ], ) ) + + fixed_size_binary_matches = re.search(r"^fixed_size_binary\[(\d+)\]$", datasets_dtype) + if fixed_size_binary_matches: + byte_width = fixed_size_binary_matches.group(1) + return pa.binary(int(byte_width)) raise ValueError( f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. " diff --git a/tests/features/test_features.py b/tests/features/test_features.py index fe23dee5402..3e70dc1e525 100644 --- a/tests/features/test_features.py +++ b/tests/features/test_features.py @@ -76,6 +76,7 @@ def test_string_to_arrow_bijection_for_primitive_types(self): pa.int32(), pa.float64(), pa.array([datetime.time(1, 1, 1)]).type, # arrow type: DataType(time64[us]) + pa.binary(16), ] for dt in supported_pyarrow_datatypes: self.assertEqual(dt, string_to_arrow(_arrow_to_datasets_dtype(dt))) From ea76ca426661455f532f52c2ab5fd06b373bc87c Mon Sep 17 00:00:00 2001 From: Alexis Placet Date: Tue, 23 Jun 2026 14:44:33 +0200 Subject: [PATCH 2/2] wip --- src/datasets/features/features.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py index 453fb4b748b..169d590181f 100644 --- a/src/datasets/features/features.py +++ b/src/datasets/features/features.py @@ -119,8 +119,6 @@ def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: return "string_view" elif pyarrow.types.is_fixed_size_binary(arrow_type): return f"fixed_size_binary[{arrow_type.byte_width}]" - elif pyarrow.types.is_list(arrow_type): - return f"list[{_arrow_to_datasets_dtype(arrow_type.value_type)}]" elif pyarrow.types.is_dictionary(arrow_type): return _arrow_to_datasets_dtype(arrow_type.value_type) else: