diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py index 17b3d441960..169d590181f 100644 --- a/src/datasets/features/features.py +++ b/src/datasets/features/features.py @@ -117,6 +117,8 @@ def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: return "large_string" elif pyarrow.types.is_string_view(arrow_type): return "string_view" + elif pyarrow.types.is_fixed_size_binary(arrow_type): + return f"fixed_size_binary[{arrow_type.byte_width}]" elif pyarrow.types.is_dictionary(arrow_type): return _arrow_to_datasets_dtype(arrow_type.value_type) else: @@ -266,6 +268,11 @@ def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None): ], ) ) + + fixed_size_binary_matches = re.search(r"^fixed_size_binary\[(\d+)\]$", datasets_dtype) + if fixed_size_binary_matches: + byte_width = fixed_size_binary_matches.group(1) + return pa.binary(int(byte_width)) raise ValueError( f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. " diff --git a/tests/features/test_features.py b/tests/features/test_features.py index fe23dee5402..3e70dc1e525 100644 --- a/tests/features/test_features.py +++ b/tests/features/test_features.py @@ -76,6 +76,7 @@ def test_string_to_arrow_bijection_for_primitive_types(self): pa.int32(), pa.float64(), pa.array([datetime.time(1, 1, 1)]).type, # arrow type: DataType(time64[us]) + pa.binary(16), ] for dt in supported_pyarrow_datatypes: self.assertEqual(dt, string_to_arrow(_arrow_to_datasets_dtype(dt)))