Add fixed-size binary support

encodings/sparse/src/canonical.rs

@@ -9,6 +9,8 @@ use vortex_array::ArrayRef;
 use vortex_array::ExecutionCtx;
 use vortex_array::IntoArray;
 use vortex_array::arrays::BoolArray;
+use vortex_array::arrays::FixedSizeBinary;
+use vortex_array::arrays::FixedSizeBinaryArray;
 use vortex_array::arrays::FixedSizeList;
 use vortex_array::arrays::FixedSizeListArray;
 use vortex_array::arrays::ListView;
@@ -20,6 +22,7 @@ use vortex_array::arrays::Struct;
 use vortex_array::arrays::StructArray;
 use vortex_array::arrays::VarBinView;
 use vortex_array::arrays::VarBinViewArray;
+use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
 use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
 use vortex_array::arrays::listview::ListViewArrayExt;
 use vortex_array::arrays::primitive::PrimitiveArrayExt;
@@ -54,6 +57,7 @@ use vortex_buffer::BitBuffer;
 use vortex_buffer::Buffer;
 use vortex_buffer::BufferString;
 use vortex_buffer::ByteBuffer;
+use vortex_buffer::ByteBufferMut;
 use vortex_buffer::buffer;
 use vortex_buffer::buffer_mut;
 use vortex_error::VortexError;
@@ -140,6 +144,14 @@ pub(super) fn execute_sparse(parts: SparseParts, ctx: &mut ExecutionCtx) -> Vort
             let fill = fill_value.as_binary().value().cloned();
             execute_varbin(&patches, &fill_value, dtype.clone(), fill, len, ctx)?
         }
+        DType::FixedSizeBinary(byte_width, nullability) => execute_sparse_fixed_size_binary(
+            &patches,
+            &fill_value,
+            *byte_width,
+            *nullability,
+            len,
+            ctx,
+        )?,
         DType::List(values_dtype, nullability) => execute_sparse_lists(
             &patches,
             &fill_value,
@@ -165,6 +177,43 @@ pub(super) fn execute_sparse(parts: SparseParts, ctx: &mut ExecutionCtx) -> Vort
     })
 }
 
+fn execute_sparse_fixed_size_binary(
+    resolved: &Patches,
+    fill_scalar: &Scalar,
+    byte_width: u32,
+    nullability: Nullability,
+    len: usize,
+    ctx: &mut ExecutionCtx,
+) -> VortexResult<ArrayRef> {
+    let byte_width_usize = byte_width as usize;
+    let fill = fill_scalar
+        .as_binary()
+        .value()
+        .cloned()
+        .unwrap_or_else(|| ByteBuffer::zeroed(byte_width_usize));
+    let mut dense = ByteBufferMut::with_capacity(len.saturating_mul(byte_width_usize));
+    for _ in 0..len {
+        dense.extend_from_slice(fill.as_slice());
+    }
+
+    let indices = resolved.indices().as_::<Primitive>().into_owned();
+    let values = resolved.values().as_::<FixedSizeBinary>().into_owned();
+    let patch_bytes = values.buffer_handle().to_host_sync();
+    match_each_integer_ptype!(indices.ptype(), |I| {
+        for (patch_row, patch_index) in indices.as_slice::<I>().iter().enumerate() {
+            let patch_index = <usize as NumCast>::from(*patch_index)
+                .vortex_expect("fixed-size binary patch index must fit in usize");
+            let source = patch_row * byte_width_usize;
+            let target = patch_index * byte_width_usize;
+            dense[target..target + byte_width_usize]
+                .copy_from_slice(&patch_bytes[source..source + byte_width_usize]);
+        }
+    });
+
+    let validity = sparse_validity(resolved, fill_scalar, nullability, len, ctx)?;
+    Ok(FixedSizeBinaryArray::new(dense.freeze(), byte_width, len, validity).into_array())
+}
+
 fn execute_sparse_lists(
     resolved: &Patches,
     fill_value: &Scalar,

fuzz/src/array/compare.rs

@@ -174,7 +174,10 @@ pub fn compare_canonical_array(
                 result_nullability,
             )
         }
-        DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
+        DType::FixedSizeBinary(..)
+        | DType::List(..)
+        | DType::FixedSizeList(..)
+        | DType::Struct(..) => {
             let scalar_vals: Vec<Scalar> = (0..array.len())
                 .map(|i| array.execute_scalar(i, ctx).vortex_expect("scalar_at"))
                 .collect();

fuzz/src/array/fill_null.rs

@@ -43,6 +43,7 @@ pub fn fill_null_canonical_array(
             fill_primitive_array(array, fill_value, result_nullability, ctx)
         }
         Canonical::Decimal(array) => fill_decimal_array(array, fill_value, result_nullability, ctx),
+        Canonical::FixedSizeBinary(array) => array.into_array().fill_null(fill_value.clone())?,
         Canonical::VarBinView(array) => {
             fill_varbinview_array(array, fill_value, result_nullability, ctx)
         }

fuzz/src/array/filter.rs

@@ -97,7 +97,7 @@ pub fn filter_canonical_array(
                 .collect::<Vec<_>>();
             Ok(VarBinViewArray::from_iter(values, array.dtype().clone()).into_array())
         }
-        DType::List(..) | DType::FixedSizeList(..) => {
+        DType::FixedSizeBinary(..) | DType::List(..) | DType::FixedSizeList(..) => {
             let mut indices = Vec::new();
             for (idx, bool) in filter.iter().enumerate() {
                 if *bool {

fuzz/src/array/mask.rs

@@ -18,6 +18,7 @@ use vortex_array::arrays::VarBinViewArray;
 use vortex_array::arrays::bool::BoolArrayExt;
 use vortex_array::arrays::decimal::DecimalArrayExt;
 use vortex_array::arrays::extension::ExtensionArrayExt;
+use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
 use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
 use vortex_array::arrays::listview::ListViewArrayExt;
 use vortex_array::arrays::struct_::StructArrayExt;
@@ -92,6 +93,16 @@ pub fn mask_canonical_array(
                     .into_array()
             })
         }
+        Canonical::FixedSizeBinary(array) => {
+            let new_validity = mask_validity(&array.validity()?, mask, ctx);
+            vortex_array::arrays::FixedSizeBinaryArray::new(
+                array.buffer_handle().to_host_sync(),
+                array.byte_width(),
+                array.len(),
+                new_validity,
+            )
+            .into_array()
+        }
         Canonical::VarBinView(array) => {
             let new_validity = mask_validity(&array.validity()?, mask, ctx);
             VarBinViewArray::new_handle(

fuzz/src/array/mod.rs

@@ -486,7 +486,7 @@ fn actions_for_dtype(dtype: &DType) -> HashSet<ActionType> {
             // These support all actions
             ActionType::iter().collect()
         }
-        DType::Utf8(_) | DType::Binary(_) => {
+        DType::Utf8(_) | DType::Binary(_) | DType::FixedSizeBinary(..) => {
             // Utf8/Binary supports everything except Sum and FillNull
             // Actions: Compress, Slice, Take, SearchSorted, Filter, Compare, Cast, MinMax, Mask, ScalarAt
             [

fuzz/src/array/scalar_at.rs

@@ -9,6 +9,7 @@ use vortex_array::IntoArray;
 use vortex_array::arrays::bool::BoolArrayExt;
 use vortex_array::arrays::decimal::DecimalArrayExt;
 use vortex_array::arrays::extension::ExtensionArrayExt;
+use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
 use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
 use vortex_array::arrays::listview::ListViewArrayExt;
 use vortex_array::arrays::struct_::StructArrayExt;
@@ -39,19 +40,18 @@ pub fn scalar_at_canonical_array(
             array.to_bit_buffer().value(index),
             array.dtype().nullability(),
         ),
-        Canonical::Primitive(array) => {
-            match_each_native_ptype!(array.ptype(), |T| {
-                Scalar::primitive(array.as_slice::<T>()[index], array.dtype().nullability())
-            })
-        }
-        Canonical::Decimal(array) => {
-            match_each_decimal_value_type!(array.values_type(), |D| {
-                Scalar::decimal(
-                    DecimalValue::from(array.buffer::<D>()[index]),
-                    array.decimal_dtype(),
-                    array.dtype().nullability(),
-                )
-            })
+        Canonical::Primitive(array) => match_each_native_ptype!(array.ptype(), |T| {
+            Scalar::primitive(array.as_slice::<T>()[index], array.dtype().nullability())
+        }),
+        Canonical::Decimal(array) => match_each_decimal_value_type!(array.values_type(), |D| {
+            Scalar::decimal(
+                DecimalValue::from(array.buffer::<D>()[index]),
+                array.decimal_dtype(),
+                array.dtype().nullability(),
+            )
+        }),
+        Canonical::FixedSizeBinary(array) => {
+            Scalar::fixed_size_binary(array.value(index), array.dtype().nullability())
         }
         Canonical::VarBinView(array) => varbin_scalar(array.bytes_at(index), array.dtype()),
         Canonical::List(array) => {

fuzz/src/array/search_sorted.rs

@@ -143,7 +143,10 @@ pub fn search_sorted_canonical_array(
             };
             SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side)
         }
-        DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
+        DType::FixedSizeBinary(..)
+        | DType::List(..)
+        | DType::FixedSizeList(..)
+        | DType::Struct(..) => {
             let scalar_vals = (0..array.len())
                 .map(|i| array.execute_scalar(i, ctx))
                 .collect::<VortexResult<Vec<_>>>()?;

fuzz/src/array/slice.rs

@@ -15,6 +15,7 @@ use vortex_array::arrays::bool::BoolArrayExt;
 use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
 use vortex_array::arrays::listview::ListViewArrayExt;
 use vortex_array::arrays::struct_::StructArrayExt;
+use vortex_array::builders::builder_with_capacity;
 use vortex_array::dtype::DType;
 use vortex_array::match_each_decimal_value_type;
 use vortex_array::match_each_native_ptype;
@@ -124,6 +125,13 @@ pub fn slice_canonical_array(
             )
             .map(|a| a.into_array())
         }
+        DType::FixedSizeBinary(..) => {
+            let mut builder = builder_with_capacity(array.dtype(), stop - start);
+            for index in start..stop {
+                builder.append_scalar(&array.execute_scalar(index, ctx)?)?;
+            }
+            Ok(builder.finish())
+        }
         d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
             unreachable!("DType {d} not supported for fuzzing")
         }

fuzz/src/array/sort.rs

@@ -92,7 +92,10 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR
             opt_values.sort();
             Ok(VarBinViewArray::from_iter(opt_values, array.dtype().clone()).into_array())
         }
-        DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
+        DType::FixedSizeBinary(..)
+        | DType::List(..)
+        | DType::FixedSizeList(..)
+        | DType::Struct(..) => {
             let mut sort_indices = (0..array.len()).collect::<Vec<_>>();
             sort_indices.sort_by(|a, b| {
                 let lhs = array.execute_scalar(*a, ctx).vortex_expect("scalar_at");

fuzz/src/array/take.rs

@@ -114,7 +114,7 @@ pub fn take_canonical_array(
             )
             .into_array())
         }
-        DType::List(..) | DType::FixedSizeList(..) => {
+        DType::FixedSizeBinary(..) | DType::List(..) | DType::FixedSizeList(..) => {
             let mut builder = builder_with_capacity(
                 &array.dtype().union_nullability(nullable),
                 indices_slice_non_opt.len(),

lang/cpp/include/vortex/dtype.hpp

@@ -37,6 +37,8 @@ enum class DataTypeVariant {
     Decimal = DTYPE_DECIMAL,
     // Nested fixed-size list
     FixedSizeList = DTYPE_FIXED_SIZE_LIST,
+    // Fixed-size binary data
+    FixedSizeBinary = DTYPE_FIXED_SIZE_BINARY,
 };
 
 // Primitive type
@@ -95,6 +97,7 @@ class DataType {
     DataType list_element() const;
     DataType fixed_size_list_element() const;
     uint32_t fixed_size_list_size() const;
+    uint32_t fixed_size_binary_size() const;
 
 private:
     friend struct detail::Access;
@@ -135,6 +138,7 @@ DataType float32(bool nullable = false);
 DataType float64(bool nullable = false);
 DataType utf8(bool nullable = false);
 DataType binary(bool nullable = false);
+DataType fixed_size_binary(uint32_t byte_width, bool nullable = false);
 DataType decimal(uint8_t precision, int8_t scale, bool nullable = false);
 DataType list(DataType element, bool nullable = false);
 DataType fixed_size_list(DataType element, uint32_t size, bool nullable = false);

lang/cpp/src/dtype.cpp

@@ -124,6 +124,10 @@ uint32_t DataType::fixed_size_list_size() const {
     return vx_dtype_fixed_size_list_size(handle_.get());
 }
 
+uint32_t DataType::fixed_size_binary_size() const {
+    return vx_dtype_fixed_size_binary_size(handle_.get());
+}
+
 namespace dtype {
 
 DataType null() {
@@ -174,6 +178,9 @@ DataType utf8(bool nullable) {
 DataType binary(bool nullable) {
     return Access::adopt<DataType>(vx_dtype_new_binary(nullable));
 }
+DataType fixed_size_binary(uint32_t byte_width, bool nullable) {
+    return Access::adopt<DataType>(vx_dtype_new_fixed_size_binary(byte_width, nullable));
+}
 DataType decimal(uint8_t precision, int8_t scale, bool nullable) {
     return Access::adopt<DataType>(vx_dtype_new_decimal(precision, scale, nullable));
 }

lang/cpp/tests/dtype.cpp

@@ -26,6 +26,13 @@ TEST_CASE("Decimal dtype", "[dtype]") {
     REQUIRE_THROWS_AS(d.list_element(), VortexException);
 }
 
+TEST_CASE("Fixed-size binary dtype", "[dtype]") {
+    auto d = dtype::fixed_size_binary(16, dtype::Nullable);
+    REQUIRE(d.variant() == DataTypeVariant::FixedSizeBinary);
+    REQUIRE(d.fixed_size_binary_size() == 16);
+    REQUIRE(d.nullable());
+}
+
 TEST_CASE("copy dtype", "[dtype]") {
     auto d = dtype::int32(true);
     DataType d2 = d;

vortex-array/src/aggregate_fn/fns/all_non_distinct/mod.rs

@@ -257,6 +257,10 @@ fn check_canonical_identical(
             check_primitive_identical(lhs, rhs)
         }
         (Canonical::Decimal(lhs), Canonical::Decimal(rhs)) => check_decimal_identical(lhs, rhs),
+        (Canonical::FixedSizeBinary(lhs), Canonical::FixedSizeBinary(rhs)) => {
+            Ok(lhs.buffer_handle().to_host_sync().as_slice()
+                == rhs.buffer_handle().to_host_sync().as_slice())
+        }
         (Canonical::VarBinView(lhs), Canonical::VarBinView(rhs)) => {
             check_varbinview_identical(lhs, rhs)
         }

vortex-array/src/aggregate_fn/fns/is_constant/mod.rs

@@ -10,6 +10,7 @@ pub mod primitive;
 mod struct_;
 mod varbin;
 
+use itertools::Itertools;
 use vortex_error::VortexExpect;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
@@ -35,6 +36,7 @@ use crate::aggregate_fn::DynAccumulator;
 use crate::aggregate_fn::EmptyOptions;
 use crate::arrays::Constant;
 use crate::arrays::Null;
+use crate::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
 use crate::builtins::ArrayBuiltins;
 use crate::dtype::DType;
 use crate::dtype::FieldNames;
@@ -395,10 +397,19 @@ impl AggregateFnVTable for IsConstant {
                 }
 
                 let batch_is_constant = match c {
-                    Canonical::Primitive(p) => check_primitive_constant(p),
+                    Canonical::Primitive(a) => check_primitive_constant(a),
+                    Canonical::Decimal(a) => check_decimal_constant(a),
+                    Canonical::FixedSizeBinary(a) => {
+                        let byte_width = a.byte_width() as usize;
+                        let values = a.buffer_handle().to_host_sync();
+                        values
+                            .as_slice()
+                            .chunks_exact(byte_width.max(1))
+                            .map(|value| if byte_width == 0 { &[][..] } else { value })
+                            .all_equal()
+                    }
                     Canonical::Bool(b) => check_bool_constant(b),
                     Canonical::VarBinView(v) => check_varbinview_constant(v),
-                    Canonical::Decimal(d) => check_decimal_constant(d),
                     Canonical::Struct(s) => check_struct_constant(s, ctx)?,
                     Canonical::Extension(e) => check_extension_constant(e, ctx)?,
                     Canonical::List(l) => check_listview_constant(l, ctx)?,