int_to_binary

Author: coderfender

docs/source/user-guide/latest/compatibility.md

@@ -73,16 +73,17 @@ should not be used in production. The feature will be enabled in a future releas
 
 Cast operations in Comet fall into three levels of support:
 
-- **C (Compatible)**: The results match Apache Spark
-- **I (Incompatible)**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
+- **Compatible**: The results match Apache Spark
+- **Incompatible**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
   will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting
   `spark.comet.expression.Cast.allowIncompatible=true` will allow all incompatible casts to run natively in Comet, but this is not
   recommended for production use.
-- **U (Unsupported)**: Comet does not provide a native version of this cast expression and the query stage will fall back to
+- **Unsupported**: Comet does not provide a native version of this cast expression and the query stage will fall back to
   Spark.
-- **N/A**: Spark does not support this cast.
 
-### Legacy Mode
+### Compatible Casts
+
+The following cast operations are generally compatible with Spark except for the differences noted here.
 
 <!--BEGIN:CAST_LEGACY_TABLE-->
 <!--END:CAST_LEGACY_TABLE-->

native/spark-expr/src/conversion_funcs/cast.rs

@@ -20,13 +20,13 @@ use crate::{timezone, BinaryOutputStyle};
 use crate::{EvalMode, SparkError, SparkResult};
 use arrow::array::builder::StringBuilder;
 use arrow::array::{
-    BooleanBuilder, Decimal128Builder, DictionaryArray, GenericByteArray, ListArray,
+    BinaryBuilder, BooleanBuilder, Decimal128Builder, DictionaryArray, GenericByteArray, ListArray,
     PrimitiveBuilder, StringArray, StructArray,
 };
 use arrow::compute::can_cast_types;
 use arrow::datatypes::{
-    i256, ArrowDictionaryKeyType, ArrowNativeType, DataType, Decimal256Type, GenericBinaryType,
-    Schema,
+    i256, ArrowDictionaryKeyType, ArrowNativeType, DataType, Decimal256Type,
+    GenericBinaryType, Schema,
 };
 use arrow::{
     array::{
@@ -311,22 +311,22 @@ fn can_cast_from_byte(to_type: &DataType, _: &SparkCastOptions) -> bool {
     use DataType::*;
     matches!(
         to_type,
-        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _)
+        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _) | Binary
     )
 }
 
 fn can_cast_from_short(to_type: &DataType, _: &SparkCastOptions) -> bool {
     use DataType::*;
     matches!(
         to_type,
-        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _)
+        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _) | Binary
     )
 }
 
 fn can_cast_from_int(to_type: &DataType, options: &SparkCastOptions) -> bool {
     use DataType::*;
     match to_type {
-        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 => true,
+        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 | Binary => true,
         Decimal128(_, _) => {
             // incompatible: no overflow check
             options.allow_incompat
@@ -338,7 +338,7 @@ fn can_cast_from_int(to_type: &DataType, options: &SparkCastOptions) -> bool {
 fn can_cast_from_long(to_type: &DataType, options: &SparkCastOptions) -> bool {
     use DataType::*;
     match to_type {
-        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 => true,
+        Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Binary => true,
         Decimal128(_, _) => {
             // incompatible: no overflow check
             options.allow_incompat
@@ -501,6 +501,29 @@ macro_rules! cast_float_to_string {
     }};
 }
 
+// eval mode is not needed since all ints can be implemented in binary format
+macro_rules! cast_whole_num_to_binary {
+    ($array:expr, $primitive_type:ty, $byte_size:expr) => {{
+        let input_arr = $array
+            .as_any()
+            .downcast_ref::<$primitive_type>()
+            .ok_or_else(|| SparkError::Internal("Expected numeric array".to_string()))?;
+
+        let len = input_arr.len();
+        let mut builder = BinaryBuilder::with_capacity(len, len * $byte_size);
+
+        for i in 0..input_arr.len() {
+            if input_arr.is_null(i) {
+                builder.append_null();
+            } else {
+                builder.append_value(input_arr.value(i).to_be_bytes());
+            }
+        }
+
+        Ok(Arc::new(builder.finish()) as ArrayRef)
+    }};
+}
+
 macro_rules! cast_int_to_int_macro {
     (
         $array: expr,
@@ -1100,6 +1123,10 @@ fn cast_array(
             Ok(cast_with_options(&array, to_type, &CAST_OPTIONS)?)
         }
         (Binary, Utf8) => Ok(cast_binary_to_string::<i32>(&array, cast_options)?),
+        (Int8, Binary) => cast_whole_num_to_binary!(&array, Int8Array, 1),
+        (Int16, Binary) => cast_whole_num_to_binary!(&array, Int16Array, 2),
+        (Int32, Binary) => cast_whole_num_to_binary!(&array, Int32Array, 4),
+        (Int64, Binary) => cast_whole_num_to_binary!(&array, Int64Array, 8),
         _ if cast_options.is_adapting_schema
             || is_datafusion_spark_compatible(from_type, to_type) =>
         {

spark/src/main/scala/org/apache/comet/expressions/CometCast.scala

@@ -21,7 +21,7 @@ package org.apache.comet.expressions
 
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, NullType, StructType}
+import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, DataTypes, DecimalType, NullType, StructType}
 
 import org.apache.comet.CometConf
 import org.apache.comet.CometSparkSessionExtensions.withInfo
@@ -126,6 +126,7 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
         isSupported(dt.elementType, DataTypes.StringType, timeZoneId, evalMode)
       case (dt: ArrayType, dt1: ArrayType) =>
         isSupported(dt.elementType, dt1.elementType, timeZoneId, evalMode)
+      case (from: DataType, _: BinaryType) => canCastToBinary(from)
       case (dt: DataType, _) if dt.typeName == "timestamp_ntz" =>
         // https://github.com/apache/datafusion-comet/issues/378
         toType match {
@@ -344,6 +345,12 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
     case _ => Unsupported(Some(s"Cast from DecimalType to $toType is not supported"))
   }
 
+  private def canCastToBinary(fromType: DataType): SupportLevel = fromType match {
+    case DataTypes.ByteType | DataTypes.ShortType | DataTypes.IntegerType | DataTypes.LongType =>
+      Compatible()
+    case _ => Unsupported(Some(s"Cast from BinaryType to $fromType is not supported"))
+  }
+
   private def unsupported(fromType: DataType, toType: DataType): Unsupported = {
     Unsupported(Some(s"Cast from $fromType to $toType is not supported"))
   }

spark/src/test/scala/org/apache/comet/CometCastSuite.scala

@@ -281,7 +281,7 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
       hasIncompatibleType = usingParquetExecWithIncompatTypes)
   }
 
-  ignore("cast ShortType to BinaryType") {
+  test("cast ShortType to BinaryType") {
     castTest(
       generateShorts(),
       DataTypes.BinaryType,