Skip to content

Commit 6f8e1c6

Browse files
author
B Vadlamani
committed
int_to_binary
1 parent 4bd664e commit 6f8e1c6

4 files changed

Lines changed: 145 additions & 104 deletions

File tree

‎docs/source/user-guide/latest/compatibility.md‎

Lines changed: 102 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -73,118 +73,125 @@ should not be used in production. The feature will be enabled in a future releas
7373

7474
Cast operations in Comet fall into three levels of support:
7575

76-
- **C (Compatible)**: The results match Apache Spark
77-
- **I (Incompatible)**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
76+
- **Compatible**: The results match Apache Spark
77+
- **Incompatible**: The results may match Apache Spark for some inputs, but there are known issues where some inputs
7878
will result in incorrect results or exceptions. The query stage will fall back to Spark by default. Setting
7979
`spark.comet.expression.Cast.allowIncompatible=true` will allow all incompatible casts to run natively in Comet, but this is not
8080
recommended for production use.
81-
- **U (Unsupported)**: Comet does not provide a native version of this cast expression and the query stage will fall back to
81+
- **Unsupported**: Comet does not provide a native version of this cast expression and the query stage will fall back to
8282
Spark.
83-
- **N/A**: Spark does not support this cast.
8483

85-
### Legacy Mode
84+
### Compatible Casts
8685

87-
<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
88-
89-
<!--BEGIN:CAST_LEGACY_TABLE-->
90-
<!-- prettier-ignore-start -->
91-
| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp |
92-
|---|---|---|---|---|---|---|---|---|---|---|---|---|
93-
| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A |
94-
| boolean | N/A | - | C | N/A | U | C | C | C | C | C | C | U |
95-
| byte | U | C | - | N/A | C | C | C | C | C | C | C | U |
96-
| date | N/A | U | U | - | U | U | U | U | U | U | C | U |
97-
| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | U |
98-
| double | N/A | C | C | N/A | I | - | C | C | C | C | C | U |
99-
| float | N/A | C | C | N/A | I | C | - | C | C | C | C | U |
100-
| integer | U | C | C | N/A | C | C | C | - | C | C | C | U |
101-
| long | U | C | C | N/A | C | C | C | C | - | C | C | U |
102-
| short | U | C | C | N/A | C | C | C | C | C | - | C | U |
103-
| string | C | C | C | C | I | C | C | C | C | C | - | I |
104-
| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - |
105-
<!-- prettier-ignore-end -->
106-
107-
**Notes:**
108-
109-
- **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
110-
- **double -> decimal**: There can be rounding differences
111-
- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
112-
- **float -> decimal**: There can be rounding differences
113-
- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
114-
- **string -> date**: Only supports years between 262143 BC and 262142 AD
115-
- **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
116-
or strings containing null bytes (e.g \\u0000)
117-
- **string -> timestamp**: Not all valid formats are supported
118-
<!--END:CAST_LEGACY_TABLE-->
119-
120-
### Try Mode
86+
The following cast operations are generally compatible with Spark except for the differences noted here.
12187

12288
<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
12389

124-
<!--BEGIN:CAST_TRY_TABLE-->
90+
<!--BEGIN:COMPAT_CAST_TABLE-->
12591
<!-- prettier-ignore-start -->
126-
| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp |
127-
|---|---|---|---|---|---|---|---|---|---|---|---|---|
128-
| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A |
129-
| boolean | N/A | - | C | N/A | U | C | C | C | C | C | C | U |
130-
| byte | U | C | - | N/A | C | C | C | C | C | C | C | U |
131-
| date | N/A | U | U | - | U | U | U | U | U | U | C | U |
132-
| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | U |
133-
| double | N/A | C | C | N/A | I | - | C | C | C | C | C | U |
134-
| float | N/A | C | C | N/A | I | C | - | C | C | C | C | U |
135-
| integer | U | C | C | N/A | C | C | C | - | C | C | C | U |
136-
| long | U | C | C | N/A | C | C | C | C | - | C | C | U |
137-
| short | U | C | C | N/A | C | C | C | C | C | - | C | U |
138-
| string | C | C | C | C | I | C | C | C | C | C | - | I |
139-
| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - |
92+
| From Type | To Type | Notes |
93+
|-|-|-|
94+
| boolean | byte | |
95+
| boolean | short | |
96+
| boolean | integer | |
97+
| boolean | long | |
98+
| boolean | float | |
99+
| boolean | double | |
100+
| boolean | string | |
101+
| byte | boolean | |
102+
| byte | short | |
103+
| byte | integer | |
104+
| byte | long | |
105+
| byte | float | |
106+
| byte | double | |
107+
| byte | decimal | |
108+
| byte | string | |
109+
| byte | binary | |
110+
| short | boolean | |
111+
| short | byte | |
112+
| short | integer | |
113+
| short | long | |
114+
| short | float | |
115+
| short | double | |
116+
| short | decimal | |
117+
| short | string | |
118+
| short | binary | |
119+
| integer | boolean | |
120+
| integer | byte | |
121+
| integer | short | |
122+
| integer | long | |
123+
| integer | float | |
124+
| integer | double | |
125+
| integer | decimal | |
126+
| integer | string | |
127+
| integer | binary | |
128+
| long | boolean | |
129+
| long | byte | |
130+
| long | short | |
131+
| long | integer | |
132+
| long | float | |
133+
| long | double | |
134+
| long | decimal | |
135+
| long | string | |
136+
| long | binary | |
137+
| float | boolean | |
138+
| float | byte | |
139+
| float | short | |
140+
| float | integer | |
141+
| float | long | |
142+
| float | double | |
143+
| float | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
144+
| double | boolean | |
145+
| double | byte | |
146+
| double | short | |
147+
| double | integer | |
148+
| double | long | |
149+
| double | float | |
150+
| double | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
151+
| decimal | boolean | |
152+
| decimal | byte | |
153+
| decimal | short | |
154+
| decimal | integer | |
155+
| decimal | long | |
156+
| decimal | float | |
157+
| decimal | double | |
158+
| decimal | decimal | |
159+
| decimal | string | There can be formatting differences in some case due to Spark using scientific notation where Comet does not |
160+
| string | boolean | |
161+
| string | byte | |
162+
| string | short | |
163+
| string | integer | |
164+
| string | long | |
165+
| string | float | |
166+
| string | double | |
167+
| string | date | Only supports years between 262143 BC and 262142 AD |
168+
| binary | string | |
169+
| date | string | |
170+
| timestamp | long | |
171+
| timestamp | string | |
172+
| timestamp | date | |
140173
<!-- prettier-ignore-end -->
174+
<!--END:COMPAT_CAST_TABLE-->
141175

142-
**Notes:**
176+
### Incompatible Casts
143177

144-
- **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
145-
- **double -> decimal**: There can be rounding differences
146-
- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
147-
- **float -> decimal**: There can be rounding differences
148-
- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
149-
- **string -> date**: Only supports years between 262143 BC and 262142 AD
150-
- **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
151-
or strings containing null bytes (e.g \\u0000)
152-
- **string -> timestamp**: Not all valid formats are supported
153-
<!--END:CAST_TRY_TABLE-->
154-
155-
### ANSI Mode
178+
The following cast operations are not compatible with Spark for all inputs and are disabled by default.
156179

157180
<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
158181

159-
<!--BEGIN:CAST_ANSI_TABLE-->
182+
<!--BEGIN:INCOMPAT_CAST_TABLE-->
160183
<!-- prettier-ignore-start -->
161-
| | binary | boolean | byte | date | decimal | double | float | integer | long | short | string | timestamp |
162-
|---|---|---|---|---|---|---|---|---|---|---|---|---|
163-
| binary | - | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | N/A | C | N/A |
164-
| boolean | N/A | - | C | N/A | U | C | C | C | C | C | C | U |
165-
| byte | U | C | - | N/A | C | C | C | C | C | C | C | U |
166-
| date | N/A | U | U | - | U | U | U | U | U | U | C | U |
167-
| decimal | N/A | C | C | N/A | - | C | C | C | C | C | C | U |
168-
| double | N/A | C | C | N/A | I | - | C | C | C | C | C | U |
169-
| float | N/A | C | C | N/A | I | C | - | C | C | C | C | U |
170-
| integer | U | C | C | N/A | C | C | C | - | C | C | C | U |
171-
| long | U | C | C | N/A | C | C | C | C | - | C | C | U |
172-
| short | U | C | C | N/A | C | C | C | C | C | - | C | U |
173-
| string | C | C | C | C | I | C | C | C | C | C | - | I |
174-
| timestamp | N/A | U | U | C | U | U | U | U | C | U | C | - |
184+
| From Type | To Type | Notes |
185+
|-|-|-|
186+
| float | decimal | There can be rounding differences |
187+
| double | decimal | There can be rounding differences |
188+
| string | decimal | Does not support fullwidth unicode digits (e.g \\uFF10)
189+
or strings containing null bytes (e.g \\u0000) |
190+
| string | timestamp | Not all valid formats are supported |
175191
<!-- prettier-ignore-end -->
192+
<!--END:INCOMPAT_CAST_TABLE-->
176193

177-
**Notes:**
178-
179-
- **decimal -> string**: There can be formatting differences in some case due to Spark using scientific notation where Comet does not
180-
- **double -> decimal**: There can be rounding differences
181-
- **double -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
182-
- **float -> decimal**: There can be rounding differences
183-
- **float -> string**: There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45
184-
- **string -> date**: Only supports years between 262143 BC and 262142 AD
185-
- **string -> decimal**: Does not support fullwidth unicode digits (e.g \\uFF10)
186-
or strings containing null bytes (e.g \\u0000)
187-
- **string -> timestamp**: ANSI mode not supported
188-
<!--END:CAST_ANSI_TABLE-->
194+
### Unsupported Casts
189195

190-
See the [tracking issue](https://github.com/apache/datafusion-comet/issues/286) for more details.
196+
Any cast not listed in the previous tables is currently unsupported. We are working on adding more. See the
197+
[tracking issue](https://github.com/apache/datafusion-comet/issues/286) for more details.

‎native/spark-expr/src/conversion_funcs/cast.rs‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ use crate::{timezone, BinaryOutputStyle};
2020
use crate::{EvalMode, SparkError, SparkResult};
2121
use arrow::array::builder::StringBuilder;
2222
use arrow::array::{
23-
BooleanBuilder, Decimal128Builder, DictionaryArray, GenericByteArray, ListArray,
23+
BinaryBuilder, BooleanBuilder, Decimal128Builder, DictionaryArray, GenericByteArray, ListArray,
2424
PrimitiveBuilder, StringArray, StructArray,
2525
};
2626
use arrow::compute::can_cast_types;
2727
use arrow::datatypes::{
28-
i256, ArrowDictionaryKeyType, ArrowNativeType, DataType, Decimal256Type, GenericBinaryType,
29-
Schema,
28+
i256, ArrowDictionaryKeyType, ArrowNativeType, DataType, Decimal256Type,
29+
GenericBinaryType, Schema,
3030
};
3131
use arrow::{
3232
array::{
@@ -311,22 +311,22 @@ fn can_cast_from_byte(to_type: &DataType, _: &SparkCastOptions) -> bool {
311311
use DataType::*;
312312
matches!(
313313
to_type,
314-
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _)
314+
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _) | Binary
315315
)
316316
}
317317

318318
fn can_cast_from_short(to_type: &DataType, _: &SparkCastOptions) -> bool {
319319
use DataType::*;
320320
matches!(
321321
to_type,
322-
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _)
322+
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Decimal128(_, _) | Binary
323323
)
324324
}
325325

326326
fn can_cast_from_int(to_type: &DataType, options: &SparkCastOptions) -> bool {
327327
use DataType::*;
328328
match to_type {
329-
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 => true,
329+
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 | Binary => true,
330330
Decimal128(_, _) => {
331331
// incompatible: no overflow check
332332
options.allow_incompat
@@ -338,7 +338,7 @@ fn can_cast_from_int(to_type: &DataType, options: &SparkCastOptions) -> bool {
338338
fn can_cast_from_long(to_type: &DataType, options: &SparkCastOptions) -> bool {
339339
use DataType::*;
340340
match to_type {
341-
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 => true,
341+
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Binary => true,
342342
Decimal128(_, _) => {
343343
// incompatible: no overflow check
344344
options.allow_incompat
@@ -501,6 +501,29 @@ macro_rules! cast_float_to_string {
501501
}};
502502
}
503503

504+
// eval mode is not needed since all ints can be implemented in binary format
505+
macro_rules! cast_whole_num_to_binary {
506+
($array:expr, $primitive_type:ty, $byte_size:expr) => {{
507+
let input_arr = $array
508+
.as_any()
509+
.downcast_ref::<$primitive_type>()
510+
.ok_or_else(|| SparkError::Internal("Expected numeric array".to_string()))?;
511+
512+
let len = input_arr.len();
513+
let mut builder = BinaryBuilder::with_capacity(len, len * $byte_size);
514+
515+
for i in 0..input_arr.len() {
516+
if input_arr.is_null(i) {
517+
builder.append_null();
518+
} else {
519+
builder.append_value(input_arr.value(i).to_be_bytes());
520+
}
521+
}
522+
523+
Ok(Arc::new(builder.finish()) as ArrayRef)
524+
}};
525+
}
526+
504527
macro_rules! cast_int_to_int_macro {
505528
(
506529
$array: expr,
@@ -1096,6 +1119,10 @@ fn cast_array(
10961119
Ok(cast_with_options(&array, to_type, &CAST_OPTIONS)?)
10971120
}
10981121
(Binary, Utf8) => Ok(cast_binary_to_string::<i32>(&array, cast_options)?),
1122+
(Int8, Binary) => cast_whole_num_to_binary!(&array, Int8Array, 1),
1123+
(Int16, Binary) => cast_whole_num_to_binary!(&array, Int16Array, 2),
1124+
(Int32, Binary) => cast_whole_num_to_binary!(&array, Int32Array, 4),
1125+
(Int64, Binary) => cast_whole_num_to_binary!(&array, Int64Array, 8),
10991126
_ if cast_options.is_adapting_schema
11001127
|| is_datafusion_spark_compatible(from_type, to_type) =>
11011128
{

‎spark/src/main/scala/org/apache/comet/expressions/CometCast.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ package org.apache.comet.expressions
2121

2222
import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal}
2323
import org.apache.spark.sql.internal.SQLConf
24-
import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, NullType, StructType}
24+
import org.apache.spark.sql.types.{ArrayType, BinaryType, DataType, DataTypes, DecimalType, NullType, StructType}
2525

2626
import org.apache.comet.CometConf
2727
import org.apache.comet.CometSparkSessionExtensions.withInfo
@@ -126,6 +126,7 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
126126
isSupported(dt.elementType, DataTypes.StringType, timeZoneId, evalMode)
127127
case (dt: ArrayType, dt1: ArrayType) =>
128128
isSupported(dt.elementType, dt1.elementType, timeZoneId, evalMode)
129+
case (from: DataType, _: BinaryType) => canCastToBinary(from)
129130
case (dt: DataType, _) if dt.typeName == "timestamp_ntz" =>
130131
// https://github.com/apache/datafusion-comet/issues/378
131132
toType match {
@@ -344,6 +345,12 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
344345
case _ => Unsupported(Some(s"Cast from DecimalType to $toType is not supported"))
345346
}
346347

348+
private def canCastToBinary(fromType: DataType): SupportLevel = fromType match {
349+
case DataTypes.ByteType | DataTypes.ShortType | DataTypes.IntegerType | DataTypes.LongType =>
350+
Compatible()
351+
case _ => Unsupported(Some(s"Cast from BinaryType to $fromType is not supported"))
352+
}
353+
347354
private def unsupported(fromType: DataType, toType: DataType): Unsupported = {
348355
Unsupported(Some(s"Cast from $fromType to $toType is not supported"))
349356
}

‎spark/src/test/scala/org/apache/comet/CometCastSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
281281
hasIncompatibleType = usingParquetExecWithIncompatTypes)
282282
}
283283

284-
ignore("cast ShortType to BinaryType") {
284+
test("cast ShortType to BinaryType") {
285285
castTest(
286286
generateShorts(),
287287
DataTypes.BinaryType,

0 commit comments

Comments
 (0)