Skip to content

Commit a6007db

Browse files
author
B Vadlamani
committed
int_to_binary
1 parent 6f8e1c6 commit a6007db

4 files changed

Lines changed: 116 additions & 169 deletions

File tree

‎docs/source/user-guide/latest/compatibility.md‎

Lines changed: 0 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -88,89 +88,6 @@ The following cast operations are generally compatible with Spark except for the
8888
<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
8989

9090
<!--BEGIN:COMPAT_CAST_TABLE-->
91-
<!-- prettier-ignore-start -->
92-
| From Type | To Type | Notes |
93-
|-|-|-|
94-
| boolean | byte | |
95-
| boolean | short | |
96-
| boolean | integer | |
97-
| boolean | long | |
98-
| boolean | float | |
99-
| boolean | double | |
100-
| boolean | string | |
101-
| byte | boolean | |
102-
| byte | short | |
103-
| byte | integer | |
104-
| byte | long | |
105-
| byte | float | |
106-
| byte | double | |
107-
| byte | decimal | |
108-
| byte | string | |
109-
| byte | binary | |
110-
| short | boolean | |
111-
| short | byte | |
112-
| short | integer | |
113-
| short | long | |
114-
| short | float | |
115-
| short | double | |
116-
| short | decimal | |
117-
| short | string | |
118-
| short | binary | |
119-
| integer | boolean | |
120-
| integer | byte | |
121-
| integer | short | |
122-
| integer | long | |
123-
| integer | float | |
124-
| integer | double | |
125-
| integer | decimal | |
126-
| integer | string | |
127-
| integer | binary | |
128-
| long | boolean | |
129-
| long | byte | |
130-
| long | short | |
131-
| long | integer | |
132-
| long | float | |
133-
| long | double | |
134-
| long | decimal | |
135-
| long | string | |
136-
| long | binary | |
137-
| float | boolean | |
138-
| float | byte | |
139-
| float | short | |
140-
| float | integer | |
141-
| float | long | |
142-
| float | double | |
143-
| float | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
144-
| double | boolean | |
145-
| double | byte | |
146-
| double | short | |
147-
| double | integer | |
148-
| double | long | |
149-
| double | float | |
150-
| double | string | There can be differences in precision. For example, the input "1.4E-45" will produce 1.0E-45 instead of 1.4E-45 |
151-
| decimal | boolean | |
152-
| decimal | byte | |
153-
| decimal | short | |
154-
| decimal | integer | |
155-
| decimal | long | |
156-
| decimal | float | |
157-
| decimal | double | |
158-
| decimal | decimal | |
159-
| decimal | string | There can be formatting differences in some case due to Spark using scientific notation where Comet does not |
160-
| string | boolean | |
161-
| string | byte | |
162-
| string | short | |
163-
| string | integer | |
164-
| string | long | |
165-
| string | float | |
166-
| string | double | |
167-
| string | date | Only supports years between 262143 BC and 262142 AD |
168-
| binary | string | |
169-
| date | string | |
170-
| timestamp | long | |
171-
| timestamp | string | |
172-
| timestamp | date | |
173-
<!-- prettier-ignore-end -->
17491
<!--END:COMPAT_CAST_TABLE-->
17592

17693
### Incompatible Casts
@@ -180,15 +97,6 @@ The following cast operations are not compatible with Spark for all inputs and a
18097
<!-- WARNING! DO NOT MANUALLY MODIFY CONTENT BETWEEN THE BEGIN AND END TAGS -->
18198

18299
<!--BEGIN:INCOMPAT_CAST_TABLE-->
183-
<!-- prettier-ignore-start -->
184-
| From Type | To Type | Notes |
185-
|-|-|-|
186-
| float | decimal | There can be rounding differences |
187-
| double | decimal | There can be rounding differences |
188-
| string | decimal | Does not support fullwidth unicode digits (e.g \\uFF10)
189-
or strings containing null bytes (e.g \\u0000) |
190-
| string | timestamp | Not all valid formats are supported |
191-
<!-- prettier-ignore-end -->
192100
<!--END:INCOMPAT_CAST_TABLE-->
193101

194102
### Unsupported Casts

‎native/spark-expr/src/conversion_funcs/cast.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ use std::{
6666
num::Wrapping,
6767
sync::Arc,
6868
};
69+
use crate::EvalMode::Legacy;
6970

7071
static TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f");
7172

@@ -1119,10 +1120,10 @@ fn cast_array(
11191120
Ok(cast_with_options(&array, to_type, &CAST_OPTIONS)?)
11201121
}
11211122
(Binary, Utf8) => Ok(cast_binary_to_string::<i32>(&array, cast_options)?),
1122-
(Int8, Binary) => cast_whole_num_to_binary!(&array, Int8Array, 1),
1123-
(Int16, Binary) => cast_whole_num_to_binary!(&array, Int16Array, 2),
1124-
(Int32, Binary) => cast_whole_num_to_binary!(&array, Int32Array, 4),
1125-
(Int64, Binary) => cast_whole_num_to_binary!(&array, Int64Array, 8),
1123+
(Int8, Binary) if (eval_mode == Legacy) => cast_whole_num_to_binary!(&array, Int8Array, 1),
1124+
(Int16, Binary) if (eval_mode == Legacy) => cast_whole_num_to_binary!(&array, Int16Array, 2),
1125+
(Int32, Binary) if (eval_mode == Legacy) => cast_whole_num_to_binary!(&array, Int32Array, 4),
1126+
(Int64, Binary) if (eval_mode == Legacy) => cast_whole_num_to_binary!(&array, Int64Array, 8),
11261127
_ if cast_options.is_adapting_schema
11271128
|| is_datafusion_spark_compatible(from_type, to_type) =>
11281129
{

‎spark/src/main/scala/org/apache/comet/expressions/CometCast.scala‎

Lines changed: 80 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
126126
isSupported(dt.elementType, DataTypes.StringType, timeZoneId, evalMode)
127127
case (dt: ArrayType, dt1: ArrayType) =>
128128
isSupported(dt.elementType, dt1.elementType, timeZoneId, evalMode)
129-
case (from: DataType, _: BinaryType) => canCastToBinary(from)
130129
case (dt: DataType, _) if dt.typeName == "timestamp_ntz" =>
131130
// https://github.com/apache/datafusion-comet/issues/378
132131
toType match {
@@ -148,13 +147,13 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
148147
case (DataTypes.BooleanType, _) =>
149148
canCastFromBoolean(toType)
150149
case (DataTypes.ByteType, _) =>
151-
canCastFromByte(toType)
150+
canCastFromByte(toType, evalMode)
152151
case (DataTypes.ShortType, _) =>
153-
canCastFromShort(toType)
152+
canCastFromShort(toType, evalMode)
154153
case (DataTypes.IntegerType, _) =>
155-
canCastFromInt(toType)
154+
canCastFromInt(toType, evalMode)
156155
case (DataTypes.LongType, _) =>
157-
canCastFromLong(toType)
156+
canCastFromLong(toType, evalMode)
158157
case (DataTypes.FloatType, _) =>
159158
canCastFromFloat(toType)
160159
case (DataTypes.DoubleType, _) =>
@@ -269,53 +268,85 @@ object CometCast extends CometExpressionSerde[Cast] with CometExprShim {
269268
case _ => unsupported(DataTypes.BooleanType, toType)
270269
}
271270

272-
private def canCastFromByte(toType: DataType): SupportLevel = toType match {
273-
case DataTypes.BooleanType =>
274-
Compatible()
275-
case DataTypes.ShortType | DataTypes.IntegerType | DataTypes.LongType =>
276-
Compatible()
277-
case DataTypes.FloatType | DataTypes.DoubleType | _: DecimalType =>
278-
Compatible()
279-
case _ =>
280-
unsupported(DataTypes.ByteType, toType)
281-
}
271+
private def canCastFromByte(toType: DataType, evalMode: CometEvalMode.Value): SupportLevel =
272+
toType match {
273+
case DataTypes.BooleanType =>
274+
Compatible()
275+
case DataTypes.ShortType | DataTypes.IntegerType | DataTypes.LongType =>
276+
Compatible()
277+
case DataTypes.FloatType | DataTypes.DoubleType | _: DecimalType =>
278+
Compatible()
279+
case DataTypes.BinaryType =>
280+
if (evalMode == CometEvalMode.LEGACY) {
281+
Compatible()
282+
} else {
283+
Unsupported(
284+
Some(s"Spark does not support byte to binary conversion in ${evalMode} eval mode"))
285+
}
286+
case _ =>
287+
unsupported(DataTypes.ByteType, toType)
288+
}
282289

283-
private def canCastFromShort(toType: DataType): SupportLevel = toType match {
284-
case DataTypes.BooleanType =>
285-
Compatible()
286-
case DataTypes.ByteType | DataTypes.IntegerType | DataTypes.LongType =>
287-
Compatible()
288-
case DataTypes.FloatType | DataTypes.DoubleType | _: DecimalType =>
289-
Compatible()
290-
case _ =>
291-
unsupported(DataTypes.ShortType, toType)
292-
}
290+
private def canCastFromShort(toType: DataType, evalMode: CometEvalMode.Value): SupportLevel =
291+
toType match {
292+
case DataTypes.BooleanType =>
293+
Compatible()
294+
case DataTypes.ByteType | DataTypes.IntegerType | DataTypes.LongType =>
295+
Compatible()
296+
case DataTypes.FloatType | DataTypes.DoubleType | _: DecimalType =>
297+
Compatible()
298+
case DataTypes.BinaryType =>
299+
if (evalMode == CometEvalMode.LEGACY) {
300+
Compatible()
301+
} else {
302+
Unsupported(
303+
Some(s"Spark does not support short to binary conversion in ${evalMode} eval mode"))
304+
}
305+
case _ =>
306+
unsupported(DataTypes.ShortType, toType)
307+
}
293308

294-
private def canCastFromInt(toType: DataType): SupportLevel = toType match {
295-
case DataTypes.BooleanType =>
296-
Compatible()
297-
case DataTypes.ByteType | DataTypes.ShortType | DataTypes.LongType =>
298-
Compatible()
299-
case DataTypes.FloatType | DataTypes.DoubleType =>
300-
Compatible()
301-
case _: DecimalType =>
302-
Compatible()
303-
case _ =>
304-
unsupported(DataTypes.IntegerType, toType)
305-
}
309+
private def canCastFromInt(toType: DataType, evalMode: CometEvalMode.Value): SupportLevel =
310+
toType match {
311+
case DataTypes.BooleanType =>
312+
Compatible()
313+
case DataTypes.ByteType | DataTypes.ShortType | DataTypes.LongType =>
314+
Compatible()
315+
case DataTypes.FloatType | DataTypes.DoubleType =>
316+
Compatible()
317+
case _: DecimalType =>
318+
Compatible()
319+
case DataTypes.BinaryType =>
320+
if (evalMode == CometEvalMode.LEGACY) {
321+
Compatible()
322+
} else {
323+
Unsupported(
324+
Some(s"Spark does not support int to binary conversion in ${evalMode} eval mode"))
325+
}
326+
case _ =>
327+
unsupported(DataTypes.IntegerType, toType)
328+
}
306329

307-
private def canCastFromLong(toType: DataType): SupportLevel = toType match {
308-
case DataTypes.BooleanType =>
309-
Compatible()
310-
case DataTypes.ByteType | DataTypes.ShortType | DataTypes.IntegerType =>
311-
Compatible()
312-
case DataTypes.FloatType | DataTypes.DoubleType =>
313-
Compatible()
314-
case _: DecimalType =>
315-
Compatible()
316-
case _ =>
317-
unsupported(DataTypes.LongType, toType)
318-
}
330+
private def canCastFromLong(toType: DataType, evalMode: CometEvalMode.Value): SupportLevel =
331+
toType match {
332+
case DataTypes.BooleanType =>
333+
Compatible()
334+
case DataTypes.ByteType | DataTypes.ShortType | DataTypes.IntegerType =>
335+
Compatible()
336+
case DataTypes.FloatType | DataTypes.DoubleType =>
337+
Compatible()
338+
case _: DecimalType =>
339+
Compatible()
340+
case DataTypes.BinaryType =>
341+
if (evalMode == CometEvalMode.LEGACY) {
342+
Compatible()
343+
} else {
344+
Unsupported(
345+
Some(s"Spark does not support long to binary conversion in ${evalMode} eval mode"))
346+
}
347+
case _ =>
348+
unsupported(DataTypes.LongType, toType)
349+
}
319350

320351
private def canCastFromFloat(toType: DataType): SupportLevel = toType match {
321352
case DataTypes.BooleanType | DataTypes.DoubleType | DataTypes.ByteType | DataTypes.ShortType |

0 commit comments

Comments
 (0)