Merge remote-tracking branch 'apache/main' into cast-array-to-string

Author: cfmcgrady

.github/actions/rust-test/action.yaml

@@ -61,8 +61,14 @@ runs:
         cd common
         ../mvnw -B clean compile -DskipTests
 
+    - name: Install nextest
+      shell: bash
+      run: |
+        cargo install cargo-nextest --locked
+
     - name: Run Cargo test
       shell: bash
       run: |
         cd native
-        RUST_BACKTRACE=1 cargo test
+        RUST_BACKTRACE=1 cargo nextest run
+

Makefile

@@ -18,7 +18,7 @@
 .PHONY: all core jvm test clean release-linux release bench
 
 define spark_jvm_17_extra_args
-$(shell ./mvnw help:evaluate -Dexpression=extraJavaTestArgs | grep -v '\[')
+$(shell ./mvnw help:evaluate -q -DforceStdout -Dexpression=extraJavaTestArgs)
 endef
 
 # Build optional Comet native features (like hdfs e.g)

common/src/main/spark-3.x/org/apache/comet/shims/CometTypeShim.scala

@@ -16,13 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.comet.shims
 
-import org.apache.spark.sql.types.DataType
+package org.apache.comet.shims
 
 import scala.annotation.nowarn
 
+import org.apache.spark.sql.types.DataType
+
 trait CometTypeShim {
-    @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility.
-    def isStringCollationType(dt: DataType): Boolean = false
-}
+  @nowarn // Spark 4 feature; stubbed to false in Spark 3.x for compatibility.
+  def isStringCollationType(dt: DataType): Boolean = false
+}

common/src/main/spark-4.0/org/apache/comet/shims/CometTypeShim.scala

@@ -16,11 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.apache.comet.shims
 
 import org.apache.spark.sql.internal.types.StringTypeWithCollation
 import org.apache.spark.sql.types.DataType
 
 trait CometTypeShim {
-    def isStringCollationType(dt: DataType): Boolean = dt.isInstanceOf[StringTypeWithCollation]
-}
+  def isStringCollationType(dt: DataType): Boolean = dt.isInstanceOf[StringTypeWithCollation]
+}

common/src/main/spark-4.0/org/apache/comet/shims/ShimBatchReader.scala

@@ -32,6 +32,5 @@ object ShimBatchReader {
       -1,
       Array.empty[String],
       0,
-      0
-    )
+      0)
 }

native/Cargo.toml

@@ -52,7 +52,7 @@ thiserror = "2"
 object_store = { version = "0.12.3", features = ["gcp", "azure", "aws", "http"] }
 url = "2.2"
 aws-config = "1.6.3"
-aws-credential-types = "1.2.6"
+aws-credential-types = "1.2.7"
 
 [profile.release]
 debug = true

native/core/Cargo.toml

@@ -76,7 +76,7 @@ datafusion-comet-objectstore-hdfs = { path = "../hdfs", optional = true, default
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2"] }
 object_store_opendal = {version = "0.54.0", optional = true}
 hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]}
-opendal = { version ="0.54.0", optional = true, features = ["services-hdfs"] }
+opendal = { version ="0.54.1", optional = true, features = ["services-hdfs"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = "0.18.0"

native/core/src/execution/planner.rs

@@ -287,8 +287,6 @@ impl PhysicalPlanner {
                 )
             }
             ExprStruct::IntegralDivide(expr) => {
-                // TODO respect eval mode
-                // https://github.com/apache/datafusion-comet/issues/533
                 let eval_mode = from_protobuf_eval_mode(expr.eval_mode)?;
                 self.create_binary_expr_with_options(
                     expr.left.as_ref().unwrap(),
@@ -987,11 +985,12 @@ impl PhysicalPlanner {
                 } else {
                     "decimal_div"
                 };
-                let fun_expr = create_comet_physical_fun(
+                let fun_expr = create_comet_physical_fun_with_eval_mode(
                     func_name,
                     data_type.clone(),
                     &self.session_ctx.state(),
                     None,
+                    eval_mode,
                 )?;
                 Ok(Arc::new(ScalarFunctionExpr::new(
                     func_name,

native/spark-expr/benches/decimal_div.rs

@@ -20,7 +20,7 @@ use arrow::compute::cast;
 use arrow::datatypes::DataType;
 use criterion::{criterion_group, criterion_main, Criterion};
 use datafusion::physical_plan::ColumnarValue;
-use datafusion_comet_spark_expr::{spark_decimal_div, spark_decimal_integral_div};
+use datafusion_comet_spark_expr::{spark_decimal_div, spark_decimal_integral_div, EvalMode};
 use std::hint::black_box;
 use std::sync::Arc;
 
@@ -48,6 +48,7 @@ fn criterion_benchmark(c: &mut Criterion) {
             black_box(spark_decimal_div(
                 black_box(&args),
                 black_box(&DataType::Decimal128(10, 4)),
+                EvalMode::Legacy,
             ))
         })
     });
@@ -57,6 +58,7 @@ fn criterion_benchmark(c: &mut Criterion) {
             black_box(spark_decimal_integral_div(
                 black_box(&args),
                 black_box(&DataType::Decimal128(10, 4)),
+                EvalMode::Legacy,
             ))
         })
     });

native/spark-expr/src/comet_scalar_funcs.rs

@@ -137,13 +137,14 @@ pub fn create_comet_physical_fun_with_eval_mode(
             make_comet_scalar_udf!("unhex", func, without data_type)
         }
         "decimal_div" => {
-            make_comet_scalar_udf!("decimal_div", spark_decimal_div, data_type)
+            make_comet_scalar_udf!("decimal_div", spark_decimal_div, data_type, eval_mode)
         }
         "decimal_integral_div" => {
             make_comet_scalar_udf!(
                 "decimal_integral_div",
                 spark_decimal_integral_div,
-                data_type
+                data_type,
+                eval_mode
             )
         }
         "checked_add" => {