fix ci

Author: cfmcgrady

spark/src/test/scala/org/apache/comet/CometCastSuite.scala

@@ -21,6 +21,7 @@ package org.apache.comet
 
 import java.io.File
 
+import scala.collection.mutable.ListBuffer
 import scala.util.Random
 import scala.util.matching.Regex
 
@@ -30,10 +31,11 @@ import org.apache.spark.sql.catalyst.expressions.Cast
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DataTypes, DecimalType, IntegerType, LongType, ShortType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DataTypes, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType, StructField, StructType}
 
 import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus
 import org.apache.comet.expressions.{CometCast, CometEvalMode}
+import org.apache.comet.rules.CometScanTypeChecker
 import org.apache.comet.serde.Compatible
 
 class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
@@ -1035,7 +1037,9 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
   }
 
   test("cast ArrayType to StringType") {
-    sql("set spark.comet.explainFallback.enabled=true")
+    val cometScanTypeChecker = CometScanTypeChecker(conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key))
+    val scanImpl = conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key)
+    val hasIncompatibleType = (dt: DataType) => cometScanTypeChecker.isTypeSupported(dt, scanImpl, ListBuffer.empty)
     Seq(
       BooleanType,
       StringType,
@@ -1046,9 +1050,10 @@ class CometCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
       //      FloatType,
       //      DoubleType,
       DecimalType(10, 2),
-      BinaryType).foreach { tpe =>
-      val input = generateArrays(100, tpe)
-      castTest(input, StringType)
+      DecimalType(38,18),
+      BinaryType).foreach { dt =>
+      val input = generateArrays(100, dt)
+      castTest(input, StringType, hasIncompatibleType(input.schema))
     }
   }
 

spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala

@@ -21,16 +21,21 @@ package org.apache.spark.sql
 
 import org.apache.comet.{CometConf, CometFuzzTestBase}
 import org.apache.comet.expressions.{CometCast, CometEvalMode}
+import org.apache.comet.rules.CometScanTypeChecker
 import org.apache.comet.serde.Compatible
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.expressions.{Alias, ToPrettyString}
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.types.DataTypes
 
+import scala.collection.mutable.ListBuffer
+
 class CometToPrettyStringSuite extends CometFuzzTestBase {
 
   test("ToPrettyString") {
+    val cometScanTypeChecker = CometScanTypeChecker(conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key))
+    val scanImpl = conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key)
     val df = spark.read.parquet(filename)
     df.createOrReplaceTempView("t1")
     val table = spark.sessionState.catalog.lookupRelation(TableIdentifier("t1"))
@@ -42,8 +47,7 @@ class CometToPrettyStringSuite extends CometFuzzTestBase {
       val analyzed = spark.sessionState.analyzer.execute(plan)
       val result: DataFrame = Dataset.ofRows(spark, analyzed)
       CometCast.isSupported(field.dataType, DataTypes.StringType, Some(spark.sessionState.conf.sessionLocalTimeZone), CometEvalMode.TRY) match {
-        // `native_comet` doesn't support complex type scan, see CometScanTypeChecker#isTypeSupported for more details
-        case _: Compatible if conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key) != CometConf.SCAN_NATIVE_COMET =>
+        case _: Compatible if cometScanTypeChecker.isTypeSupported(field.dataType, scanImpl, ListBuffer.empty) =>
           checkSparkAnswerAndOperator(result)
         case _ => checkSparkAnswer(result)
       }

spark/src/test/spark-4.0/org/apache/spark/sql/CometToPrettyStringSuite.scala

@@ -34,6 +34,8 @@ import org.apache.spark.sql.types.DataTypes
 class CometToPrettyStringSuite extends CometFuzzTestBase {
 
   test("ToPrettyString") {
+    val cometScanTypeChecker = CometScanTypeChecker(conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key))
+    val scanImpl = conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key)
     val style = List(
       BinaryOutputStyle.UTF8,
       BinaryOutputStyle.BASIC,
@@ -54,8 +56,7 @@ class CometToPrettyStringSuite extends CometFuzzTestBase {
           val analyzed = spark.sessionState.analyzer.execute(plan)
           val result: DataFrame = Dataset.ofRows(spark, analyzed)
           CometCast.isSupported(field.dataType, DataTypes.StringType, Some(spark.sessionState.conf.sessionLocalTimeZone), CometEvalMode.TRY) match {
-            // `native_comet` doesn't support complex type scan, see CometScanTypeChecker#isTypeSupported for more details
-            case _: Compatible if conf.getConfString(CometConf.COMET_NATIVE_SCAN_IMPL.key) != CometConf.SCAN_NATIVE_COMET =>
+            case _: Compatible if cometScanTypeChecker.isTypeSupported(field.dataType, scanImpl, ListBuffer.empty) =>
               checkSparkAnswerAndOperator(result)
             case _ => checkSparkAnswer(result)
           }