@@ -1998,9 +1998,9 @@ index 104b4e416cd..b8af360fa14 100644
19981998 // Note that, if record level filtering is enabled, it should be a single record.
19991999 // If no filter is pushed down to Parquet, it should be the total length of data.
20002000- assert(actual > 1 && actual < data.length)
2001- + // Only enable Comet test iff it's scan only, since with native execution
2001+ + // Skip when Comet is enabled since with native execution
20022002+ // `stripSparkFilter` can't remove the native filter
2003- + if (!isCometEnabled || isCometScanOnly ) {
2003+ + if (!isCometEnabled) {
20042004+ assert(actual > 1 && actual < data.length)
20052005+ }
20062006 }
@@ -2011,9 +2011,9 @@ index 104b4e416cd..b8af360fa14 100644
20112011 // Note that, if record level filtering is enabled, it should be a single record.
20122012 // If no filter is pushed down to Parquet, it should be the total length of data.
20132013- assert(actual > 1 && actual < data.length)
2014- + // Only enable Comet test iff it's scan only, since with native execution
2014+ + // Skip when Comet is enabled since with native execution
20152015+ // `stripSparkFilter` can't remove the native filter
2016- + if (!isCometEnabled || isCometScanOnly ) {
2016+ + if (!isCometEnabled) {
20172017+ assert(actual > 1 && actual < data.length)
20182018+ }
20192019 }
@@ -2926,32 +2926,14 @@ index dd55fcfe42c..99bc018008a 100644
29262926 if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
29272927 super.test(testName, testTags: _*) {
29282928 withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
2929- @@ -242,6 +272,29 @@ private[sql] trait SQLTestUtilsBase
2929+ @@ -242,6 +272,11 @@ private[sql] trait SQLTestUtilsBase
29302930 protected override def _sqlContext: SQLContext = self.spark.sqlContext
29312931 }
29322932
29332933+ /**
29342934+ * Whether Comet extension is enabled
29352935+ */
29362936+ protected def isCometEnabled: Boolean = SparkSession.isCometEnabled
2937- +
2938- + /**
2939- + * Whether to enable ansi mode This is only effective when
2940- + * [[isCometEnabled]] returns true.
2941- + */
2942- + protected def enableCometAnsiMode: Boolean = {
2943- + val v = System.getenv("ENABLE_COMET_ANSI_MODE")
2944- + v != null && v.toBoolean
2945- + }
2946- +
2947- + /**
2948- + * Whether Spark should only apply Comet scan optimization. This is only effective when
2949- + * [[isCometEnabled]] returns true.
2950- + */
2951- + protected def isCometScanOnly: Boolean = {
2952- + val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
2953- + v != null && v.toBoolean
2954- + }
29552937+
29562938 protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
29572939 SparkSession.setActiveSession(spark)
@@ -2969,7 +2951,7 @@ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSessio
29692951index ed2e309fa07..a5ea58146ad 100644
29702952--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
29712953+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
2972- @@ -74,6 +74,31 @@ trait SharedSparkSessionBase
2954+ @@ -74,6 +74,20 @@ trait SharedSparkSessionBase
29732955 // this rule may potentially block testing of other optimization rules such as
29742956 // ConstantPropagation etc.
29752957 .set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
@@ -2980,23 +2962,12 @@ index ed2e309fa07..a5ea58146ad 100644
29802962+ .set("spark.comet.enabled", "true")
29812963+ .set("spark.comet.parquet.respectFilterPushdown", "true")
29822964+
2983- + if (!isCometScanOnly) {
2984- + conf
2985- + .set("spark.comet.exec.enabled", "true")
2986- + .set("spark.shuffle.manager",
2987- + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
2988- + .set("spark.comet.exec.shuffle.enabled", "true")
2989- + .set("spark.comet.memoryOverhead", "10g")
2990- + } else {
2991- + conf
2992- + .set("spark.comet.exec.enabled", "false")
2993- + .set("spark.comet.exec.shuffle.enabled", "false")
2994- + }
2995- +
2996- + if (enableCometAnsiMode) {
2997- + conf
2998- + .set("spark.sql.ansi.enabled", "true")
2999- + }
2965+ + conf
2966+ + .set("spark.comet.exec.enabled", "true")
2967+ + .set("spark.shuffle.manager",
2968+ + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
2969+ + .set("spark.comet.exec.shuffle.enabled", "true")
2970+ + .set("spark.comet.memoryOverhead", "10g")
30002971+ }
30012972 conf.set(
30022973 StaticSQLConf.WAREHOUSE_PATH,
@@ -3093,7 +3064,7 @@ diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.sca
30933064index 07361cfdce9..97dab2a3506 100644
30943065--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
30953066+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
3096- @@ -55,25 +55,54 @@ object TestHive
3067+ @@ -55,25 +55,41 @@ object TestHive
30973068 new SparkContext(
30983069 System.getProperty("spark.sql.test.master", "local[1]"),
30993070 "TestSQLContext",
@@ -3140,24 +3111,11 @@ index 07361cfdce9..97dab2a3506 100644
31403111+ .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
31413112+ .set("spark.comet.enabled", "true")
31423113+
3143- + val v = System.getenv("ENABLE_COMET_SCAN_ONLY")
3144- + if (v == null || !v.toBoolean) {
3145- + conf
3146- + .set("spark.comet.exec.enabled", "true")
3147- + .set("spark.shuffle.manager",
3148- + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
3149- + .set("spark.comet.exec.shuffle.enabled", "true")
3150- + } else {
3151- + conf
3152- + .set("spark.comet.exec.enabled", "false")
3153- + .set("spark.comet.exec.shuffle.enabled", "false")
3154- + }
3155- +
3156- + val a = System.getenv("ENABLE_COMET_ANSI_MODE")
3157- + if (a != null && a.toBoolean) {
3158- + conf
3159- + .set("spark.sql.ansi.enabled", "true")
3160- + }
3114+ + conf
3115+ + .set("spark.comet.exec.enabled", "true")
3116+ + .set("spark.shuffle.manager",
3117+ + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
3118+ + .set("spark.comet.exec.shuffle.enabled", "true")
31613119+ }
31623120
31633121+ conf
0 commit comments