Skip to content

Commit cd36073

Browse files
committed
fix: check Comet ignore tags before DisableAdaptiveExecution in 3.4.3 diff
The test override in SQLTestUtils checked DisableAdaptiveExecution first, so tests with both DisableAdaptiveExecution and IgnoreCometNativeDataFusion tags (like "static scan metrics") would enter the DAE branch and never check the ignore tag. Reorder to match the 3.5.8 diff: check Comet skip tags first with early returns, then handle DisableAdaptiveExecution.
1 parent 60368c7 commit cd36073

1 file changed

Lines changed: 33 additions & 37 deletions

File tree

dev/diffs/3.4.3.diff

Lines changed: 33 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2948,7 +2948,7 @@ index abe606ad9c1..2d930b64cca 100644
29482948
val tblTargetName = "tbl_target"
29492949
val tblSourceQualified = s"default.$tblSourceName"
29502950
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
2951-
index dd55fcfe42c..a1d390c93d0 100644
2951+
index dd55fcfe42c..99bc018008a 100644
29522952
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
29532953
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
29542954
@@ -27,6 +27,7 @@ import scala.concurrent.duration._
@@ -2967,46 +2967,42 @@ index dd55fcfe42c..a1d390c93d0 100644
29672967
import org.apache.spark.sql.execution.FilterExec
29682968
import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
29692969
import org.apache.spark.sql.execution.datasources.DataSourceUtils
2970-
@@ -118,7 +120,7 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
2971-
}
2970+
@@ -119,6 +121,34 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
29722971

29732972
override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
2974-
- (implicit pos: Position): Unit = {
2975-
+ (implicit pos: Position): Unit = {
2973+
(implicit pos: Position): Unit = {
2974+
+ // Check Comet skip tags first, before DisableAdaptiveExecution handling
2975+
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
2976+
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
2977+
+ return
2978+
+ }
2979+
+ if (isCometEnabled) {
2980+
+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
2981+
+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
2982+
+ cometScanImpl == CometConf.SCAN_AUTO
2983+
+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
2984+
+ cometScanImpl == CometConf.SCAN_AUTO
2985+
+ if (isNativeIcebergCompat &&
2986+
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
2987+
+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
2988+
+ return
2989+
+ }
2990+
+ if (isNativeDataFusion &&
2991+
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
2992+
+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
2993+
+ return
2994+
+ }
2995+
+ if ((isNativeDataFusion || isNativeIcebergCompat) &&
2996+
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
2997+
+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
2998+
+ testTags: _*)(testFun)
2999+
+ return
3000+
+ }
3001+
+ }
29763002
if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
29773003
super.test(testName, testTags: _*) {
29783004
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
2979-
@@ -126,7 +128,28 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
2980-
}
2981-
}
2982-
} else {
2983-
- super.test(testName, testTags: _*)(testFun)
2984-
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
2985-
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
2986-
+ } else {
2987-
+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
2988-
+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
2989-
+ cometScanImpl == CometConf.SCAN_AUTO
2990-
+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
2991-
+ cometScanImpl == CometConf.SCAN_AUTO
2992-
+ if (isCometEnabled && isNativeIcebergCompat &&
2993-
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
2994-
+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
2995-
+ } else if (isCometEnabled && isNativeDataFusion &&
2996-
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
2997-
+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
2998-
+ } else if (isCometEnabled && (isNativeDataFusion || isNativeIcebergCompat) &&
2999-
+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
3000-
+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
3001-
+ testTags: _*)(testFun)
3002-
+ } else {
3003-
+ super.test(testName, testTags: _*)(testFun)
3004-
+ }
3005-
+ }
3006-
}
3007-
}
3008-
3009-
@@ -242,6 +265,29 @@ private[sql] trait SQLTestUtilsBase
3005+
@@ -242,6 +272,29 @@ private[sql] trait SQLTestUtilsBase
30103006
protected override def _sqlContext: SQLContext = self.spark.sqlContext
30113007
}
30123008

@@ -3036,7 +3032,7 @@ index dd55fcfe42c..a1d390c93d0 100644
30363032
protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
30373033
SparkSession.setActiveSession(spark)
30383034
super.withSQLConf(pairs: _*)(f)
3039-
@@ -434,6 +480,8 @@ private[sql] trait SQLTestUtilsBase
3035+
@@ -434,6 +487,8 @@ private[sql] trait SQLTestUtilsBase
30403036
val schema = df.schema
30413037
val withoutFilters = df.queryExecution.executedPlan.transform {
30423038
case FilterExec(_, child) => child

0 commit comments

Comments
 (0)