@@ -18,6 +18,7 @@ package org.apache.gluten.execution
1818
1919import org .apache .spark .SparkConf
2020import org .apache .spark .sql .Row
21+ import org .apache .spark .sql .execution .SparkPlan
2122import org .apache .spark .sql .types ._
2223import org .apache .spark .util .SparkVersionUtil
2324
@@ -44,6 +45,15 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
4445 .set(" spark.sql.catalog.spark_catalog" , " org.apache.spark.sql.delta.catalog.DeltaCatalog" )
4546 }
4647
48+ private def hasNativeDeltaMorScan (plan : SparkPlan ): Boolean = {
49+ collect(plan) {
50+ case _ : DeltaScanTransformer => true
51+ case scan : BatchScanExecTransformer
52+ if scan.scan.getClass.getSimpleName == " ParquetScan" =>
53+ true
54+ }.nonEmpty
55+ }
56+
4757 // IdMapping is supported in Delta 2.2 (related to Spark3.3.1)
4858 test(" column mapping mode = id" ) {
4959 withTable(" delta_cm1" ) {
@@ -213,16 +223,20 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
213223 withSQLConf(" spark.databricks.delta.deletionVectors.useMetadataRowIndex" -> " true" ) {
214224 spark.sql(s " DELETE FROM delta.` $path` WHERE id IN ( ${values2.mkString(" , " )}) " )
215225 val df = spark.read.format(" delta" ).load(path)
226+ checkAnswer(df, df1)
216227 val executedPlan = df.queryExecution.executedPlan
217228 val planText = executedPlan.toString()
218229 if (SparkVersionUtil .gteSpark35) {
219- assert(executedPlan.collect { case _ : DeltaScanTransformer => true }.nonEmpty, planText)
230+ assert(hasNativeDeltaMorScan(executedPlan), planText)
231+ assert(! planText.contains(" fallback Delta DV DML row-index scan" ), planText)
232+ assert(
233+ ! planText.contains(" fallback Delta DV scan without metadata row index" ),
234+ planText)
220235 assert(! planText.contains(" __delta_internal_is_row_deleted" ))
221236 assert(! planText.contains(" __delta_internal_row_index" ))
222237 } else {
223- assert(executedPlan.collect { case _ : DeltaScanTransformer => true }.isEmpty , planText)
238+ assert(! hasNativeDeltaMorScan( executedPlan) , planText)
224239 }
225- checkAnswer(df, df1)
226240 }
227241 }
228242 }
0 commit comments