Skip to content

Commit 1740ff2

Browse files
committed
[VL][Delta] Broaden DV native scan assertion
1 parent c015f63 commit 1740ff2

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package org.apache.gluten.execution
1818

1919
import org.apache.spark.SparkConf
2020
import org.apache.spark.sql.Row
21+
import org.apache.spark.sql.execution.SparkPlan
2122
import org.apache.spark.sql.types._
2223
import org.apache.spark.util.SparkVersionUtil
2324

@@ -44,6 +45,15 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
4445
.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
4546
}
4647

48+
private def hasNativeDeltaMorScan(plan: SparkPlan): Boolean = {
49+
collect(plan) {
50+
case _: DeltaScanTransformer => true
51+
case scan: BatchScanExecTransformer
52+
if scan.scan.getClass.getSimpleName == "ParquetScan" =>
53+
true
54+
}.nonEmpty
55+
}
56+
4757
// IdMapping is supported in Delta 2.2 (related to Spark3.3.1)
4858
test("column mapping mode = id") {
4959
withTable("delta_cm1") {
@@ -213,16 +223,20 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
213223
withSQLConf("spark.databricks.delta.deletionVectors.useMetadataRowIndex" -> "true") {
214224
spark.sql(s"DELETE FROM delta.`$path` WHERE id IN (${values2.mkString(", ")})")
215225
val df = spark.read.format("delta").load(path)
226+
checkAnswer(df, df1)
216227
val executedPlan = df.queryExecution.executedPlan
217228
val planText = executedPlan.toString()
218229
if (SparkVersionUtil.gteSpark35) {
219-
assert(executedPlan.collect { case _: DeltaScanTransformer => true }.nonEmpty, planText)
230+
assert(hasNativeDeltaMorScan(executedPlan), planText)
231+
assert(!planText.contains("fallback Delta DV DML row-index scan"), planText)
232+
assert(
233+
!planText.contains("fallback Delta DV scan without metadata row index"),
234+
planText)
220235
assert(!planText.contains("__delta_internal_is_row_deleted"))
221236
assert(!planText.contains("__delta_internal_row_index"))
222237
} else {
223-
assert(executedPlan.collect { case _: DeltaScanTransformer => true }.isEmpty, planText)
238+
assert(!hasNativeDeltaMorScan(executedPlan), planText)
224239
}
225-
checkAnswer(df, df1)
226240
}
227241
}
228242
}

0 commit comments

Comments
 (0)