Skip to content

Commit f313556

Browse files
committed
[VL][Delta] Fix DML row-index guard Spark compatibility
1 parent 26dcd2d commit f313556

2 files changed

Lines changed: 6 additions & 4 deletions

File tree

gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaDeletionVectorDmlUtils.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@ import org.apache.spark.sql.delta.DeltaParquetFileFormat
2020
import org.apache.spark.sql.delta.files.TahoeFileIndex
2121
import org.apache.spark.sql.delta.stats.PreparedDeltaFileIndex
2222
import org.apache.spark.sql.execution.FileSourceScanExec
23-
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
2423

2524
object DeltaDeletionVectorDmlUtils {
25+
// Spark 3.5+ exposes this as ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME.
26+
private val parquetTemporaryRowIndexColumnName = "_tmp_metadata_row_index"
2627
private val deletionVectorRowIndexColumnNames =
2728
Set(
2829
"__delta_internal_row_index",
2930
DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME,
30-
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME)
31+
parquetTemporaryRowIndexColumnName)
3132
private val filePathColumnNames = Set("file_path", "filePath")
3233

3334
def isDeltaScan(scan: FileSourceScanExec): Boolean = {

gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.trees.TreeNodeTag
2828
import org.apache.spark.sql.delta.{DeltaColumnMapping, DeltaParquetFileFormat, NoMapping}
2929
import org.apache.spark.sql.execution.{FileSourceScanExec, FilterExec, ProjectExec, SparkPlan}
3030
import org.apache.spark.sql.execution.datasources.FileFormat
31-
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
3231
import org.apache.spark.sql.types.StructType
3332

3433
import scala.collection.mutable.ListBuffer
@@ -43,11 +42,13 @@ object DeltaPostTransformRules {
4342

4443
private val deletionVectorDeletedRowColumnName = "__delta_internal_is_row_deleted"
4544
private val deletionVectorRowIndexColumnName = "__delta_internal_row_index"
45+
// Spark 3.5+ exposes this as ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME.
46+
private val parquetTemporaryRowIndexColumnName = "_tmp_metadata_row_index"
4647
private val deletionVectorRowIndexColumnNames =
4748
Set(
4849
deletionVectorRowIndexColumnName,
4950
DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME,
50-
ParquetFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME)
51+
parquetTemporaryRowIndexColumnName)
5152
private val deletionVectorInternalColumnNames =
5253
Set(deletionVectorDeletedRowColumnName, deletionVectorRowIndexColumnName)
5354
private val deletionVectorPredicateColumnNames =

0 commit comments

Comments
 (0)