Skip to content

Commit 8790a2d

Browse files
committed
use reserved __spark_autocdc* prefix
1 parent 76d775d commit 8790a2d

2 files changed

Lines changed: 6 additions & 34 deletions

File tree

sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/OutOfOrderCdcMergeUtils.scala

Lines changed: 0 additions & 33 deletions
This file was deleted.

sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ case class Scd1BatchProcessor(changeArgs: ChangeArgs) {
4444
def deduplicateMicrobatch(validatedMicrobatch: DataFrame): DataFrame = {
4545
// The `max_by` API can only return a single column, so pack/unpack the entire row into a
4646
// temporary column before and after the `max_by` operation.
47-
val winningRowCol = OutOfOrderCdcMergeUtils.tempColName("__winning_row")
47+
val winningRowCol = Scd1BatchProcessor.winningRowColName
4848

4949
val allMicrobatchColumns =
5050
validatedMicrobatch.columns
@@ -60,3 +60,8 @@ case class Scd1BatchProcessor(changeArgs: ChangeArgs) {
6060
.select(F.col(s"$winningRowCol.*"))
6161
}
6262
}
63+
64+
object Scd1BatchProcessor {
65+
// Columns prefixed with `__spark_autocdc_` are reserved for internal SDP AutoCDC processing.
66+
private val winningRowColName = "__spark_autocdc_winning_row"
67+
}

0 commit comments

Comments
 (0)