Skip to content

Commit 84c6c8d

Browse files
committed
Revert "[spark] Fire PaimonMergeInto for data-evolution tables on Spark 4.1 useV2Write=true"
This reverts commit 54591a9.
1 parent 54591a9 commit 84c6c8d

1 file changed

Lines changed: 5 additions & 16 deletions

File tree

paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -48,26 +48,15 @@ trait PaimonMergeIntoBase
4848
// `AnalysisHelper.allowInvokingTransformsInAnalyzer` so the in-analyzer assertion does not
4949
// trip.
5050
val skipV2RowLevelTables = org.apache.spark.SPARK_VERSION >= "4.1"
51-
52-
// Whether to let Spark's built-in RewriteMergeIntoTable take the plan. On Spark 4.1 we skip our
53-
// rewrite when the table already exposes SupportsRowLevelOperations AND data-evolution is
54-
// disabled. Data-evolution tables (e.g. Blob tables with 'blob-field' + 'data-evolution.enabled')
55-
// cause Spark 4.1's RewriteMergeIntoTable to silently skip (its `rewritable` / `aligned` /
56-
// `needSchemaEvolution` gate fails on such tables), leaving the plan as a MergeIntoTable that
57-
// the physical planner rejects with UNSUPPORTED_FEATURE.TABLE_OPERATION. Paimon's rule knows
58-
// how to route those tables through MergeIntoPaimonDataEvolutionTable, so it must still fire.
59-
def spark41V2OwnsPlan(target: LogicalPlan): Boolean = {
60-
if (!skipV2RowLevelTables) return false
61-
val v2Table = PaimonRelation.getPaimonRelation(target).table
62-
v2Table.isInstanceOf[SparkTableWithRowLevelOps] &&
63-
!v2Table.asInstanceOf[SparkTable].coreOptions.dataEvolutionEnabled()
64-
}
65-
6651
AnalysisHelper.allowInvokingTransformsInAnalyzer {
6752
plan.transformDown {
6853
case merge: MergeIntoTable
6954
if merge.resolved && PaimonRelation.isPaimonTable(merge.targetTable) &&
70-
!spark41V2OwnsPlan(merge.targetTable) =>
55+
(!skipV2RowLevelTables ||
56+
!PaimonRelation
57+
.getPaimonRelation(merge.targetTable)
58+
.table
59+
.isInstanceOf[SparkTableWithRowLevelOps]) =>
7160
val relation = PaimonRelation.getPaimonRelation(merge.targetTable)
7261
val v2Table = relation.table.asInstanceOf[SparkTable]
7362
val dataEvolutionEnabled = v2Table.coreOptions.dataEvolutionEnabled()

0 commit comments

Comments
 (0)