Skip to content

Commit 5fcc13d

Browse files
committed
fix comments
1 parent e06666b commit 5fcc13d

3 files changed

Lines changed: 73 additions & 4 deletions

File tree

paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonDataEvolutionTable.scala

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,8 +373,12 @@ case class MergeIntoPaimonDataEvolutionTable(
373373
// The final output is composed by updated columns, metadata columns and blob marker columns.
374374
// Marker columns are used to mark whether a blob field should be written with placeholder
375375
val rawBlobUpdateColumns = updateColumnsSorted.filter(isRawBlobUpdateColumn)
376-
val rawBlobMarkerNamesByColumn = rawBlobUpdateColumns.zipWithIndex.map {
377-
case (attr, index) => attr.name -> rawBlobMarkerName(index)
376+
val rawBlobMarkerNames =
377+
rawBlobMarkerNamesAvoiding(
378+
rawBlobUpdateColumns.size,
379+
updateColumnsSorted.map(_.name) ++ sourceTable.output.map(_.name))
380+
val rawBlobMarkerNamesByColumn = rawBlobUpdateColumns.zip(rawBlobMarkerNames).map {
381+
case (attr, markerName) => attr.name -> markerName
378382
}.toMap
379383
val rawBlobMarkerAttributes = rawBlobUpdateColumns.map(
380384
attr =>
@@ -832,6 +836,22 @@ object MergeIntoPaimonDataEvolutionTable {
832836
RAW_BLOB_PLACEHOLDER_MARKER_PREFIX + index
833837
}
834838

839+
private[commands] def rawBlobMarkerNamesAvoiding(
840+
count: Int,
841+
reservedNames: Seq[String]): Seq[String] = {
842+
var nextIndex = 0
843+
(0 until count).map {
844+
_ =>
845+
var markerName = rawBlobMarkerName(nextIndex)
846+
while (reservedNames.exists(reservedName => resolver(reservedName, markerName))) {
847+
nextIndex += 1
848+
markerName = rawBlobMarkerName(nextIndex)
849+
}
850+
nextIndex += 1
851+
markerName
852+
}
853+
}
854+
835855
private def quotedColumn(name: String) = {
836856
col("`" + name.replace("`", "``") + "`")
837857
}

paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonDataEvolutionTable.scala

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,12 @@ case class MergeIntoPaimonDataEvolutionTable(
372372
// The final output is composed by updated columns, metadata columns and blob marker columns.
373373
// Marker columns are used to mark whether a blob field should be written with placeholder
374374
val rawBlobUpdateColumns = updateColumnsSorted.filter(isRawBlobUpdateColumn)
375-
val rawBlobMarkerNamesByColumn = rawBlobUpdateColumns.zipWithIndex.map {
376-
case (attr, index) => attr.name -> rawBlobMarkerName(index)
375+
val rawBlobMarkerNames =
376+
rawBlobMarkerNamesAvoiding(
377+
rawBlobUpdateColumns.size,
378+
updateColumnsSorted.map(_.name) ++ sourceTable.output.map(_.name))
379+
val rawBlobMarkerNamesByColumn = rawBlobUpdateColumns.zip(rawBlobMarkerNames).map {
380+
case (attr, markerName) => attr.name -> markerName
377381
}.toMap
378382
val rawBlobMarkerAttributes = rawBlobUpdateColumns.map(
379383
attr =>
@@ -831,6 +835,22 @@ object MergeIntoPaimonDataEvolutionTable {
831835
RAW_BLOB_PLACEHOLDER_MARKER_PREFIX + index
832836
}
833837

838+
private[commands] def rawBlobMarkerNamesAvoiding(
839+
count: Int,
840+
reservedNames: Seq[String]): Seq[String] = {
841+
var nextIndex = 0
842+
(0 until count).map {
843+
_ =>
844+
var markerName = rawBlobMarkerName(nextIndex)
845+
while (reservedNames.exists(reservedName => resolver(reservedName, markerName))) {
846+
nextIndex += 1
847+
markerName = rawBlobMarkerName(nextIndex)
848+
}
849+
nextIndex += 1
850+
markerName
851+
}
852+
}
853+
834854
private def quotedColumn(name: String) = {
835855
col("`" + name.replace("`", "``") + "`")
836856
}

paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/BlobUpdateTestBase.scala

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,35 @@ class BlobUpdateTestBase extends PaimonSparkTestBase {
9999
}
100100
}
101101

102+
test("Blob: merge-into raw-data BLOB marker name does not collide with target column") {
103+
withTable("s", "t") {
104+
sql(
105+
"CREATE TABLE t (id INT, `__paimon_raw_blob_placeholder_0` STRING, picture BINARY) " +
106+
"TBLPROPERTIES ('row-tracking.enabled'='true', 'data-evolution.enabled'='true', " +
107+
"'blob-field'='picture')")
108+
sql("INSERT INTO t VALUES (1, 'old_marker_name', X'01'), (2, 'kept', X'02')")
109+
110+
sql("CREATE TABLE s (id INT, `__paimon_raw_blob_placeholder_0` STRING, picture BINARY)")
111+
sql("INSERT INTO s VALUES (1, 'new_marker_name', X'4E4557')")
112+
113+
sql("""
114+
|MERGE INTO t
115+
|USING s
116+
|ON t.id = s.id
117+
|WHEN MATCHED THEN UPDATE SET
118+
| t.`__paimon_raw_blob_placeholder_0` = s.`__paimon_raw_blob_placeholder_0`,
119+
| t.picture = s.picture
120+
|""".stripMargin)
121+
122+
checkAnswer(
123+
sql("SELECT id, `__paimon_raw_blob_placeholder_0`, picture FROM t ORDER BY id"),
124+
Seq(
125+
Row(1, "new_marker_name", Array[Byte](78, 69, 87)),
126+
Row(2, "kept", Array[Byte](2)))
127+
)
128+
}
129+
}
130+
102131
test("Blob: self merge updates raw-data BLOB column") {
103132
withTable("t") {
104133
sql("CREATE TABLE t (id INT, name STRING, picture BINARY) TBLPROPERTIES " +

0 commit comments

Comments
 (0)