Skip to content

Commit 409dd85

Browse files
authored
[spark] Fix view query failure with VARCHAR/CHAR column types (#7585)
Fix querying Paimon View fails with "Cannot up cast from STRING to VARCHAR(n)" when the underlying table has VARCHAR or CHAR columns. Spark replaces CharType/VarcharType with StringType during V2 table resolution, but the view schema still preserves the original types. This causes UpCast(StringType → VarcharType(n)) to fail. The fix uses CharVarcharUtils.replaceCharVarcharWithStringInSchema to normalize the view schema before applying UpCast.
1 parent 5427a95 commit 409dd85

2 files changed

Lines changed: 56 additions & 1 deletion

File tree

paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.parser.ParseException
3030
import org.apache.spark.sql.catalyst.parser.extensions.{CurrentOrigin, Origin}
3131
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Project, SubqueryAlias}
3232
import org.apache.spark.sql.catalyst.rules.Rule
33+
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
3334
import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog}
3435

3536
case class PaimonViewResolver(spark: SparkSession)
@@ -66,7 +67,13 @@ case class PaimonViewResolver(spark: SparkSession)
6667
val earlyRules = Seq(CTESubstitution, SubstituteUnresolvedOrdinals)
6768
val rewritten = earlyRules.foldLeft(parsedPlan)((plan, rule) => rule.apply(plan))
6869

69-
val aliases = SparkTypeUtils.fromPaimonRowType(view.rowType()).fields.zipWithIndex.map {
70+
// Spark internally replaces CharType/VarcharType with StringType during V2 table resolution,
71+
// so the view's schema must also use StringType to avoid UpCast failures
72+
// (e.g., "Cannot up cast from STRING to VARCHAR(50)").
73+
val viewSchema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(
74+
SparkTypeUtils.fromPaimonRowType(view.rowType()))
75+
76+
val aliases = viewSchema.fields.zipWithIndex.map {
7077
case (expected, pos) =>
7178
val attr = GetColumnByOrdinal(pos, expected.dataType)
7279
Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =

paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,4 +242,52 @@ abstract class PaimonViewTestBase extends PaimonHiveTestBase {
242242
}
243243
}
244244
}
245+
246+
test("Paimon View: create view on table with VARCHAR columns") {
247+
Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
248+
catalogName =>
249+
sql(s"USE $catalogName")
250+
withDatabase("test_db") {
251+
sql("CREATE DATABASE test_db")
252+
sql("USE test_db")
253+
withTable("t") {
254+
withView("v1") {
255+
sql("CREATE TABLE t (id INT, name VARCHAR(50), country STRING) USING paimon")
256+
sql("INSERT INTO t VALUES (1, 'alice', 'beijing'), (2, 'bob', 'shanghai')")
257+
sql("CREATE VIEW v1 AS SELECT * FROM t")
258+
checkAnswer(
259+
sql("SELECT * FROM v1"),
260+
Seq(Row(1, "alice", "beijing"), Row(2, "bob", "shanghai")))
261+
}
262+
}
263+
}
264+
}
265+
}
266+
267+
test("Paimon View: create view on table with nested VARCHAR and CHAR columns") {
268+
Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
269+
catalogName =>
270+
sql(s"USE $catalogName")
271+
withDatabase("test_db") {
272+
sql("CREATE DATABASE test_db")
273+
sql("USE test_db")
274+
withTable("t") {
275+
withView("v1") {
276+
sql("""CREATE TABLE t (
277+
| id INT,
278+
| tags ARRAY<VARCHAR(100)>,
279+
| props MAP<STRING, VARCHAR(200)>
280+
|) USING paimon""".stripMargin)
281+
sql("""INSERT INTO t VALUES
282+
| (1, ARRAY('a', 'b'), MAP('k1', 'v1')),
283+
| (2, ARRAY('c'), MAP('k2', 'v2'))""".stripMargin)
284+
sql("CREATE VIEW v1 AS SELECT * FROM t")
285+
checkAnswer(
286+
sql("SELECT * FROM v1"),
287+
Seq(Row(1, Seq("a", "b"), Map("k1" -> "v1")), Row(2, Seq("c"), Map("k2" -> "v2"))))
288+
}
289+
}
290+
}
291+
}
292+
}
245293
}

0 commit comments

Comments
 (0)