Skip to content

Commit 03328cd

Browse files
committed
fix: accept native DataFusion error message for duplicate fields in 3.5.8 diff
The SPARK-25207 test expects a specific error message for duplicate fields in case-insensitive mode, but native DataFusion produces a different schema error. Update the test to accept either message format.
1 parent e630c71 commit 03328cd

1 file changed

Lines changed: 22 additions & 5 deletions

File tree

dev/diffs/3.5.8.diff

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1999,7 +1999,7 @@ index 07e2849ce6f..3e73645b638 100644
19991999
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
20002000
)
20012001
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
2002-
index 8e88049f51e..e21a5797996 100644
2002+
index 8e88049f51e..6150a556f9b 100644
20032003
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20042004
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20052005
@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2088,7 +2088,24 @@ index 8e88049f51e..e21a5797996 100644
20882088
val schema = StructType(Seq(
20892089
StructField("a", IntegerType, nullable = false)
20902090
))
2091-
@@ -1984,7 +2000,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2091+
@@ -1952,8 +1968,14 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2092+
val e = intercept[SparkException] {
2093+
sql(s"select a from $tableName where b > 0").collect()
2094+
}
2095+
- assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
2096+
- """Found duplicate field(s) "B": [B, b] in case-insensitive mode"""))
2097+
+ assert(e.getCause.isInstanceOf[RuntimeException])
2098+
+ val msg = e.getCause.getMessage
2099+
+ // native_datafusion produces a different error message for duplicate fields
2100+
+ assert(
2101+
+ msg.contains(
2102+
+ """Found duplicate field(s) "B": [B, b] in case-insensitive mode""") ||
2103+
+ msg.contains("Unable to get field named"),
2104+
+ s"Unexpected error message: $msg")
2105+
}
2106+
2107+
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
2108+
@@ -1984,7 +2006,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
20922109
}
20932110
}
20942111

@@ -2098,7 +2115,7 @@ index 8e88049f51e..e21a5797996 100644
20982115
// block 1:
20992116
// null count min max
21002117
// page-0 0 0 99
2101-
@@ -2044,7 +2061,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2118+
@@ -2044,7 +2067,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21022119
}
21032120
}
21042121

@@ -2108,7 +2125,7 @@ index 8e88049f51e..e21a5797996 100644
21082125
withTempPath { dir =>
21092126
val path = dir.getCanonicalPath
21102127
spark.range(100).selectExpr("id * 2 AS id")
2111-
@@ -2276,7 +2294,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
2128+
@@ -2276,7 +2300,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
21122129
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21132130
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21142131

@@ -2121,7 +2138,7 @@ index 8e88049f51e..e21a5797996 100644
21212138
} else {
21222139
assert(selectedFilters.isEmpty, "There is filter pushed down")
21232140
}
2124-
@@ -2336,7 +2358,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
2141+
@@ -2336,7 +2364,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
21252142
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21262143
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21272144

0 commit comments

Comments
 (0)