Skip to content

Commit 27469e0

Browse files
committed
fix: skip duplicate fields test for native DataFusion in 3.5.8 diff
Add IgnoreCometNativeDataFusion tag to SPARK-25207 test instead of trying to accept both error messages. A separate PR will fix the underlying issue.
1 parent 37d10e3 commit 27469e0

1 file changed

Lines changed: 15 additions & 5 deletions

File tree

dev/diffs/3.5.8.diff

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1999,7 +1999,7 @@ index 07e2849ce6f..3e73645b638 100644
19991999
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
20002000
)
20012001
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
2002-
index 8e88049f51e..e21a5797996 100644
2002+
index 8e88049f51e..49f2001dc6b 100644
20032003
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20042004
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
20052005
@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -2088,7 +2088,17 @@ index 8e88049f51e..e21a5797996 100644
20882088
val schema = StructType(Seq(
20892089
StructField("a", IntegerType, nullable = false)
20902090
))
2091-
@@ -1984,7 +2000,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2091+
@@ -1933,7 +1949,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2092+
}
2093+
}
2094+
2095+
- test("SPARK-25207: exception when duplicate fields in case-insensitive mode") {
2096+
+ test("SPARK-25207: exception when duplicate fields in case-insensitive mode",
2097+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3311")) {
2098+
withTempPath { dir =>
2099+
val count = 10
2100+
val tableName = "spark_25207"
2101+
@@ -1984,7 +2001,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
20922102
}
20932103
}
20942104

@@ -2098,7 +2108,7 @@ index 8e88049f51e..e21a5797996 100644
20982108
// block 1:
20992109
// null count min max
21002110
// page-0 0 0 99
2101-
@@ -2044,7 +2061,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
2111+
@@ -2044,7 +2062,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
21022112
}
21032113
}
21042114

@@ -2108,7 +2118,7 @@ index 8e88049f51e..e21a5797996 100644
21082118
withTempPath { dir =>
21092119
val path = dir.getCanonicalPath
21102120
spark.range(100).selectExpr("id * 2 AS id")
2111-
@@ -2276,7 +2294,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
2121+
@@ -2276,7 +2295,11 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
21122122
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21132123
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21142124

@@ -2121,7 +2131,7 @@ index 8e88049f51e..e21a5797996 100644
21212131
} else {
21222132
assert(selectedFilters.isEmpty, "There is filter pushed down")
21232133
}
2124-
@@ -2336,7 +2358,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
2134+
@@ -2336,7 +2359,11 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
21252135
assert(pushedParquetFilters.exists(_.getClass === filterClass),
21262136
s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
21272137

0 commit comments

Comments
 (0)