Skip to content

Commit 27c38aa

Browse files
committed
fix: restore IgnoreCometNativeDataFusion tags for 4 parquet tests in 3.5.8.diff
Regenerated diff from Spark v3.5.8 checkout to restore tags that were accidentally removed when the diff was edited directly.
1 parent 9929745 commit 27c38aa

1 file changed

Lines changed: 48 additions & 8 deletions

File tree

dev/diffs/3.5.8.diff

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2155,10 +2155,30 @@ index 8e88049f51e..b713ccddfcb 100644
21552155
case _ =>
21562156
throw new AnalysisException("Can not match ParquetTable in the query.")
21572157
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
2158-
index 8ed9ef1630e..eed2a6f5ad5 100644
2158+
index 8ed9ef1630e..a865928c1b2 100644
21592159
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
21602160
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
2161-
@@ -1345,7 +1345,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
2161+
@@ -1064,7 +1064,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
2162+
}
2163+
}
2164+
2165+
- test("SPARK-35640: read binary as timestamp should throw schema incompatible error") {
2166+
+ test("SPARK-35640: read binary as timestamp should throw schema incompatible error",
2167+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720")) {
2168+
val data = (1 to 4).map(i => Tuple1(i.toString))
2169+
val readSchema = StructType(Seq(StructField("_1", DataTypes.TimestampType)))
2170+
2171+
@@ -1075,7 +1076,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
2172+
}
2173+
}
2174+
2175+
- test("SPARK-35640: int as long should throw schema incompatible error") {
2176+
+ test("SPARK-35640: int as long should throw schema incompatible error",
2177+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720")) {
2178+
val data = (1 to 4).map(i => Tuple1(i))
2179+
val readSchema = StructType(Seq(StructField("_1", DataTypes.LongType)))
2180+
2181+
@@ -1345,7 +1347,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
21622182
}
21632183
}
21642184

@@ -2169,10 +2189,20 @@ index 8ed9ef1630e..eed2a6f5ad5 100644
21692189
checkAnswer(
21702190
// "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
21712191
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
2172-
index f6472ba3d9d..18295e0b0f0 100644
2192+
index f6472ba3d9d..7f00caf5063 100644
21732193
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
21742194
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
2175-
@@ -998,7 +998,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2195+
@@ -185,7 +185,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2196+
}
2197+
}
2198+
2199+
- test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") {
2200+
+ test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ",
2201+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720")) {
2202+
val data = (1 to 1000).map { i =>
2203+
val ts = new java.sql.Timestamp(i)
2204+
Row(ts)
2205+
@@ -998,7 +999,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
21762206
}
21772207
}
21782208

@@ -2182,7 +2212,7 @@ index f6472ba3d9d..18295e0b0f0 100644
21822212
withAllParquetReaders {
21832213
withTempPath { path =>
21842214
// Repeated values for dictionary encoding.
2185-
@@ -1051,7 +1052,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2215+
@@ -1051,7 +1053,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
21862216
testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96")
21872217
}
21882218

@@ -2192,7 +2222,7 @@ index f6472ba3d9d..18295e0b0f0 100644
21922222
def readParquet(schema: String, path: File): DataFrame = {
21932223
spark.read.schema(schema).parquet(path.toString)
21942224
}
2195-
@@ -1067,7 +1069,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2225+
@@ -1067,7 +1070,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
21962226
checkAnswer(readParquet(schema, path), df)
21972227
}
21982228

@@ -2202,7 +2232,7 @@ index f6472ba3d9d..18295e0b0f0 100644
22022232
val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
22032233
checkAnswer(readParquet(schema1, path), df)
22042234
val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
2205-
@@ -1089,7 +1092,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2235+
@@ -1089,7 +1093,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
22062236
val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d")
22072237
df.write.parquet(path.toString)
22082238

@@ -2212,7 +2242,17 @@ index f6472ba3d9d..18295e0b0f0 100644
22122242
checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
22132243
checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
22142244
checkAnswer(readParquet("b DECIMAL(11, 1)", path), sql("SELECT 123456.0"))
2215-
@@ -1148,7 +1152,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2245+
@@ -1133,7 +1138,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
2246+
}
2247+
}
2248+
2249+
- test("row group skipping doesn't overflow when reading into larger type") {
2250+
+ test("row group skipping doesn't overflow when reading into larger type",
2251+
+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720")) {
2252+
withTempPath { path =>
2253+
Seq(0).toDF("a").write.parquet(path.toString)
2254+
// The vectorized and non-vectorized readers will produce different exceptions, we don't need
2255+
@@ -1148,7 +1154,7 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
22162256
.where(s"a < ${Long.MaxValue}")
22172257
.collect()
22182258
}

0 commit comments

Comments
 (0)