Skip to content

Commit c9b27fb

Browse files
committed
test(spark): add regression test for datediff on dictionary-encoded timestamps
1 parent 08f67c0 commit c9b27fb

1 file changed

Lines changed: 21 additions & 0 deletions

File tree

spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,27 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
257257
}
258258
}
259259

260+
test("datediff works with dictionary-encoded timestamp columns") {
261+
withTempDir { path =>
262+
withSQLConf(
263+
CometConf.COMET_NATIVE_SCAN_IMPL.key -> CometConf.SCAN_NATIVE_COMET,
264+
"spark.sql.parquet.enableDictionary" -> "true") {
265+
val df = spark
266+
.createDataFrame(
267+
Seq(
268+
("a", java.sql.Timestamp.valueOf("2024-01-02 10:00:00")),
269+
("b", java.sql.Timestamp.valueOf("2024-01-03 11:00:00"))))
270+
.toDF("id", "ts")
271+
272+
df.write.mode(SaveMode.Overwrite).parquet(path.toString)
273+
spark.read.parquet(path.toString).createOrReplaceTempView("ts_tbl")
274+
275+
checkSparkAnswerAndOperator(
276+
"SELECT id, datediff(DATE('2024-01-10'), ts) AS diff FROM ts_tbl ORDER BY id")
277+
}
278+
}
279+
}
280+
260281
test("date_format with timestamp column") {
261282
// Filter out formats with embedded quotes that need special handling
262283
val supportedFormats = CometDateFormat.supportedFormats.keys.toSeq

0 commit comments

Comments
 (0)