Skip to content

Commit b2ded2a

Browse files
andygroveclaude
andcommitted
Benchmark native_datafusion and native_iceberg_compat only
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent bd6eb62 commit b2ded2a

1 file changed

Lines changed: 15 additions & 5 deletions

File tree

spark/src/test/scala/org/apache/spark/sql/benchmark/CometPartitionColumnBenchmark.scala

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.spark.sql.benchmark
2222
import org.apache.spark.benchmark.Benchmark
2323

2424
import org.apache.comet.CometConf
25-
import org.apache.comet.CometConf.{SCAN_NATIVE_COMET, SCAN_NATIVE_ICEBERG_COMPAT}
25+
import org.apache.comet.CometConf.{SCAN_NATIVE_DATAFUSION, SCAN_NATIVE_ICEBERG_COMPAT}
2626

2727
/**
2828
* Benchmark to measure partition column scan performance. This exercises the CometConstantVector
@@ -63,15 +63,16 @@ object CometPartitionColumnBenchmark extends CometBenchmarkBase {
6363
spark.sql("select sum(id) from parquetV1Table").noop()
6464
}
6565

66-
sqlBenchmark.addCase("SQL Parquet - Comet (Scan Only)") { _ =>
66+
sqlBenchmark.addCase("SQL Parquet - Comet Native DataFusion") { _ =>
6767
withSQLConf(
6868
CometConf.COMET_ENABLED.key -> "true",
69-
CometConf.COMET_NATIVE_SCAN_IMPL.key -> SCAN_NATIVE_COMET) {
69+
CometConf.COMET_EXEC_ENABLED.key -> "true",
70+
CometConf.COMET_NATIVE_SCAN_IMPL.key -> SCAN_NATIVE_DATAFUSION) {
7071
spark.sql("select sum(id) from parquetV1Table").noop()
7172
}
7273
}
7374

74-
sqlBenchmark.addCase("SQL Parquet - Comet (Scan + Exec)") { _ =>
75+
sqlBenchmark.addCase("SQL Parquet - Comet Native Iceberg Compat") { _ =>
7576
withSQLConf(
7677
CometConf.COMET_ENABLED.key -> "true",
7778
CometConf.COMET_EXEC_ENABLED.key -> "true",
@@ -88,7 +89,16 @@ object CometPartitionColumnBenchmark extends CometBenchmarkBase {
8889
spark.sql(s"select $partSumExpr from parquetV1Table").noop()
8990
}
9091

91-
sqlBenchmark.addCase("SQL Parquet - Comet (read partition cols)") { _ =>
92+
sqlBenchmark.addCase("SQL Parquet - Comet Native DataFusion (partition cols)") { _ =>
93+
withSQLConf(
94+
CometConf.COMET_ENABLED.key -> "true",
95+
CometConf.COMET_EXEC_ENABLED.key -> "true",
96+
CometConf.COMET_NATIVE_SCAN_IMPL.key -> SCAN_NATIVE_DATAFUSION) {
97+
spark.sql(s"select $partSumExpr from parquetV1Table").noop()
98+
}
99+
}
100+
101+
sqlBenchmark.addCase("SQL Parquet - Comet Native Iceberg Compat (partition cols)") { _ =>
92102
withSQLConf(
93103
CometConf.COMET_ENABLED.key -> "true",
94104
CometConf.COMET_EXEC_ENABLED.key -> "true",

0 commit comments

Comments
 (0)