Skip to content

Commit fdc2514

Browse files
committed
Add regression test: native writer rejects non-Arrow CometExec children
1 parent 406bb2f commit fdc2514

1 file changed

Lines changed: 28 additions & 0 deletions

File tree

spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -712,4 +712,32 @@ class CometParquetWriterSuite extends CometTestBase {
712712
rows
713713
}
714714

715+
test("native writer rejects non-Arrow CometExec children (regression for #3524)") {
716+
withTempDir { dir =>
717+
withSQLConf(
718+
CometConf.COMET_EXEC_ENABLED.key -> "true",
719+
CometConf.COMET_NATIVE_PARQUET_WRITE_ENABLED.key -> "true",
720+
CometConf.COMET_OPERATOR_DATA_WRITING_COMMAND_ALLOW_INCOMPAT.key -> "true",
721+
CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
722+
723+
val out = new File(dir, "literal_write").getAbsolutePath
724+
val df = Seq((1, "a"), (2, "b")).toDF("id", "v")
725+
726+
val plan = captureWritePlan(p => df.write.parquet(p), out)
727+
728+
val hasNativeWrite = plan.exists {
729+
case _: CometNativeWriteExec => true
730+
case d: DataWritingCommandExec =>
731+
d.child.exists(_.isInstanceOf[CometNativeWriteExec])
732+
case _ => false
733+
}
734+
assert(
735+
!hasNativeWrite,
736+
s"CometNativeWriteExec must NOT wrap a CometLocalTableScanExec child:\n${plan.treeString}")
737+
738+
assert(spark.read.parquet(out).count() == 2)
739+
}
740+
}
741+
}
742+
715743
}

0 commit comments

Comments
 (0)