enable_spark_tests_comet_native_writer

Author: B Vadlamani

dev/diffs/4.0.1.diff

@@ -110,6 +110,20 @@ index 4410fe50912..43bcce2a038 100644
        case _ => Map[String, String]()
      }
      val childrenInfo = children.flatMap {
+diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
+index 524797015a2..09576b169bd 100644
+--- a/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
++++ b/sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar.sql.out
+@@ -738,3 +738,9 @@ SELECT to_varchar(12454.8, '99,999.9S')
+ -- !query analysis
+ Project [to_char(12454.8, 99,999.9S) AS to_char(12454.8, 99,999.9S)#x]
+ +- OneRowRelation
++
++
++-- !query
++SET spark.comet.parquet.write.enabled = true
++-- !query analysis
++SetCommand (spark.comet.parquet.write.enabled,Some(true))
 diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/listagg-collations.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/listagg-collations.sql.out
 index 7aca17dcb25..8afeb3b4a2f 100644
 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/listagg-collations.sql.out
@@ -130,6 +144,23 @@ index 7aca17dcb25..8afeb3b4a2f 100644
  -- !query
  WITH t(c1) AS (SELECT replace(listagg(DISTINCT col1 COLLATE unicode_rtrim) COLLATE utf8_binary, ' ', '') FROM (VALUES ('xbc  '), ('xbc '), ('a'), ('xbc'))) SELECT len(c1), regexp_count(c1, 'a'), regexp_count(c1, 'xbc') FROM t
  -- !query analysis
+diff --git a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
+index be038e1083c..b3de0ee2641 100644
+--- a/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
++++ b/sql/core/src/test/resources/sql-tests/inputs/charvarchar.sql
+@@ -1,3 +1,6 @@
++-- TODO: support empty table write / CTAS in native parquet writer
++--SET spark.comet.parquet.write.enabled = false
++
+ create table char_tbl(c char(5), v varchar(6)) using parquet;
+ desc formatted char_tbl;
+ desc formatted char_tbl c;
+@@ -124,3 +127,5 @@ select chr(167), chr(247), chr(215);
+ SELECT to_varchar(78.12, '$99.99');
+ SELECT to_varchar(111.11, '99.9');
+ SELECT to_varchar(12454.8, '99,999.9S');
++
++SET spark.comet.parquet.write.enabled = true;
 diff --git a/sql/core/src/test/resources/sql-tests/inputs/collations.sql b/sql/core/src/test/resources/sql-tests/inputs/collations.sql
 index 17815ed5dde..baad440b1ce 100644
 --- a/sql/core/src/test/resources/sql-tests/inputs/collations.sql
@@ -312,6 +343,22 @@ index 21a3ce1e122..f4762ab98f0 100644
  SET spark.sql.ansi.enabled = false;
 
  -- In COMPENSATION views get invalidated if the type can't cast
+diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
+index 3026e463947..bf3f20906f1 100644
+--- a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
++++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
+@@ -1254,3 +1254,11 @@ SELECT to_varchar(12454.8, '99,999.9S')
+ struct<to_char(12454.8, 99,999.9S):string>
+ -- !query output
+ 12,454.8+
++
++
++-- !query
++SET spark.comet.parquet.write.enabled = true
++-- !query schema
++struct<key:string,value:string>
++-- !query output
++spark.comet.parquet.write.enabled	true
 diff --git a/sql/core/src/test/resources/sql-tests/results/listagg-collations.sql.out b/sql/core/src/test/resources/sql-tests/results/listagg-collations.sql.out
 index 1f8c5822e7d..b7de4e28813 100644
 --- a/sql/core/src/test/resources/sql-tests/results/listagg-collations.sql.out
@@ -2506,6 +2553,64 @@ index 272be70f9fe..06957694002 100644
          }.isEmpty)
          assert(collect(initialExecutedPlan) {
            case i: InMemoryTableScanLike => i
+diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala
+index a5bb3058bed..de51b21f21a 100644
+--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala
++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CharVarcharDDLTestBase.scala
+@@ -18,7 +18,7 @@
+ package org.apache.spark.sql.execution.command
+ 
+ import org.apache.spark.SparkConf
+-import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
++import org.apache.spark.sql.{AnalysisException, IgnoreComet, QueryTest, Row}
+ import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+ import org.apache.spark.sql.connector.catalog.InMemoryPartitionTableCatalog
+ import org.apache.spark.sql.internal.SQLConf
+@@ -150,7 +150,9 @@ trait CharVarcharDDLTestBase extends QueryTest with SQLTestUtils {
+     }
+   }
+ 
+-  test("SPARK-33901: ctas should should not change table's schema") {
++//  TODO support CTAS in comet native parquet writer
++  test("SPARK-33901: ctas should should not change table's schema",
++    IgnoreComet("comet native writer does not support empty dir / table creation yet")) {
+     withTable("t1", "t2") {
+       sql(s"CREATE TABLE t1(i CHAR(5), c VARCHAR(4)) USING $format")
+       sql(s"CREATE TABLE t2 USING $format AS SELECT * FROM t1")
+@@ -167,7 +169,8 @@ trait CharVarcharDDLTestBase extends QueryTest with SQLTestUtils {
+     }
+   }
+ 
+-  test("SPARK-37160: CREATE TABLE AS SELECT with CHAR_AS_VARCHAR") {
++  test("SPARK-37160: CREATE TABLE AS SELECT with CHAR_AS_VARCHAR",
++    IgnoreComet("comet native writer does not support empty dir/table creation yet")) {
+     withTable("t1", "t2") {
+       sql(s"CREATE TABLE t1(col CHAR(5)) USING $format")
+       checkTableSchemaTypeStr("t1", Seq(Row("char(5)")))
+diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
+index 343b59a311e..05f3cab13f9 100644
+--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
++++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileFormatWriterSuite.scala
+@@ -17,7 +17,7 @@
+ 
+ package org.apache.spark.sql.execution.datasources
+ 
+-import org.apache.spark.sql.{QueryTest, Row}
++import org.apache.spark.sql.{IgnoreComet, QueryTest, Row}
+ import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
+ import org.apache.spark.sql.test.SharedSparkSession
+ 
+@@ -28,7 +28,9 @@ class FileFormatWriterSuite
+ 
+   import testImplicits._
+ 
+-  test("empty file should be skipped while write to file") {
++//  TODO :: comet native write does not write empty files / dirs
++  test("empty file should be skipped while write to file",
++    IgnoreComet("comet native writer does not create empty files / dir")) {
+     withTempPath { path =>
+       spark.range(100).repartition(10).where("id = 50").write.parquet(path.toString)
+       val partFiles = path.listFiles()
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
 index 0a0b23d1e60..5685926250f 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -3314,6 +3419,62 @@ index c5c56f081d8..197cd241f48 100644
        assert(bucketedScan.length == expectedNumBucketedScan)
      }
 
+diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+index baf99798965..e23d8223e05 100644
+--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
++++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+@@ -263,7 +263,8 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
+     }
+   }
+ 
+-  test("INSERT INTO TABLE - complex type but different names") {
++  test("INSERT INTO TABLE - complex type but different names",
++    IgnoreComet("comet native writer issue")) {
+     val tab1 = "tab1"
+     val tab2 = "tab2"
+     withTable(tab1, tab2) {
+@@ -1002,7 +1003,8 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
+     }
+   }
+ 
+-  test("SPARK-38336 INSERT INTO statements with tables with default columns: positive tests") {
++  test("SPARK-38336 INSERT INTO statements with tables with default columns: positive tests",
++    IgnoreComet("comet native writer insert overwrite bug")) {
+     // When the INSERT INTO statement provides fewer values than expected, NULL values are appended
+     // in their place.
+     withTable("t") {
+@@ -1517,7 +1519,8 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
+     }
+   }
+ 
+-  test("SPARK-38811 INSERT INTO on columns added with ALTER TABLE ADD COLUMNS: Positive tests") {
++  test("SPARK-38811 INSERT INTO on columns added with ALTER TABLE ADD COLUMNS: Positive tests",
++    IgnoreComet("bug with insert overwrite mode comet native writer")) {
+     // There is a complex expression in the default value.
+     val createTableBooleanCol = "create table t(i boolean) using parquet"
+     val createTableIntCol = "create table t(i int) using parquet"
+@@ -2331,7 +2334,9 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
+     }
+   }
+ 
+-  test("SPARK-43071: INSERT INTO from queries whose final operators are not projections") {
++//  TODO : fix overwrite mode issues comet native writer
++  test("SPARK-43071: INSERT INTO from queries whose final operators are not projections",
++    IgnoreComet("comet writer issues with insert overwrite")) {
+     def runTest(insert: String, expected: Seq[Row]): Unit = {
+       withTable("t1", "t2") {
+         sql("create table t1(i boolean, s bigint default 42) using parquet")
+@@ -2399,7 +2404,9 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
+     }
+   }
+ 
+-  test("SPARK-29174 Support LOCAL in INSERT OVERWRITE DIRECTORY to data source") {
++//  TODO : Insert overwrite not supported comet native writer
++  test("SPARK-29174 Support LOCAL in INSERT OVERWRITE DIRECTORY to data source",
++    IgnoreComet("comet native writer does not support insert overwrite")) {
+     withTempPath { dir =>
+       val path = dir.toURI.getPath
+       sql(s"""create table tab1 ( a int) using parquet location '$path'""")
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
 index 9742a004545..4e0417d730a 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
@@ -3494,6 +3655,33 @@ index 86c4e49f6f6..2e639e5f38d 100644
      val tblSourceName = "tbl_src"
      val tblTargetName = "tbl_target"
      val tblSourceQualified = s"default.$tblSourceName"
+diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+index 300807cf058..8dab4f9a2af 100644
+--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
++++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+@@ -620,7 +620,10 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
+     Option(dir).map(spark.read.format("org.apache.spark.sql.test").load)
+   }
+ 
+-  test("write path implements onTaskCommit API correctly") {
++
++//  TODO : fix test for native writer
++  test("write path implements onTaskCommit API correctly",
++    IgnoreComet("Comet native writer partial support")) {
+     withSQLConf(
+         SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key ->
+           classOf[MessageCapturingCommitProtocol].getCanonicalName) {
+@@ -1184,7 +1187,9 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
+     }
+   }
+ 
+-  test("Insert overwrite table command should output correct schema: basic") {
++//  TODO : support vairous write modes in comet native writer
++  test("Insert overwrite table command should output correct schema: basic",
++    IgnoreComet("Comet native writer does not supportinsert overwrite mode")) {
+     withTable("tbl", "tbl2") {
+       withView("view1") {
+         val df = spark.range(10).toDF("id")
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
 index f0f3f94b811..fd9744a58cf 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala