From 6f8e5ac3626bccd1d84ed631e039da96b958d0e6 Mon Sep 17 00:00:00 2001 From: Norio Date: Sun, 3 May 2026 00:00:00 +0000 Subject: [PATCH] [GLUTEN-11916][VL] Enable 7 Spark 4.1 TODO SQL query test files Enable 7 of 11 TODO SQL query test files in spark41 VeloxSQLQueryTestSettings: - cast.sql, describe.sql, literals.sql (+ nonansi imports): already passing - nonansi/st-functions.sql: already passing - type-coercion-edge-cases.sql: fix golden file for Spark 4.1 semi-structured field access behavior change (col2.field on MAP now returns field extraction instead of CAST_INVALID_INPUT error) Remaining 4 TODOs triaged to separate issues: - subquery/exists-subquery/exists-orderby-limit.sql: ANSI fallback + OFFSET - scripting/randomly_generated_scripts.sql: NPE in condition handling - st-functions.sql: geospatial types (new Spark 4.1, not in Velox) - variant-field-extractions.sql: Variant type (#11371) --- .../results/type-coercion-edge-cases.sql.out | 21 ++----------------- .../velox/VeloxSQLQueryTestSettings.scala | 14 ++++++------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/type-coercion-edge-cases.sql.out b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/type-coercion-edge-cases.sql.out index da0ceafa893..d889a891605 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/type-coercion-edge-cases.sql.out +++ b/gluten-ut/spark41/src/test/resources/backends-velox/sql-tests/results/type-coercion-edge-cases.sql.out @@ -18,26 +18,9 @@ NULL -- !query SELECT col2.field FROM t1 -- !query schema -struct<> +struct -- !query output -org.apache.spark.SparkNumberFormatException -{ - "errorClass" : "CAST_INVALID_INPUT", - "sqlState" : "22018", - "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "expression" : "'field'", - "sourceType" : "\"STRING\"", - "targetType" : "\"BIGINT\"" - }, - "queryContext" : [ { - "objectType" : "", - "objectName" : "", - "startIndex" : 8, - "stopIndex" : 17, - "fragment" : "col2.field" - } ] -} + -- !query diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala index ea58cc7a5cd..aba54f998fd 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxSQLQueryTestSettings.scala @@ -38,7 +38,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { val SUPPORTED_SQL_QUERY_LIST: Set[String] = Set( "ansi/conditional-functions.sql", "ansi/decimalArithmeticOperations.sql", - // TODO: fix on Spark-4.1 "cast.sql", + "cast.sql", "change-column.sql", "ceil-floor-with-scale-param.sql", "column-resolution-aggregate.sql", @@ -57,7 +57,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { // "datetime-formatting-invalid.sql", "datetime-special.sql", "decimalArithmeticOperations.sql", - // TODO: fix on Spark-4.1 "describe.sql", + "describe.sql", "describe-part-after-analyze.sql", "describe-table-after-alter-table.sql", "describe-query.sql", @@ -90,7 +90,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "like-all.sql", "like-any.sql", // "limit.sql", - // TODO: fix on Spark-4.1 "literals.sql", + "literals.sql", "map.sql", "mask-functions.sql", "math.sql", @@ -228,7 +228,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { // "timestampNTZ/timestamp-ansi.sql", // "timestampNTZ/timestamp.sql", // "nonansi/array.sql", - // TODO: fix on Spark-4.1 "nonansi/cast.sql", + "nonansi/cast.sql", "nonansi/conditional-functions.sql", // "nonansi/date.sql", // "nonansi/datetime-parsing-invalid.sql", @@ -238,7 +238,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "nonansi/higher-order-functions.sql", // "nonansi/interval.sql", "nonansi/keywords.sql", - // TODO: fix on Spark-4.1 "nonansi/literals.sql", + "nonansi/literals.sql", "nonansi/map.sql", "nonansi/math.sql", "nonansi/parse-schema-string.sql", @@ -263,7 +263,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "join-resolution-edge-cases.sql", "kllquantiles.sql", "name-precedence-in-order-by-and-having-with-conflicting-attributes.sql", - // TODO: fix on Spark-4.1 "nonansi/st-functions.sql", + "nonansi/st-functions.sql", "order-by-alias.sql", "order-by-and-having-on-top-of-aggregate-with-join.sql", "replacing-missing-expression-with-alias.sql", @@ -274,7 +274,7 @@ object VeloxSQLQueryTestSettings extends SQLQueryTestSettings { "subquery/resolution-edge-cases.sql", "thetasketch.sql", "time.sql", - // TODO: fix on Spark-4.1 "type-coercion-edge-cases.sql", + "type-coercion-edge-cases.sql", "union-per-child-output-deduplication.sql", "union-resolution-edge-cases.sql", // TODO: fix on Spark-4.1 "variant-field-extractions.sql",