fix: fall back to Spark for str_to_map when legacy regex split truncation is enabled (apache#4627)

Author: andygrove

spark/src/main/scala/org/apache/comet/serde/maps.scala

@@ -20,6 +20,7 @@
 package org.apache.comet.serde
 
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInternal, optExprWithFallbackReason, scalarFunctionExprToProto}
@@ -164,7 +165,22 @@ object CometMapFromEntries
   }
 }
 
-object CometStrToMap extends CometScalarFunction[StringToMap]("str_to_map")
+object CometStrToMap extends CometScalarFunction[StringToMap]("str_to_map") {
+
+  // Spark 4.1.1+ honours spark.sql.legacy.truncateForEmptyRegexSplit by truncating trailing
+  // empty entries from the split result. Comet's native str_to_map always behaves as if the flag
+  // were false, so it is incompatible when legacy truncation is enabled. Read by string key so it
+  // resolves on older Spark versions where the config is not registered.
+  private val legacyTruncateConfig = "spark.sql.legacy.truncateForEmptyRegexSplit"
+
+  override def getSupportLevel(expr: StringToMap): SupportLevel = {
+    if (SQLConf.get.getConfString(legacyTruncateConfig, "false").toBoolean) {
+      Incompatible(Some(s"$legacyTruncateConfig is enabled"))
+    } else {
+      Compatible(None)
+    }
+  }
+}
 
 object CometMapFilter extends CometCodegenDispatch[MapFilter]
 

spark/src/test/resources/sql-tests/expressions/map/str_to_map_legacy_truncate.sql

@@ -0,0 +1,41 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Tests that str_to_map falls back to Spark when
+-- spark.sql.legacy.truncateForEmptyRegexSplit is enabled. In legacy mode Spark truncates trailing
+-- empty entries from the split result, which Comet's native str_to_map does not honour.
+-- See https://github.com/apache/datafusion-comet/issues/4477
+
+-- Config: spark.sql.legacy.truncateForEmptyRegexSplit=true
+
+-- trailing pair delimiter: legacy mode truncates the trailing empty entry, so Comet must fall
+-- back to Spark
+query expect_fallback(truncateForEmptyRegexSplit)
+SELECT str_to_map('a:1,b:2,', ',', ':')
+
+-- column input also falls back
+statement
+CREATE TABLE test_str_to_map_legacy(s STRING, pair_delim STRING, key_value_delim STRING) USING parquet
+
+statement
+INSERT INTO test_str_to_map_legacy VALUES
+  ('a:1,b:2,', ',', ':'),
+  ('x:1;y:2;', ';', ':'),
+  (NULL, ',', ':')
+
+query expect_fallback(truncateForEmptyRegexSplit)
+SELECT str_to_map(s, pair_delim, key_value_delim) FROM test_str_to_map_legacy