From 4ed616f6233388ffa45d799e08c44c6006595794 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Tue, 30 Jun 2026 03:16:24 +0000 Subject: [PATCH 1/2] [SPARK-57555][FOLLOWUP][SQL] Add escape-hatch conf to keep legacy JDBC TIME-to-timestamp mapping ### What changes were proposed in this pull request? SPARK-57555 made the built-in JDBC data source read SQL `TIME` columns as `TimeType` (instead of the legacy `TimestampType`) when `spark.sql.timeType.enabled` is true. That is a schema-inference change for existing JDBC reads: a column that previously came back as `TimestampType` now comes back as `TimeType`, with no DDL change by the user. This PR adds an internal escape-hatch conf, `spark.sql.legacy.jdbc.timeMapping.enabled` (default `false`), following the existing `spark.sql.legacy..*Mapping.enabled` family. When set to true, JDBC `TIME` columns keep the legacy `TimestampType` mapping even when `spark.sql.timeType.enabled` is true. It has no effect when `spark.sql.timeType.enabled` is false. ### Why are the changes needed? `spark.sql.timeType.enabled` is a single global switch that governs `TIME` support across all of Spark. Enabling it flips JDBC `TIME` reads from `TimestampType` to `TimeType` in one step, which can silently break workloads that depend on the old mapping (downstream casts, comparisons, stored schemas, serialization). The escape hatch lets such workloads opt back into the legacy JDBC behavior without having to keep the entire `TIME` type disabled. ### Does this PR introduce any user-facing change? No behavioral change by default. The new conf defaults to `false`, preserving SPARK-57555 behavior. When explicitly set to `true` (and `spark.sql.timeType.enabled` is `true`), JDBC `TIME` columns read as `TimestampType` as they did before SPARK-57555. ### How was this patch tested? Added a unit test in `JDBCSuite` asserting that, with `spark.sql.timeType.enabled=true`, the column reads as `TimestampType` when the escape hatch is on and as `TimeType` when it is off. ### Was this patch authored or co-authored using generative AI tooling? No --- .../apache/spark/sql/internal/SQLConf.scala | 15 +++++++++++++++ .../execution/datasources/jdbc/JdbcUtils.scala | 2 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 6776f88ed1ef8..7f19e6c42d8e6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -6289,6 +6289,18 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_JDBC_TIME_MAPPING_ENABLED = + buildConf("spark.sql.legacy.jdbc.timeMapping.enabled") + .internal() + .doc("When true, JDBC TIME columns are read as TimestampType (the legacy behavior), even " + + "when spark.sql.timeType.enabled is true. This is an escape hatch for workloads that " + + "rely on the old TIME-to-timestamp mapping. When false, JDBC TIME columns are read as " + + "TimeType if spark.sql.timeType.enabled is true. Has no effect when " + + "spark.sql.timeType.enabled is false.") + .version("4.3.0") + .booleanConf + .createWithDefault(false) + val PYTHON_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.python.filterPushdown.enabled") .internal() .doc("When true, enable filter pushdown to Python datasource, at the cost of running " + @@ -8156,6 +8168,9 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def legacyPostgresDatetimeMappingEnabled: Boolean = getConf(LEGACY_POSTGRES_DATETIME_MAPPING_ENABLED) + def legacyJdbcTimeMappingEnabled: Boolean = + getConf(LEGACY_JDBC_TIME_MAPPING_ENABLED) + override def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value = getConf(SQLConf.LEGACY_TIME_PARSER_POLICY) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala index 1ad38e7712080..4d67061e0a0cd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala @@ -229,7 +229,7 @@ object JdbcUtils extends Logging with SQLConfHelper { case java.sql.Types.SQLXML => StringType case java.sql.Types.STRUCT => StringType case java.sql.Types.TIME => - if (conf.isTimeTypeEnabled) { + if (conf.isTimeTypeEnabled && !conf.legacyJdbcTimeMappingEnabled) { // Use reported scale (fractional digits) as precision; TIME(0) is valid val timePrecision = if (scale >= 0 && scale <= TimeType.MAX_PRECISION) scale else TimeType.DEFAULT_PRECISION diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 4e75e1807cf4c..4299438a35496 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -768,6 +768,24 @@ class JDBCSuite extends SharedSparkSession { assert(rows(0).getAs[java.time.LocalTime](0) === java.time.LocalTime.of(12, 34, 56)) } + test("SPARK-57555: legacy.jdbc.timeMapping escape hatch keeps TIME as TimestampType") { + // The escape hatch forces the legacy TIME-to-timestamp mapping even when the TIME type is + // enabled, so workloads relying on the old behavior are not silently broken. + withSQLConf( + SQLConf.TIME_TYPE_ENABLED.key -> "true", + SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "true") { + val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()) + assert(df.schema("A").dataType === TimestampType) + } + // Sanity check: without the escape hatch the column is read as TimeType. + withSQLConf( + SQLConf.TIME_TYPE_ENABLED.key -> "true", + SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "false") { + val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()) + assert(df.schema("A").dataType.isInstanceOf[TimeType]) + } + } + test("SPARK-57555: JDBC TIME write round-trip") { val url = urlWithUserAndPass val tableName = "TEST.TIME_ROUNDTRIP" From 3eaeb5a46a732393118d8a5dca6b9a2986452e8a Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Tue, 30 Jun 2026 09:45:56 +0000 Subject: [PATCH 2/2] [SPARK-57555][FOLLOWUP][SQL] Set binding policy for legacy JDBC time mapping conf and cover disabled-TIME case Add `.withBindingPolicy(ConfigBindingPolicy.SESSION)` to the new `spark.sql.legacy.jdbc.timeMapping.enabled` conf so it passes `SparkConfigBindingPolicySuite`, and add a test case asserting the escape hatch has no effect when `spark.sql.timeType.enabled` is false. Co-authored-by: Isaac --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 1 + .../test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 7f19e6c42d8e6..91635d346605c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -6298,6 +6298,7 @@ object SQLConf { "TimeType if spark.sql.timeType.enabled is true. Has no effect when " + "spark.sql.timeType.enabled is false.") .version("4.3.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) .booleanConf .createWithDefault(false) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 4299438a35496..f353f6f2a6997 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -784,6 +784,14 @@ class JDBCSuite extends SharedSparkSession { val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()) assert(df.schema("A").dataType.isInstanceOf[TimeType]) } + // The escape hatch has no effect when the TIME type is disabled: the column is read as + // TimestampType regardless of the escape hatch. + withSQLConf( + SQLConf.TIME_TYPE_ENABLED.key -> "false", + SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "true") { + val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()) + assert(df.schema("A").dataType === TimestampType) + } } test("SPARK-57555: JDBC TIME write round-trip") {