Skip to content

Commit 1ae9c3d

Browse files
vinodkcMaxGekk
authored andcommitted
[SPARK-57458][SQL] Support nanosecond-precision timestamp types in the XML datasource
### What changes were proposed in this pull request? This PR adds support for nanosecond-precision timestamp types (`TimestampLTZNanosType` and `TimestampNTZNanosType`) in the XML datasource, covering: - Writing: `StaxXmlGenerator` handles the two nano timestamp types by calling the nanosecond-aware formatter methods (formatNanos, formatWithoutTimeZoneNanos). - Reading: `StaxXmlParser` routes nano timestamp types to the corresponding `parseNanos` / `parseWithoutTimeZoneNanos` formatter methods in the schema-directed path, and delegates to castTo in the type-coercion path. - Schema inference: `XmlInferSchema` now infers `TimestampNTZNanosType(9)` when a field value carries sub-microsecond fractional seconds (>6 digits) and `TIMESTAMP_NANOS_TYPES_ENABLED` is on. The compatibleType widening function is extended to merge two nano timestamp types (taking the higher precision), downgrade `TimestampNTZNanosType` + `TimestampNTZType` to `TimestampNTZType`, and fall back to `TimestampType` for any other nano/non-nano combination. The `StructType` and `ArrayType` recursive cases are moved into the pre-TypeCoercion block so that nested fields also benefit from the nano-widening logic. ### Why are the changes needed? `xml` rejected nanos timestamp types in its datasource capability checks and lacked the conversions to round-trip them, so these columns could not be written or read through `xml`. ### Does this PR introduce _any_ user-facing change? Yes. With `spark.sql.timestampNanosTypes.enabled` set to `true`: - XML files can now be written and read back with `TimestampLTZNanosType` and `TimestampNTZNanosType` columns without error. - Schema inference promotes a timestamp field to `TimestampNTZNanosType(9)` when its string value contains more than 6 fractional-second digits . ### How was this patch tested? `XmlSuite`: Added two new inference tests — one verifying that a 9-digit NTZ timestamp string is inferred as `TimestampNTZNanosType(9)`, and one verifying that a mix of micro-precision and nano-precision NTZ rows in the same file degrades to `TimestampNTZType` Updated `FileBasedDataSourceSuite` and `XmlFunctionsSuite` ### Was this patch authored or co-authored using generative AI tooling? Yes, Generated-by: Claude (Sonnet 4.6) Closes #56854 from vinodkc/SPARK-57458. Authored-by: Vinod KC <vinod.kc.in@gmail.com> Signed-off-by: Max Gekk <max.gekk@gmail.com> (cherry picked from commit fe6bc95) Signed-off-by: Max Gekk <max.gekk@gmail.com>
1 parent 4a70d92 commit 1ae9c3d

7 files changed

Lines changed: 245 additions & 39 deletions

File tree

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlGenerator.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.util.{ArrayData, DateFormatter, DateTimeUti
3232
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
3333
import org.apache.spark.sql.types._
3434
import org.apache.spark.types.variant.VariantUtil
35-
import org.apache.spark.unsafe.types.{UTF8String, VariantVal}
35+
import org.apache.spark.unsafe.types.{TimestampNanosVal, UTF8String, VariantVal}
3636

3737
class StaxXmlGenerator(
3838
schema: DataType,
@@ -200,6 +200,10 @@ class StaxXmlGenerator(
200200
gen.writeCharacters(timestampFormatter.format(v))
201201
case (TimestampNTZType, v: Long) =>
202202
gen.writeCharacters(timestampNTZFormatter.format(DateTimeUtils.microsToLocalDateTime(v)))
203+
case (t: TimestampLTZNanosType, v: TimestampNanosVal) =>
204+
gen.writeCharacters(timestampFormatter.formatNanos(v, t.precision))
205+
case (t: TimestampNTZNanosType, v: TimestampNanosVal) =>
206+
gen.writeCharacters(timestampNTZFormatter.formatWithoutTimeZoneNanos(v, t.precision))
203207
case (DateType, v: Int) =>
204208
gen.writeCharacters(dateFormatter.format(v))
205209
case (_: TimeType, v: Long) => gen.writeCharacters(timeFormatter.format(v))

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,9 @@ class StaxXmlParser(
601601
Decimal(decimalParser(datum), dt.precision, dt.scale)
602602
case _: TimestampType => parseXmlTimestamp(datum, options)
603603
case _: TimestampNTZType => timestampNTZFormatter.parseWithoutTimeZone(datum, false)
604+
case t: TimestampLTZNanosType => timestampFormatter.parseNanos(datum, t.precision)
605+
case t: TimestampNTZNanosType =>
606+
timestampNTZFormatter.parseWithoutTimeZoneNanos(datum, t.precision, false)
604607
case _: DateType => parseXmlDate(datum, options)
605608
case _: TimeType => timeFormatter.parse(datum)
606609
case _: StringType => UTF8String.fromString(datum)
@@ -652,6 +655,8 @@ class StaxXmlParser(
652655
case DateType => castTo(value, DateType)
653656
case TimestampType => castTo(value, TimestampType)
654657
case TimestampNTZType => castTo(value, TimestampNTZType)
658+
case t: TimestampLTZNanosType => castTo(value, t)
659+
case t: TimestampNTZNanosType => castTo(value, t)
655660
case _: TimeType => castTo(value, TimeType())
656661
case FloatType => signSafeToFloat(value)
657662
case ByteType => castTo(value, ByteType)

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,15 @@ class XmlInferSchema(private val options: XmlOptions, private val caseSensitive:
542542
if ((SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY ||
543543
timestampType == TimestampNTZType) &&
544544
timestampNTZFormatter.parseWithoutTimeZoneOptional(field, false).isDefined) {
545+
if (SQLConf.get.timestampNanosTypesEnabled) {
546+
// Prefer nanosecond type when the fractional seconds part has more than 6 digits,
547+
// indicating sub-microsecond precision that cannot be represented by TimestampNTZType.
548+
val nanosOpt =
549+
timestampNTZFormatter.parseWithoutTimeZoneNanosOptional(field, 9, false)
550+
nanosOpt.filter(_.nanosWithinMicro != 0).foreach { _ =>
551+
return Some(TimestampNTZNanosType(9))
552+
}
553+
}
545554
return Some(timestampType)
546555
}
547556
} catch {
@@ -641,8 +650,45 @@ object XmlInferSchema {
641650
(t1: DataType, t2: DataType): DataType = {
642651

643652
// TODO: Optimise this logic.
653+
// AnyTimestampNanoType extends DatetimeType but is not covered by findWiderDateTimeType;
654+
// handle it first to avoid a MatchError inside TypeCoercion.findTightestCommonType.
655+
// StructType and ArrayType are also handled here so that compatibleType is used recursively
656+
// for nested field types, preserving the nano-timestamp downgrade logic at all nesting levels.
657+
// (TypeCoercion.findTightestCommonType handles same-structure StructType/ArrayType via
658+
// findTypeForComplex, which calls findWiderDateTimeType and would bypass the custom logic.)
659+
(t1, t2) match {
660+
case (n1: TimestampNTZNanosType, n2: TimestampNTZNanosType) =>
661+
return TimestampNTZNanosType(math.max(n1.precision, n2.precision))
662+
case (n1: TimestampLTZNanosType, n2: TimestampLTZNanosType) =>
663+
return TimestampLTZNanosType(math.max(n1.precision, n2.precision))
664+
case (_: TimestampNTZNanosType, TimestampNTZType) |
665+
(TimestampNTZType, _: TimestampNTZNanosType) =>
666+
return TimestampNTZType
667+
case (_: AnyTimestampNanoType, _: DatetimeType) |
668+
(_: DatetimeType, _: AnyTimestampNanoType) =>
669+
return TimestampType
670+
case (StructType(fields1), StructType(fields2)) =>
671+
val newFields = (fields1 ++ fields2)
672+
// normalize field name and pair it with original field
673+
.map(field => (normalize(field.name, caseSensitive), field))
674+
.groupBy(_._1) // group by normalized field name
675+
.map { case (_: String, fields: Array[(String, StructField)]) =>
676+
val fieldTypes = fields.map(_._2)
677+
val dataType = fieldTypes.map(_.dataType)
678+
.reduce(compatibleType(caseSensitive, valueTag))
679+
// we pick up the first field name that we've encountered for the field
680+
StructField(fields.head._2.name, dataType)
681+
}
682+
return StructType(newFields.toArray.sortBy(_.name))
683+
case (ArrayType(elementType1, containsNull1), ArrayType(elementType2, containsNull2)) =>
684+
return ArrayType(
685+
compatibleType(caseSensitive, valueTag)(elementType1, elementType2),
686+
containsNull1 || containsNull2)
687+
case _ =>
688+
}
689+
644690
TypeCoercion.findTightestCommonType(t1, t2).getOrElse {
645-
// t1 or t2 is a StructType, ArrayType, or an unexpected type.
691+
// t1 or t2 is an unexpected type combination (DecimalType variants, valueTag structs, etc.)
646692
(t1, t2) match {
647693
// Double support larger range than fixed decimal, DecimalType.Maximum should be enough
648694
// in most case, also have better precision.
@@ -661,25 +707,6 @@ object XmlInferSchema {
661707
case (TimestampNTZType, TimestampType) | (TimestampType, TimestampNTZType) =>
662708
TimestampType
663709

664-
case (StructType(fields1), StructType(fields2)) =>
665-
val newFields = (fields1 ++ fields2)
666-
// normalize field name and pair it with original field
667-
.map(field => (normalize(field.name, caseSensitive), field))
668-
.groupBy(_._1) // group by normalized field name
669-
.map { case (_: String, fields: Array[(String, StructField)]) =>
670-
val fieldTypes = fields.map(_._2)
671-
val dataType = fieldTypes.map(_.dataType)
672-
.reduce(compatibleType(caseSensitive, valueTag))
673-
// we pick up the first field name that we've encountered for the field
674-
StructField(fields.head._2.name, dataType)
675-
}
676-
StructType(newFields.toArray.sortBy(_.name))
677-
678-
case (ArrayType(elementType1, containsNull1), ArrayType(elementType2, containsNull2)) =>
679-
ArrayType(
680-
compatibleType(caseSensitive, valueTag)(
681-
elementType1, elementType2), containsNull1 || containsNull2)
682-
683710
// In XML datasource, since StructType can be compared with ArrayType.
684711
// In this case, ArrayType wraps the StructType.
685712
case (ArrayType(ty1, _), ty2) =>

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/xml/XmlFileFormat.scala

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,6 @@ case class XmlFileFormat() extends TextBasedFileFormat with DataSourceRegister {
149149

150150
case _: GeometryType | _: GeographyType => false
151151

152-
// Nanosecond-capable timestamps are not yet supported by this datasource.
153-
case _: AnyTimestampNanoType => false
154-
155152
case _: AtomicType => true
156153

157154
case st: StructType => st.forall { f => supportDataType(f.dataType) }

sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1341,7 +1341,7 @@ class FileBasedDataSourceSuite extends SharedSparkSession
13411341

13421342
test("SPARK-57166: nanosecond timestamp types are not supported in selected file data sources") {
13431343
// Parquet and ORC support nanosecond-capable timestamps, while these formats still reject them.
1344-
val unsupportedDataSources = Seq("json", "xml")
1344+
val unsupportedDataSources = Seq("json")
13451345
val nanosTypes = Seq(TimestampNTZNanosType(9), TimestampLTZNanosType(9))
13461346
withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
13471347
// Test both v1 and v2 data sources.
@@ -1635,6 +1635,89 @@ class FileBasedDataSourceSuite extends SharedSparkSession
16351635
}
16361636
}
16371637
}
1638+
1639+
test("SPARK-57458: XML supports nanosecond timestamp types") {
1640+
withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
1641+
foreachNanosPrecision { precision =>
1642+
Seq(TimestampNTZNanosType(precision), TimestampLTZNanosType(precision)).foreach {
1643+
nanosType =>
1644+
withTempDir { dir =>
1645+
val wallClock = LocalDateTime.of(1970, 1, 1, 0, 20, 34, 567890123)
1646+
val (value, fmtKey, fmtVal) = nanosType match {
1647+
case _: TimestampNTZNanosType =>
1648+
(wallClock.asInstanceOf[Any],
1649+
"timestampNTZFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS")
1650+
case _: TimestampLTZNanosType =>
1651+
(wallClock.toInstant(ZoneOffset.UTC).asInstanceOf[Any],
1652+
"timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX")
1653+
}
1654+
val df = spark.createDataFrame(
1655+
spark.sparkContext.parallelize(Seq(Row(value))),
1656+
new StructType().add("ts", nanosType))
1657+
val path = new File(dir, s"xml_nanos_${nanosType.typeName}").getCanonicalPath
1658+
df.write.format("xml").option("rowTag", "row")
1659+
.option(fmtKey, fmtVal).mode("overwrite").save(path)
1660+
val readBack = spark.read.schema(new StructType().add("ts", nanosType))
1661+
.format("xml").option("rowTag", "row").option(fmtKey, fmtVal).load(path)
1662+
checkAnswer(readBack, df)
1663+
}
1664+
}
1665+
}
1666+
}
1667+
}
1668+
1669+
test("SPARK-57458: XML rejects nanosecond timestamps under the LEGACY time parser policy") {
1670+
// The legacy timestamp formatter cannot represent sub-microsecond digits, so the nanos
1671+
// formatter methods raise TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER. Only the LTZ formatter
1672+
// is legacy under this policy (the NTZ formatter always uses the ISO-8601 path), so this
1673+
// covers TimestampLTZNanosType. XML is v1-only so there is no v1/v2 branching to test.
1674+
def rootNanosError(e: Throwable): SparkUnsupportedOperationException = {
1675+
var cause: Throwable = e
1676+
while (cause != null && !cause.isInstanceOf[SparkUnsupportedOperationException]) {
1677+
cause = cause.getCause
1678+
}
1679+
assert(cause != null,
1680+
s"Expected TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER, but got: ${e.getMessage}")
1681+
cause.asInstanceOf[SparkUnsupportedOperationException]
1682+
}
1683+
1684+
withSQLConf(
1685+
SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
1686+
SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "LEGACY") {
1687+
val nanosType = TimestampLTZNanosType(9)
1688+
val schema = new StructType().add("ts", nanosType)
1689+
val expectedParameters =
1690+
Map("config" -> ("\"" + SQLConf.LEGACY_TIME_PARSER_POLICY.key + "\""))
1691+
withTempDir { dir =>
1692+
// Write path.
1693+
val df = spark.createDataFrame(
1694+
spark.sparkContext.parallelize(
1695+
Seq(Row(LocalDateTime.of(2020, 1, 1, 0, 0, 0, 1).toInstant(ZoneOffset.UTC)))),
1696+
schema)
1697+
val writeDir = new File(dir, "write").getCanonicalPath
1698+
checkError(
1699+
exception = rootNanosError(intercept[SparkException] {
1700+
df.write.format("xml").option("rowTag", "row").mode("overwrite").save(writeDir)
1701+
}),
1702+
condition = "UNSUPPORTED_FEATURE.TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER",
1703+
parameters = expectedParameters)
1704+
1705+
// Read path: write a benign file first so schema-driven parsing is what fails. Use
1706+
// FAILFAST so the unsupported-feature error surfaces instead of being swallowed as a
1707+
// null record by the permissive bad-record handling.
1708+
val readDir = new File(dir, "read").getCanonicalPath
1709+
Seq("a").toDF("ts").write.format("xml").option("rowTag", "row")
1710+
.mode("overwrite").save(readDir)
1711+
checkError(
1712+
exception = rootNanosError(intercept[SparkException] {
1713+
spark.read.schema(schema).option("mode", "FAILFAST").option("rowTag", "row")
1714+
.format("xml").load(readDir).collect()
1715+
}),
1716+
condition = "UNSUPPORTED_FEATURE.TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER",
1717+
parameters = expectedParameters)
1718+
}
1719+
}
1720+
}
16381721
}
16391722

16401723
object TestingUDT {

sql/core/src/test/scala/org/apache/spark/sql/XmlFunctionsSuite.scala

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
package org.apache.spark.sql
1919

2020
import java.text.SimpleDateFormat
21+
import java.time.{LocalDateTime, ZoneOffset}
2122
import java.util.Locale
2223

2324
import scala.jdk.CollectionConverters._
2425

2526
import org.apache.spark.SparkException
27+
import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils
2628
import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils.foreachNanosPrecision
2729
import org.apache.spark.sql.execution.WholeStageCodegenExec
2830
import org.apache.spark.sql.functions._
@@ -45,25 +47,45 @@ class XmlFunctionsSuite extends SharedSparkSession {
4547
test("SPARK-57164: from_xml with a nanos timestamp DDL schema string") {
4648
val df = Seq("""<ROW><c>2020-01-01T00:00:00.123456789</c></ROW>""").toDF("value")
4749
// FAILFAST so the value-converter rejection propagates instead of becoming a corrupt record.
50+
// Pin the session timezone to UTC so LTZ values are predictable without a zone in the string.
4851
val options = Map("mode" -> "FAILFAST").asJava
49-
withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
52+
withSQLConf(
53+
SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
54+
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
5055
foreachNanosPrecision { p =>
56+
val truncator = TimestampNanosTestUtils.nanoOfSecTruncator(p)
57+
val truncNanos = truncator(123456789)
58+
val expectedNTZ = LocalDateTime.of(2020, 1, 1, 0, 0, 0, truncNanos)
59+
val expectedLTZ = expectedNTZ.toInstant(ZoneOffset.UTC)
5160
Seq(
52-
s"TIMESTAMP_NTZ($p)" -> TimestampNTZNanosType(p),
53-
s"TIMESTAMP_LTZ($p)" -> TimestampLTZNanosType(p),
54-
s"TIMESTAMP($p) WITHOUT TIME ZONE" -> TimestampNTZNanosType(p),
55-
s"TIMESTAMP($p) WITH LOCAL TIME ZONE" -> TimestampLTZNanosType(p)).foreach {
56-
case (spelling, expected) =>
61+
s"TIMESTAMP_NTZ($p)" -> (TimestampNTZNanosType(p), expectedNTZ.asInstanceOf[Any]),
62+
s"TIMESTAMP_LTZ($p)" -> (TimestampLTZNanosType(p), expectedLTZ.asInstanceOf[Any]),
63+
s"TIMESTAMP($p) WITHOUT TIME ZONE" ->
64+
(TimestampNTZNanosType(p), expectedNTZ.asInstanceOf[Any]),
65+
s"TIMESTAMP($p) WITH LOCAL TIME ZONE" ->
66+
(TimestampLTZNanosType(p), expectedLTZ.asInstanceOf[Any])).foreach {
67+
case (spelling, (expectedType, expectedVal)) =>
5768
val parsed = df.select(from_xml($"value", s"c $spelling", options).as("v"))
5869
// The schema string resolves to the nanos type ...
59-
assert(parsed.schema("v").dataType.asInstanceOf[StructType]("c").dataType === expected)
60-
// ... but the XML datasource does not support nanosecond timestamps yet, so the
61-
// value converter rejects it at execution (surfaced as a malformed record in
62-
// FAILFAST mode).
63-
checkError(
64-
exception = intercept[SparkException](parsed.collect()),
65-
condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
66-
parameters = Map("badRecord" -> "[null]", "failFastMode" -> "FAILFAST"))
70+
val parsedType = parsed.schema("v").dataType.asInstanceOf[StructType]("c").dataType
71+
assert(parsedType === expectedType)
72+
// ... the XML datasource parses the value and round-trips to the expected value.
73+
checkAnswer(parsed, Row(Row(expectedVal)) :: Nil)
74+
}
75+
}
76+
}
77+
}
78+
79+
test("SPARK-57458: from_xml rejects zoned input for NTZ nanos columns") {
80+
// A string with an explicit zone offset must be rejected for TIMESTAMP_NTZ(p) because
81+
// the NTZ parse path uses allowTimeZone=false, matching the micro NTZ and CSV behaviour.
82+
val df = Seq("""<ROW><c>2020-01-01T00:00:00.123456789+05:00</c></ROW>""").toDF("value")
83+
val options = Map("mode" -> "FAILFAST").asJava
84+
withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
85+
foreachNanosPrecision { p =>
86+
val parsed = df.select(from_xml($"value", s"c TIMESTAMP_NTZ($p)", options).as("v"))
87+
intercept[SparkException] {
88+
parsed.collect()
6789
}
6890
}
6991
}

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3694,6 +3694,74 @@ class XmlSuite
36943694
assert(XmlOptions.isValidOption("encoding"))
36953695
assert(XmlOptions.isValidOption("charset"))
36963696
}
3697+
3698+
// Full-precision format pattern for nanosecond NTZ timestamp schema-inference tests.
3699+
private val ntzNanosFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS"
3700+
3701+
test("SPARK-57458: XML infers nanosecond NTZ timestamps from sub-microsecond fractional digits") {
3702+
withSQLConf(
3703+
SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
3704+
SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_NTZ") {
3705+
// Write a nanosecond DataFrame with a 9-digit format so the XML values carry 9 fractional
3706+
// digits; then read without a schema so the type is inferred from the string values.
3707+
val wallClock = LocalDateTime.of(2025, 6, 15, 12, 30, 45, 123456789)
3708+
val ntzType = TimestampNTZNanosType(9)
3709+
val inputDf = spark.createDataFrame(
3710+
spark.sparkContext.parallelize(Seq(Row(wallClock))),
3711+
new StructType().add("ts", ntzType))
3712+
withTempPath { dir =>
3713+
val path = dir.getCanonicalPath
3714+
inputDf.write.format("xml").option("rowTag", "ROW")
3715+
.option("timestampNTZFormat", ntzNanosFormat).mode("overwrite").save(path)
3716+
val df = spark.read.format("xml").option("rowTag", "ROW").load(path)
3717+
assert(df.schema("ts").dataType === TimestampNTZNanosType(9),
3718+
s"Expected TimestampNTZNanosType(9), got ${df.schema("ts").dataType}")
3719+
}
3720+
}
3721+
}
3722+
3723+
test("SPARK-57458: XML inferred type is TimestampNTZType for mixed micro/nano NTZ rows") {
3724+
// When some rows have >6 fractional digits (nano) and others have <=6 (micro), the inferred
3725+
// type must be TimestampNTZType (not TimestampType / LTZ), because all values are zone-free.
3726+
withSQLConf(
3727+
SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
3728+
SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_NTZ") {
3729+
val xmlContent =
3730+
"""<root><row><ts>2025-06-15T12:30:45.123456789</ts></row>
3731+
|<row><ts>2025-06-15T12:30:45.123456</ts></row></root>""".stripMargin
3732+
withTempDir { dir =>
3733+
val path = new File(dir, "mixed.xml").getCanonicalPath
3734+
Files.write(Paths.get(path),
3735+
xmlContent.getBytes(StandardCharsets.UTF_8))
3736+
val df = spark.read.format("xml").option("rowTag", "row")
3737+
.option("rootTag", "root").load(path)
3738+
assert(df.schema("ts").dataType === TimestampNTZType,
3739+
s"Expected TimestampNTZType, got ${df.schema("ts").dataType}")
3740+
}
3741+
}
3742+
}
3743+
3744+
test("SPARK-57458: nano timestamp + non-datetime field widens to StringType during inference") {
3745+
// When a field is a nano-precision timestamp in some rows and a non-datetime value in others,
3746+
// inference must fall back to StringType (matching the micro-precision path) rather than
3747+
// widening to TimestampType and then failing at read time.
3748+
withSQLConf(
3749+
SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
3750+
SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_NTZ") {
3751+
val xmlContent =
3752+
"""<root><row><ts>2025-06-15T12:30:45.123456789</ts></row>
3753+
|<row><ts>not-a-timestamp</ts></row></root>""".stripMargin
3754+
withTempDir { dir =>
3755+
val path = new File(dir, "nano_nondatetime.xml").getCanonicalPath
3756+
Files.write(Paths.get(path), xmlContent.getBytes(StandardCharsets.UTF_8))
3757+
val df = spark.read.format("xml").option("rowTag", "row")
3758+
.option("rootTag", "root").load(path)
3759+
assert(df.schema("ts").dataType === StringType,
3760+
s"Expected StringType for nano + non-datetime, got ${df.schema("ts").dataType}")
3761+
}
3762+
}
3763+
}
3764+
36973765
}
36983766

36993767
class XmlSuiteWithLegacyParser extends XmlSuite {

0 commit comments

Comments
 (0)