Skip to content

Commit 5ab0a91

Browse files
github-actions[bot]feiniaofeiafei
github-actions[bot]
and
feiniaofeiafei
authored
branch-4.1: [fix](fe) Align convert_tz folding with BE DST handling #64029 (#64195)
Cherry-picked from #64029 Co-authored-by: feiniaofeiafei <moailing@selectdb.com>
1 parent b29c3e4 commit 5ab0a91

9 files changed

Lines changed: 136 additions & 14 deletions

File tree

‎be/src/exprs/function/function_convert_tz.cpp‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,19 @@ class FunctionConvertTZ : public IFunction {
213213
}
214214
}
215215

216+
static std::pair<int64_t, int64_t> unix_timestamp_for_convert_tz(
217+
const DateValueType& ts_value, const cctz::time_zone& from_tz) {
218+
cctz::civil_second civil_time(ts_value.year(), ts_value.month(), ts_value.day(),
219+
ts_value.hour(), ts_value.minute(), ts_value.second());
220+
const auto lookup = from_tz.lookup(civil_time);
221+
const bool skipped = lookup.kind == cctz::time_zone::civil_lookup::SKIPPED;
222+
const auto tp = skipped ? lookup.trans : lookup.pre;
223+
224+
// Skipped civil times map to the transition instant. Do not keep the
225+
// input fractional part inside a local time interval that never existed.
226+
return {tp.time_since_epoch().count(), skipped ? 0 : ts_value.microsecond()};
227+
}
228+
216229
static void execute_tz_const_with_state(ConvertTzState* convert_tz_state,
217230
const ColumnType* date_column,
218231
ColumnType* result_column, NullMap& result_null_map,
@@ -240,9 +253,7 @@ class FunctionConvertTZ : public IFunction {
240253
DateValueType ts_value = date_column->get_element(i);
241254
DateValueType ts_value2;
242255

243-
std::pair<int64_t, int64_t> timestamp;
244-
ts_value.unix_timestamp(&timestamp, from_tz);
245-
ts_value2.from_unixtime(timestamp, to_tz);
256+
ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz);
246257

247258
if (!ts_value2.is_valid_date()) [[unlikely]] {
248259
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(i),
@@ -293,9 +304,7 @@ class FunctionConvertTZ : public IFunction {
293304
to_tz_name);
294305
}
295306

296-
std::pair<int64_t, int64_t> timestamp;
297-
ts_value.unix_timestamp(&timestamp, from_tz);
298-
ts_value2.from_unixtime(timestamp, to_tz);
307+
ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz);
299308

300309
if (!ts_value2.is_valid_date()) [[unlikely]] {
301310
throw_out_of_bound_convert_tz<DateValueType>(date_column->get_element(index_now),

‎be/test/exprs/function/function_time_test.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,23 @@ TEST(VTimestampFunctionsTest, convert_tz_test) {
326326
check_function<DataTypeDateTimeV2, true>(func_name, input_types, data_set));
327327
}
328328

329+
{
330+
InputTypeSet input_types_scale6 = {{PrimitiveType::TYPE_DATETIMEV2, 6},
331+
PrimitiveType::TYPE_VARCHAR,
332+
PrimitiveType::TYPE_VARCHAR};
333+
DataSet data_set = {{{std::string {"2021-03-28 02:15:30.123456"},
334+
std::string {"Europe/Paris"}, std::string {"UTC"}},
335+
std::string("2021-03-28 01:00:00.000000")},
336+
{{std::string {"2021-03-28 03:00:30.123456"},
337+
std::string {"Europe/Paris"}, std::string {"UTC"}},
338+
std::string("2021-03-28 01:00:30.123456")},
339+
{{std::string {"2021-10-31 02:15:30.123456"},
340+
std::string {"Europe/Paris"}, std::string {"UTC"}},
341+
std::string("2021-10-31 00:15:30.123456")}};
342+
static_cast<void>(check_function<DataTypeDateTimeV2, true>(func_name, input_types_scale6,
343+
data_set, 6));
344+
}
345+
329346
{
330347
DataSet data_set = {{{std::string {"2019-08-01 02:18:27"}, std::string {"Asia/Shanghai"},
331348
std::string {"UTC"}},

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.doris.nereids.trees.expressions.functions.scalar.FromSecond;
2727
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
2828
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
29+
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
2930
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
3031
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
3132
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
@@ -814,8 +815,9 @@ public static Expression convertTz(DateTimeV2Literal datetime, StringLikeLiteral
814815
ZoneId toZone = ZoneId.from(zoneFormatter.parse(toTz.getStringValue()));
815816

816817
LocalDateTime localDateTime = datetime.toJavaDateType();
817-
ZonedDateTime resultDateTime = localDateTime.atZone(fromZone).withZoneSameInstant(toZone);
818-
return DateTimeV2Literal.fromJavaDateType(resultDateTime.toLocalDateTime(), datetime.getDataType().getScale());
818+
Instant instant = DateTimeLiteral.convertLocalToInstant(localDateTime, fromZone);
819+
return DateTimeV2Literal.fromJavaDateType(LocalDateTime.ofInstant(instant, toZone),
820+
datetime.getDataType().getScale());
819821
}
820822

821823
private static void validateTimezoneOffset(String timezone) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,16 @@
3737
import org.apache.logging.log4j.Logger;
3838

3939
import java.math.BigInteger;
40+
import java.time.Instant;
4041
import java.time.LocalDateTime;
4142
import java.time.ZoneId;
43+
import java.time.ZoneOffset;
4244
import java.time.temporal.ChronoField;
4345
import java.time.temporal.TemporalAccessor;
4446
import java.time.temporal.TemporalQueries;
47+
import java.time.zone.ZoneOffsetTransition;
48+
import java.time.zone.ZoneRules;
49+
import java.util.List;
4550
import java.util.Objects;
4651

4752
/**
@@ -271,10 +276,33 @@ protected void init(String s) throws AnalysisException {
271276

272277
private static LocalDateTime convertTimeZone(long year, long month, long day, long hour, long minute,
273278
long second, ZoneId fromZone, ZoneId toZone) {
274-
return LocalDateTime.of((int) year, (int) month, (int) day, (int) hour, (int) minute, (int) second)
275-
.atZone(fromZone)
276-
.withZoneSameInstant(toZone)
277-
.toLocalDateTime();
279+
LocalDateTime localDateTime = LocalDateTime.of((int) year, (int) month, (int) day,
280+
(int) hour, (int) minute, (int) second);
281+
Instant instant = convertLocalToInstant(localDateTime, fromZone);
282+
return LocalDateTime.ofInstant(instant, toZone);
283+
}
284+
285+
/**
286+
* Convert a local civil datetime in {@code fromZone} to an instant with the same DST transition
287+
* policy as BE cctz::convert(civil_second, zone).
288+
*
289+
* <p>For normal local times, there is one valid offset. For fall-back overlap times, two offsets
290+
* are valid and the first one is the pre-transition offset. For spring-forward gap times, the
291+
* local time does not exist, so any value inside the skipped interval maps to the transition
292+
* instant.
293+
*/
294+
public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) {
295+
ZoneRules rules = fromZone.getRules();
296+
List<ZoneOffset> validOffsets = rules.getValidOffsets(localDateTime);
297+
int size = validOffsets.size();
298+
// Match BE cctz::convert(civil_second, zone) semantics for constant folding.
299+
// Normal local time has one offset; repeated local time uses the pre-transition offset.
300+
if (size == 1 || size == 2) {
301+
return localDateTime.atOffset(validOffsets.get(0)).toInstant();
302+
}
303+
// Skipped local time maps to the transition instant, e.g. 2021-03-28 02:15 Europe/Paris.
304+
ZoneOffsetTransition transition = rules.getTransition(localDateTime);
305+
return transition.getInstant();
278306
}
279307

280308
public boolean checkRange() {

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
2525
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
2626
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
27+
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
2728
import org.apache.doris.nereids.types.DateTimeV2Type;
2829

2930
import org.junit.jupiter.api.Assertions;
@@ -166,4 +167,46 @@ public void testFromUnixTimeOutOfRangeThrows() {
166167
Assertions.assertThrows(AnalysisException.class,
167168
() -> DateTimeExtractAndTransform.fromUnixTime(dec));
168169
}
170+
171+
@Test
172+
void testConvertTzDstTransition() {
173+
// Spring gap maps skipped local times to the transition instant.
174+
Assertions.assertEquals(
175+
new DateTimeV2Literal("2021-03-28 01:00:00"),
176+
DateTimeExtractAndTransform.convertTz(
177+
new DateTimeV2Literal("2021-03-28 02:15:00"),
178+
new VarcharLiteral("Europe/Paris"),
179+
new VarcharLiteral("UTC")));
180+
Assertions.assertEquals(
181+
new DateTimeV2Literal("2021-03-28 01:00:00"),
182+
DateTimeExtractAndTransform.convertTz(
183+
new DateTimeV2Literal("2021-03-28 02:00:00"),
184+
new VarcharLiteral("Europe/Paris"),
185+
new VarcharLiteral("UTC")));
186+
Assertions.assertEquals(
187+
new DateTimeV2Literal("2021-03-28 01:00:00"),
188+
DateTimeExtractAndTransform.convertTz(
189+
new DateTimeV2Literal("2021-03-28 03:00:00"),
190+
new VarcharLiteral("Europe/Paris"),
191+
new VarcharLiteral("UTC")));
192+
// Fall overlap uses the pre-transition offset.
193+
Assertions.assertEquals(
194+
new DateTimeV2Literal("2021-10-31 00:15:00"),
195+
DateTimeExtractAndTransform.convertTz(
196+
new DateTimeV2Literal("2021-10-31 02:15:00"),
197+
new VarcharLiteral("Europe/Paris"),
198+
new VarcharLiteral("UTC")));
199+
Assertions.assertEquals(
200+
new DateTimeV2Literal("2021-10-31 00:00:00"),
201+
DateTimeExtractAndTransform.convertTz(
202+
new DateTimeV2Literal("2021-10-31 02:00:00"),
203+
new VarcharLiteral("Europe/Paris"),
204+
new VarcharLiteral("UTC")));
205+
Assertions.assertEquals(
206+
new DateTimeV2Literal("2021-10-31 02:00:00"),
207+
DateTimeExtractAndTransform.convertTz(
208+
new DateTimeV2Literal("2021-10-31 03:00:00"),
209+
new VarcharLiteral("Europe/Paris"),
210+
new VarcharLiteral("UTC")));
211+
}
169212
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,24 @@ void testConstructorsAndParsing() {
7575
Assertions.assertEquals(0, literal.minute);
7676
Assertions.assertEquals(0, literal.second);
7777
Assertions.assertEquals(0, literal.microSecond);
78+
79+
literal = new TimestampTzLiteral(TimeStampTzType.of(6), "2024-03-10 02:30:00 America/New_York");
80+
Assertions.assertEquals(2024, literal.year);
81+
Assertions.assertEquals(3, literal.month);
82+
Assertions.assertEquals(10, literal.day);
83+
Assertions.assertEquals(7, literal.hour);
84+
Assertions.assertEquals(0, literal.minute);
85+
Assertions.assertEquals(0, literal.second);
86+
Assertions.assertEquals(0, literal.microSecond);
87+
88+
literal = new TimestampTzLiteral(TimeStampTzType.of(6), "2024-11-03 01:30:00 America/New_York");
89+
Assertions.assertEquals(2024, literal.year);
90+
Assertions.assertEquals(11, literal.month);
91+
Assertions.assertEquals(3, literal.day);
92+
Assertions.assertEquals(5, literal.hour);
93+
Assertions.assertEquals(30, literal.minute);
94+
Assertions.assertEquals(0, literal.second);
95+
Assertions.assertEquals(0, literal.microSecond);
7896
}
7997

8098
@Test

‎regression-test/data/datatype_p0/timestamptz/test_timestamptz_dst_gap.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
-- This file is automatically generated. You should know what you did if you want to edit this
22
-- !sql --
3-
1 named_gap_ny 2024-03-10 07:30:00.000000+00:00
3+
1 named_gap_ny 2024-03-10 07:00:00.000000+00:00
44
2 explicit_before_gap 2024-03-10 06:30:00.000000+00:00
55
3 explicit_after_gap 2024-03-10 07:30:00.000000+00:00
6-
4 implicit_gap_ny 2024-03-10 07:30:00.000000+00:00
6+
4 implicit_gap_ny 2024-03-10 07:00:00.000000+00:00
77
5 implicit_after_gap 2024-03-10 07:30:00.000000+00:00
88

99
-- !sql --

‎regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,6 @@
2121
2024-04-18T23:20
2222
2024-04-18T23:20
2323

24+
-- !spring_gp_with_micro_sec --
25+
2021-03-28T01:00
26+

‎regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,6 @@ suite("test_convert_tz") {
3535
order_qt_sql1 """
3636
select convert_tz(dt, '+00:00', IF(property_value IS NULL, '+00:00', property_value)) from cvt_tz
3737
"""
38+
sql "set debug_skip_fold_constant=true;"
39+
qt_spring_gp_with_micro_sec "select convert_tz('2021-03-28 02:30:00.00323', 'Europe/Paris','UTC');"
3840
}

0 commit comments

Comments
 (0)