Skip to content

Commit 52b892c

Browse files
yuancuyuancu
committed
Default to UTC for date/time functions across PPL and SQL (opensearch-project#3854)
* Use UTC for all date/time related functions Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Change zone ID to UTC in date/time tests Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Update now's timezone to UTC in datetime documentation Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Add a test to compare against UTC date Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Revert changes to sysdate: make it return timestamp in cluster timezone Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Correct utc_time and utc_timestamp to return time in HH:mm:ss format (intead of with random length of milliseconds) Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Update datetime function documentations to highlight changes to UTC for now-like functions Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Correct phrasing of sysdate and some other datetime function documentations Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Add a note directive block at the top of datetime to describe the time zone behavior Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Revert "Revert changes to sysdate: make it return timestamp in cluster timezone" This reverts commit b7be897. Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Update sysdate's description Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Update datetime.rst - remove expressions indicating PPL supporting other time zones Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Add STRING to allowed first parameter of convert_tz as it returns null for malformatted date Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Restrict the format of convert_tz's argument Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> --------- Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> (cherry picked from commit b3c0177)
1 parent b9331be commit 52b892c

18 files changed

Lines changed: 161 additions & 96 deletions

File tree

‎common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public Grok compile(String pattern) throws IllegalArgumentException {
117117
}
118118

119119
public Grok compile(final String pattern, boolean namedOnly) throws IllegalArgumentException {
120-
return compile(pattern, ZoneOffset.systemDefault(), namedOnly);
120+
return compile(pattern, ZoneOffset.UTC, namedOnly);
121121
}
122122

123123
/**

‎common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -680,8 +680,7 @@ public void testTimeZone() {
680680
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss");
681681
Grok grok = compiler.compile("%{DATESTAMP:timestamp;date;MM/dd/yyyy HH:mm:ss}", true);
682682
Instant instant = (Instant) grok.match(date).capture().get("timestamp");
683-
assertEquals(
684-
ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.systemDefault())).toInstant(), instant);
683+
assertEquals(ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.UTC)).toInstant(), instant);
685684

686685
// set default timezone to PST
687686
ZoneId pst = ZoneId.of("PST", ZoneId.SHORT_IDS);

‎core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,13 @@
1212

1313
import java.time.Instant;
1414
import java.time.ZoneId;
15+
import java.time.ZoneOffset;
1516
import java.util.ArrayList;
1617
import java.util.Collections;
1718
import java.util.List;
19+
import java.util.Set;
1820
import java.util.TimeZone;
1921
import java.util.stream.Collectors;
20-
2122
import javax.annotation.Nullable;
2223
import org.apache.calcite.DataContext;
2324
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
@@ -137,8 +138,7 @@ public static FunctionProperties restoreFunctionProperties(DataContext dataConte
137138
Instant instant =
138139
Instant.ofEpochSecond(
139140
currentTimeInNanos / 1_000_000_000, currentTimeInNanos % 1_000_000_000);
140-
TimeZone timeZone = TimeZone.getDefault();
141-
ZoneId zoneId = timeZone.toZoneId();
141+
ZoneId zoneId = ZoneOffset.UTC;
142142
return new FunctionProperties(instant, zoneId, QueryType.PPL);
143143
}
144144

‎core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_MONTH_DATE_LENGTH;
4343
import static org.opensearch.sql.utils.DateTimeFormatters.SINGLE_DIGIT_YEAR_DATE_LENGTH;
4444
import static org.opensearch.sql.utils.DateTimeUtils.UTC_ZONE_ID;
45-
import static org.opensearch.sql.utils.DateTimeFormatters.*;
4645
import static org.opensearch.sql.utils.DateTimeUtils.extractDate;
4746
import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime;
4847

@@ -70,7 +69,6 @@
7069
import java.time.temporal.TemporalAmount;
7170
import java.util.Locale;
7271
import java.util.Map;
73-
import java.util.TimeZone;
7472
import java.util.concurrent.TimeUnit;
7573
import java.util.stream.Stream;
7674
import lombok.experimental.UtilityClass;
@@ -97,6 +95,7 @@
9795
import org.opensearch.sql.expression.function.FunctionSignature;
9896
import org.opensearch.sql.expression.function.SerializableFunction;
9997
import org.opensearch.sql.expression.function.SerializableTriFunction;
98+
import org.opensearch.sql.utils.DateTimeFormatters;
10099
import org.opensearch.sql.utils.DateTimeUtils;
101100

102101
/**
@@ -285,12 +284,14 @@ private FunctionResolver sysdate() {
285284
return define(
286285
BuiltinFunctionName.SYSDATE.getName(),
287286
implWithProperties(
288-
functionProperties -> new ExprDatetimeValue(formatNow(Clock.systemDefaultZone())),
289-
DATETIME),
287+
functionProperties ->
288+
new ExprDatetimeValue(formatNow(functionProperties.getSystemClock())),
289+
DATETIME),
290290
FunctionDSL.implWithProperties(
291291
(functionProperties, v) ->
292-
new ExprDatetimeValue(formatNow(Clock.systemDefaultZone(), v.integerValue())),
293-
DATETIME,
292+
new ExprDatetimeValue(
293+
formatNow(functionProperties.getSystemClock(), v.integerValue())),
294+
DATETIME,
294295
INTEGER));
295296
}
296297

@@ -581,8 +582,15 @@ private DefaultFunctionResolver datediff() {
581582
private FunctionResolver datetime() {
582583
return define(
583584
BuiltinFunctionName.DATETIME.getName(),
584-
impl(nullMissingHandling(DateTimeFunctions::exprDateTime), DATETIME, STRING, STRING),
585-
impl(nullMissingHandling(DateTimeFunctions::exprDateTimeNoTimezone), DATETIME, STRING));
585+
implWithProperties(
586+
nullMissingHandlingWithProperties(DateTimeFunctions::exprDateTime),
587+
DATETIME,
588+
STRING,
589+
STRING),
590+
implWithProperties(
591+
nullMissingHandlingWithProperties(DateTimeFunctions::exprDateTimeNoTimezone),
592+
DATETIME,
593+
STRING));
586594
}
587595

588596
private DefaultFunctionResolver date_add() {
@@ -1531,7 +1539,16 @@ public static ExprValue exprAddTime(
15311539
public static ExprValue exprConvertTZ(
15321540
ExprValue startingDateTime, ExprValue fromTz, ExprValue toTz) {
15331541
if (startingDateTime.type() == ExprCoreType.STRING) {
1534-
startingDateTime = exprDateTimeNoTimezone(startingDateTime);
1542+
try {
1543+
// CONVERT_TZ only expects a timestamp in the format "yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]".
1544+
startingDateTime =
1545+
new ExprDatetimeValue(
1546+
LocalDateTime.parse(
1547+
startingDateTime.stringValue(),
1548+
DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS));
1549+
} catch (DateTimeParseException e) {
1550+
return ExprNullValue.of();
1551+
}
15351552
}
15361553
try {
15371554
ZoneId convertedFromTz = ZoneId.of(fromTz.stringValue());
@@ -1591,8 +1608,10 @@ public static ExprValue exprDateDiff(
15911608
* @param timeZone ExprValue of String type (or null).
15921609
* @return ExprValue of date type.
15931610
*/
1594-
public static ExprValue exprDateTime(ExprValue dateTime, ExprValue timeZone) {
1595-
String defaultTimeZone = TimeZone.getDefault().toZoneId().toString();
1611+
public static ExprValue exprDateTime(
1612+
FunctionProperties properties, ExprValue dateTime, ExprValue timeZone) {
1613+
// Get default time zone from function properties instead of ZoneId.systemDefault()
1614+
String defaultTimeZone = properties.getCurrentZoneId().toString();
15961615

15971616
try {
15981617
LocalDateTime ldtFormatted =
@@ -1632,8 +1651,9 @@ public static ExprValue exprDateTime(ExprValue dateTime, ExprValue timeZone) {
16321651
* @param dateTime ExprValue of String type.
16331652
* @return ExprValue of date type.
16341653
*/
1635-
public static ExprValue exprDateTimeNoTimezone(ExprValue dateTime) {
1636-
return exprDateTime(dateTime, ExprNullValue.of());
1654+
public static ExprValue exprDateTimeNoTimezone(
1655+
FunctionProperties properties, ExprValue dateTime) {
1656+
return exprDateTime(properties, dateTime, ExprNullValue.of());
16371657
}
16381658

16391659
/**
@@ -2246,9 +2266,8 @@ public static ExprValue exprUtcTime(FunctionProperties functionProperties) {
22462266
* @return ExprValue.
22472267
*/
22482268
public static ExprValue exprUtcTimestamp(FunctionProperties functionProperties) {
2249-
var zdt =
2250-
ZonedDateTime.now(functionProperties.getQueryStartClock()).withZoneSameInstant(UTC_ZONE_ID);
2251-
return new ExprDatetimeValue(zdt.toLocalDateTime());
2269+
var dt = formatNow(functionProperties.getQueryStartClock());
2270+
return new ExprDatetimeValue(dt);
22522271
}
22532272

22542273
/**

‎core/src/main/java/org/opensearch/sql/expression/function/FunctionProperties.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.time.Clock;
1010
import java.time.Instant;
1111
import java.time.ZoneId;
12+
import java.time.ZoneOffset;
1213
import lombok.EqualsAndHashCode;
1314
import lombok.Getter;
1415
import org.opensearch.sql.executor.QueryType;
@@ -17,7 +18,7 @@
1718
public class FunctionProperties implements Serializable {
1819

1920
private final Instant nowInstant;
20-
private final ZoneId currentZoneId;
21+
@Getter private final ZoneId currentZoneId;
2122
@Getter private final QueryType queryType;
2223

2324
/** By default, use current time and current timezone. */
@@ -26,7 +27,7 @@ public FunctionProperties() {
2627
}
2728

2829
public FunctionProperties(QueryType queryType) {
29-
this(Instant.now(), ZoneId.systemDefault(), queryType);
30+
this(Instant.now(), ZoneOffset.UTC, queryType);
3031
}
3132

3233
public FunctionProperties(Instant nowInstant, ZoneId currentZoneId) {

‎core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/DatetimeFunction.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
import org.apache.calcite.sql.type.OperandTypes;
1717
import org.apache.calcite.sql.type.SqlReturnTypeInference;
1818
import org.opensearch.sql.calcite.utils.PPLReturnTypes;
19+
import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
1920
import org.opensearch.sql.data.model.ExprStringValue;
2021
import org.opensearch.sql.data.model.ExprValue;
2122
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
23+
import org.opensearch.sql.expression.function.FunctionProperties;
2224
import org.opensearch.sql.expression.function.ImplementorUDF;
2325
import org.opensearch.sql.expression.function.UDFOperandMetadata;
2426

@@ -57,20 +59,23 @@ public static class DatetimeImplementor implements NotNullImplementor {
5759
@Override
5860
public Expression implement(
5961
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
60-
return Expressions.call(DatetimeImplementor.class, "datetime", translatedOperands);
62+
List<Expression> operandsWithProperties =
63+
UserDefinedFunctionUtils.prependFunctionProperties(translatedOperands, translator);
64+
return Expressions.call(DatetimeImplementor.class, "datetime", operandsWithProperties);
6165
}
6266

63-
public static String datetime(String timestamp) {
67+
public static String datetime(FunctionProperties properties, String timestamp) {
6468
ExprValue argTimestampExpr = new ExprStringValue(timestamp);
6569
ExprValue datetimeExpr;
66-
datetimeExpr = DateTimeFunctions.exprDateTimeNoTimezone(argTimestampExpr);
70+
datetimeExpr = DateTimeFunctions.exprDateTimeNoTimezone(properties, argTimestampExpr);
6771
return (String) datetimeExpr.valueForCalcite();
6872
}
6973

70-
public static String datetime(String timestamp, String timezone) {
74+
public static String datetime(
75+
FunctionProperties properties, String timestamp, String timezone) {
7176
ExprValue timestampExpr = new ExprStringValue(timestamp);
7277
ExprValue datetimeExpr =
73-
DateTimeFunctions.exprDateTime(timestampExpr, new ExprStringValue(timezone));
78+
DateTimeFunctions.exprDateTime(properties, timestampExpr, new ExprStringValue(timezone));
7479
return (String) datetimeExpr.valueForCalcite();
7580
}
7681
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/SysdateFunction.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.sql.expression.function.udf.datetime;
77

8+
import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.prependFunctionProperties;
89
import java.time.Clock;
910
import java.time.ZoneOffset;
1011
import java.util.List;
@@ -19,6 +20,7 @@
1920
import org.opensearch.sql.calcite.utils.PPLReturnTypes;
2021
import org.opensearch.sql.data.model.ExprTimestampValue;
2122
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
23+
import org.opensearch.sql.expression.function.FunctionProperties;
2224
import org.opensearch.sql.expression.function.ImplementorUDF;
2325
import org.opensearch.sql.expression.function.UDFOperandMetadata;
2426

@@ -55,16 +57,18 @@ public static class SysdateImplementor implements NotNullImplementor {
5557
@Override
5658
public Expression implement(
5759
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
58-
return Expressions.call(SysdateImplementor.class, "sysdate", translatedOperands);
60+
List<Expression> operandsWithProperties =
61+
prependFunctionProperties(translatedOperands, translator);
62+
return Expressions.call(SysdateImplementor.class, "sysdate", operandsWithProperties);
5963
}
6064

61-
public static String sysdate() {
62-
var localDateTime = DateTimeFunctions.formatNow(Clock.systemDefaultZone(), 0);
65+
public static String sysdate(FunctionProperties properties) {
66+
var localDateTime = DateTimeFunctions.formatNow(properties.getSystemClock(), 0);
6367
return (String) new ExprTimestampValue(localDateTime.toInstant(ZoneOffset.UTC)).valueForCalcite();
6468
}
6569

66-
public static String sysdate(int precision) {
67-
var localDateTime = DateTimeFunctions.formatNow(Clock.systemDefaultZone(), precision);
70+
public static String sysdate(FunctionProperties properties, int precision) {
71+
var localDateTime = DateTimeFunctions.formatNow(properties.getSystemClock(), precision);
6872
return (String) new ExprTimestampValue(localDateTime.toInstant(ZoneOffset.UTC)).valueForCalcite();
6973
}
7074
}

‎core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTest.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111

1212
import java.time.LocalDateTime;
1313
import java.time.ZoneId;
14+
import java.time.ZoneOffset;
1415
import java.time.ZonedDateTime;
1516
import java.time.format.DateTimeFormatter;
16-
import java.util.TimeZone;
1717
import org.junit.jupiter.api.Test;
1818
import org.opensearch.sql.data.model.ExprDatetimeValue;
1919
import org.opensearch.sql.expression.DSL;
@@ -44,21 +44,19 @@ public void positiveField1WrittenField2() {
4444
assertEquals(new ExprDatetimeValue("2008-05-15 14:00:00"), expr.valueOf());
4545
}
4646

47-
// When no timezone argument is passed inside the datetime field, it assumes local time.
47+
// When no timezone argument is passed inside the datetime field, it assumes UTC time.
4848
@Test
4949
public void localDateTimeConversion() {
50-
// needs to work for all time zones because it defaults to local timezone.
5150
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
5251
String dt = "2008-05-15 22:00:00";
5352
String timeZone = "America/Los_Angeles";
54-
LocalDateTime timeConverted = LocalDateTime.parse(dt, formatter);
55-
ZonedDateTime timeZoneLocal =
56-
timeConverted
57-
.atZone(ZoneId.of(TimeZone.getDefault().getID()))
58-
.withZoneSameInstant(ZoneId.of(timeZone));
53+
LocalDateTime utcDatetime = LocalDateTime.parse(dt, formatter);
54+
ZonedDateTime laDatetime =
55+
utcDatetime.atZone(ZoneOffset.UTC).withZoneSameInstant(ZoneId.of(timeZone));
56+
// Convert dt (assumed to be in UTC) to the specified time zone.
5957
FunctionExpression expr = DSL.datetime(DSL.literal(dt), DSL.literal(timeZone));
6058
assertEquals(DATETIME, expr.type());
61-
assertEquals(new ExprDatetimeValue(timeZoneLocal.toLocalDateTime()), expr.valueOf());
59+
assertEquals(new ExprDatetimeValue(laDatetime.toLocalDateTime()), expr.valueOf());
6260
}
6361

6462
@Test

‎core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import java.time.LocalDateTime;
1414
import java.time.LocalTime;
1515
import java.time.ZoneId;
16+
import java.time.ZoneOffset;
1617
import java.time.temporal.IsoFields;
1718
import java.util.stream.Stream;
1819
import org.junit.jupiter.api.Test;
@@ -137,7 +138,7 @@ public void testExtractDatePartWithTimeType() {
137138
public void testExtractWeekPartWithTimeType(String arg) {
138139

139140
// setup default date/time properties for the extract function
140-
ZoneId currentZoneId = ZoneId.systemDefault();
141+
ZoneId currentZoneId = ZoneOffset.UTC;
141142
Instant nowInstant =
142143
LocalDate.parse(arg).atTime(LocalTime.parse(timeInput)).atZone(currentZoneId).toInstant();
143144
FunctionProperties properties = new FunctionProperties(nowInstant, currentZoneId);

‎core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@
1616
import java.time.LocalDate;
1717
import java.time.LocalDateTime;
1818
import java.time.LocalTime;
19+
import java.time.ZoneOffset;
1920
import java.time.ZonedDateTime;
2021
import java.time.temporal.ChronoUnit;
2122
import java.time.temporal.Temporal;
2223
import java.time.temporal.TemporalUnit;
2324
import java.util.List;
24-
import java.util.TimeZone;
2525
import java.util.function.BiFunction;
2626
import java.util.function.Function;
2727
import java.util.function.Supplier;
@@ -133,9 +133,8 @@ void utc_timestamp() {
133133

134134
private static LocalDateTime utcDateTimeNow(FunctionProperties functionProperties) {
135135
ZonedDateTime zonedDateTime =
136-
LocalDateTime.now(functionProperties.getQueryStartClock())
137-
.atZone(TimeZone.getDefault().toZoneId());
138-
return zonedDateTime.withZoneSameInstant(UTC_ZONE_ID).toLocalDateTime();
136+
LocalDateTime.now(functionProperties.getQueryStartClock()).atZone(ZoneOffset.UTC);
137+
return zonedDateTime.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime();
139138
}
140139

141140
/**

0 commit comments

Comments
 (0)