Skip to content

Commit 8e80e6c

Browse files
yuancuyuancu
committed
Support millisecond span
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent c6a3a70 commit 8e80e6c

9 files changed

Lines changed: 142 additions & 9 deletions

File tree

‎core/src/main/java/org/opensearch/sql/ast/expression/IntervalUnit.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
@RequiredArgsConstructor
1515
public enum IntervalUnit {
1616
UNKNOWN,
17-
1817
MICROSECOND,
18+
MILLISECOND,
1919
SECOND,
2020
MINUTE,
2121
HOUR,

‎core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
public enum SpanUnit {
1616
UNKNOWN("unknown"),
1717
NONE(""),
18+
MICROSECOND("us"),
19+
US("us"),
1820
MILLISECOND("ms"),
1921
MS("ms"),
2022
SECONDS("s"),

‎core/src/main/java/org/opensearch/sql/ast/tree/Timechart.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import static org.opensearch.sql.ast.dsl.AstDSL.eval;
1111
import static org.opensearch.sql.ast.dsl.AstDSL.function;
1212
import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral;
13-
import static org.opensearch.sql.ast.expression.IntervalUnit.SECOND;
13+
import static org.opensearch.sql.ast.expression.IntervalUnit.MILLISECOND;
1414
import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.sum;
1515
import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampadd;
1616
import static org.opensearch.sql.ast.tree.Timechart.PerFunctionRateExprBuilder.timestampdiff;
@@ -112,11 +112,13 @@ private UnresolvedPlan transformPerFunction() {
112112
Span span = (Span) this.binExpression;
113113
Field spanStartTime = AstDSL.field(IMPLICIT_FIELD_TIMESTAMP);
114114
Function spanEndTime = timestampadd(span.getUnit(), span.getValue(), spanStartTime);
115-
Function spanSeconds = timestampdiff(SECOND, spanStartTime, spanEndTime);
116-
115+
Function spanMillis = timestampdiff(MILLISECOND, spanStartTime, spanEndTime);
116+
final int SECOND_IN_MILLISECOND = 1000;
117117
return eval(
118118
timechart(AstDSL.alias(perFunc.aggName, sum(perFunc.aggArg))),
119-
let(perFunc.aggName).multiply(perFunc.seconds).dividedBy(spanSeconds));
119+
let(perFunc.aggName)
120+
.multiply(perFunc.seconds * SECOND_IN_MILLISECOND)
121+
.dividedBy(spanMillis));
120122
}
121123

122124
private Timechart timechart(UnresolvedExpression newAggregateFunction) {

‎core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,38 +54,59 @@ public RelDataType commonType(RexNode... nodes) {
5454
public SqlIntervalQualifier createIntervalUntil(SpanUnit unit) {
5555
TimeUnit timeUnit;
5656
switch (unit) {
57+
case MICROSECOND:
58+
case US:
59+
timeUnit = TimeUnit.MICROSECOND;
60+
break;
5761
case MILLISECOND:
5862
case MS:
5963
timeUnit = TimeUnit.MILLISECOND;
6064
break;
65+
case SECONDS:
6166
case SECOND:
67+
case SECS:
68+
case SEC:
6269
case S:
6370
timeUnit = TimeUnit.SECOND;
6471
break;
72+
case MINUTES:
6573
case MINUTE:
74+
case MINS:
75+
case MIN:
6676
case m:
6777
timeUnit = TimeUnit.MINUTE;
6878
break;
79+
case HOURS:
6980
case HOUR:
81+
case HRS:
82+
case HR:
7083
case H:
7184
timeUnit = TimeUnit.HOUR;
7285
break;
86+
case DAYS:
7387
case DAY:
7488
case D:
7589
timeUnit = TimeUnit.DAY;
7690
break;
91+
case WEEKS:
7792
case WEEK:
7893
case W:
7994
timeUnit = TimeUnit.WEEK;
8095
break;
96+
case MONTHS:
8197
case MONTH:
98+
case MON:
8299
case M:
83100
timeUnit = TimeUnit.MONTH;
84101
break;
102+
case QUARTERS:
85103
case QUARTER:
104+
case QTRS:
105+
case QTR:
86106
case Q:
87107
timeUnit = TimeUnit.QUARTER;
88108
break;
109+
case YEARS:
89110
case YEAR:
90111
case Y:
91112
timeUnit = TimeUnit.YEAR;

‎core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ public interface PlanUtils {
6868

6969
static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) {
7070
return switch (unit) {
71-
case MICROSECOND -> SpanUnit.MILLISECOND;
71+
case MICROSECOND -> SpanUnit.MICROSECOND;
72+
case MILLISECOND -> SpanUnit.MILLISECOND;
7273
case SECOND -> SpanUnit.SECOND;
7374
case MINUTE -> SpanUnit.MINUTE;
7475
case HOUR -> SpanUnit.HOUR;
@@ -84,9 +85,12 @@ static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) {
8485

8586
static IntervalUnit spanUnitToIntervalUnit(SpanUnit unit) {
8687
switch (unit) {
88+
case MICROSECOND:
89+
case US:
90+
return IntervalUnit.MICROSECOND;
8791
case MILLISECOND:
8892
case MS:
89-
return IntervalUnit.MICROSECOND;
93+
return IntervalUnit.MILLISECOND;
9094
case SECOND:
9195
case SECONDS:
9296
case SEC:

‎core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import static java.time.temporal.ChronoUnit.DAYS;
99
import static java.time.temporal.ChronoUnit.HOURS;
1010
import static java.time.temporal.ChronoUnit.MICROS;
11+
import static java.time.temporal.ChronoUnit.MILLIS;
1112
import static java.time.temporal.ChronoUnit.MINUTES;
1213
import static java.time.temporal.ChronoUnit.MONTHS;
1314
import static java.time.temporal.ChronoUnit.SECONDS;
@@ -1896,6 +1897,9 @@ public static ExprValue exprTimestampAdd(
18961897
case "MICROSECOND":
18971898
temporalUnit = MICROS;
18981899
break;
1900+
case "MILLISECOND":
1901+
temporalUnit = MILLIS;
1902+
break;
18991903
case "SECOND":
19001904
temporalUnit = SECONDS;
19011905
break;
@@ -1935,10 +1939,13 @@ public static ExprValue exprTimestampAddForTimeType(
19351939

19361940
private ExprValue getTimeDifference(String part, LocalDateTime startTime, LocalDateTime endTime) {
19371941
long returnVal;
1938-
switch (part) {
1942+
switch (part.toUpperCase(Locale.ROOT)) {
19391943
case "MICROSECOND":
19401944
returnVal = MICROS.between(startTime, endTime);
19411945
break;
1946+
case "MILLISECOND":
1947+
returnVal = MILLIS.between(startTime, endTime);
1948+
break;
19421949
case "SECOND":
19431950
returnVal = SECONDS.between(startTime, endTime);
19441951
break;

‎integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4550.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
setup:
2+
- do:
3+
indices.create:
4+
index: test_data_2023
5+
body:
6+
mappings:
7+
properties:
8+
"@timestamp":
9+
type: date
10+
"packets":
11+
type: integer
12+
- do:
13+
bulk:
14+
index: test_data_2023
15+
refresh: true
16+
body:
17+
- '{"index":{}}'
18+
- '{"@timestamp":"2023-10-08T10:00:00.000Z","packets":10}'
19+
- '{"index":{}}'
20+
- '{"@timestamp":"2023-10-08T10:00:00.500Z","packets":15}'
21+
- '{"index":{}}'
22+
- '{"@timestamp":"2023-10-08T10:00:01.000Z","packets":20}'
23+
- '{"index":{}}'
24+
- '{"@timestamp":"2023-10-08T10:00:01.500Z","packets":25}'
25+
- '{"index":{}}'
26+
- '{"@timestamp":"2023-10-08T10:00:02.000Z","packets":30}'
27+
28+
---
29+
"timechart with millisecond span":
30+
- skip:
31+
features:
32+
- headers
33+
- allowed_warnings
34+
- do:
35+
headers:
36+
Content-Type: 'application/json'
37+
ppl:
38+
body:
39+
query: source=test_data_2023 | timechart span=500ms count()
40+
41+
- match: { total: 5 }
42+
- match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
43+
- match: {"datarows": [["2023-10-08 10:00:00", 1], ["2023-10-08 10:00:00.5", 1], ["2023-10-08 10:00:01", 1], ["2023-10-08 10:00:01.5", 1], ["2023-10-08 10:00:02", 1]]}
44+
45+
---
46+
"timechart with millisecond span and per_second function":
47+
- skip:
48+
features:
49+
- headers
50+
- allowed_warnings
51+
- do:
52+
headers:
53+
Content-Type: 'application/json'
54+
ppl:
55+
body:
56+
query: source=test_data_2023 | timechart span=1000ms per_second(packets)
57+
58+
- match: { total: 3 }
59+
- match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "per_second(packets)", "type": "double" }] }
60+
- match: {"datarows": [["2023-10-08 10:00:00", 25.0], ["2023-10-08 10:00:01", 45.0], ["2023-10-08 10:00:02", 30.0]]}
61+
62+
---
63+
"timechart with milliseconds":
64+
- skip:
65+
features:
66+
- headers
67+
- allowed_warnings
68+
- do:
69+
headers:
70+
Content-Type: 'application/json'
71+
ppl:
72+
body:
73+
query: source=test_data_2023 | timechart span=250milliseconds count()
74+
75+
- match: { total: 5 }
76+
- match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
77+
78+
---
79+
"timechart with second span for comparison":
80+
- skip:
81+
features:
82+
- headers
83+
- allowed_warnings
84+
- do:
85+
headers:
86+
Content-Type: 'application/json'
87+
ppl:
88+
body:
89+
query: source=test_data_2023 | timechart span=1s count()
90+
91+
- match: { total: 3 }
92+
- match: { "schema": [ { "name": "@timestamp", "type": "timestamp" }, { "name": "count", "type": "bigint" }] }
93+
- match: {"datarows": [["2023-10-08 10:00:00", 2], ["2023-10-08 10:00:01", 2], ["2023-10-08 10:00:02", 1]]}

‎ppl/src/main/antlr/OpenSearchPPLLexer.g4‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ HOUR_MINUTE: 'HOUR_MINUTE';
169169
HOUR_OF_DAY: 'HOUR_OF_DAY';
170170
HOUR_SECOND: 'HOUR_SECOND';
171171
INTERVAL: 'INTERVAL';
172+
MILLISECOND: 'MILLISECOND';
172173
MICROSECOND: 'MICROSECOND';
173174
MINUTE: 'MINUTE';
174175
MINUTE_MICROSECOND: 'MINUTE_MICROSECOND';
@@ -502,7 +503,8 @@ ALIGNTIME: 'ALIGNTIME';
502503
PERCENTILE_SHORTCUT: PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL);
503504

504505
SPANLENGTH: [0-9]+ (
505-
'US'|'MS'|'CS'|'DS'
506+
'US' |'CS'|'DS'
507+
|'MS'|'MILLISECOND'|'MILLISECONDS'
506508
|'S'|'SEC'|'SECS'|'SECOND'|'SECONDS'
507509
|'MIN'|'MINS'|'MINUTE'|'MINUTES'
508510
|'H'|'HR'|'HRS'|'HOUR'|'HOURS'

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1154,6 +1154,7 @@ extractFunctionCall
11541154

11551155
simpleDateTimePart
11561156
: MICROSECOND
1157+
| MILLISECOND
11571158
| SECOND
11581159
| MINUTE
11591160
| HOUR
@@ -1332,6 +1333,7 @@ timestampLiteral
13321333

13331334
intervalUnit
13341335
: MICROSECOND
1336+
| MILLISECOND
13351337
| SECOND
13361338
| MINUTE
13371339
| HOUR

0 commit comments

Comments
 (0)