Skip to content

Commit 01c4d7f

Browse files
yuancuyuancu
committed
Add integration tests for time modifiers
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent d0d142f commit 01c4d7f

3 files changed

Lines changed: 282 additions & 3 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java‎

Lines changed: 279 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,19 @@
1515
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
1616

1717
import java.io.IOException;
18+
import java.time.LocalDateTime;
19+
import java.time.ZoneOffset;
20+
import java.time.format.DateTimeFormatter;
21+
import java.time.temporal.ChronoUnit;
1822
import org.json.JSONObject;
1923
import org.junit.jupiter.api.Test;
2024
import org.opensearch.client.Request;
2125
import org.opensearch.client.ResponseException;
26+
import org.opensearch.sql.common.utils.StringUtils;
2227

2328
public class SearchCommandIT extends PPLIntegTestCase {
29+
private static final DateTimeFormatter PPL_TIMESTAMP_FORMATTER =
30+
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
2431

2532
@Override
2633
public void init() throws Exception {
@@ -1025,7 +1032,7 @@ public void testSearchWithChainedRelativeTimeRange() throws IOException {
10251032
JSONObject result1 =
10261033
executeQuery(
10271034
String.format(
1028-
"search source=%s earliest='2025-08-01 03:47:41' latest=+10months@year | fields"
1035+
"search source=%s earliest='2025-08-01 03:47:41' latest=+1months@year | fields"
10291036
+ " @timestamp",
10301037
TEST_INDEX_TIME_DATA));
10311038
verifySchema(result1, schema("@timestamp", "timestamp"));
@@ -1042,4 +1049,275 @@ public void testSearchWithNumericTimeRange() throws IOException {
10421049
verifySchema(result1, schema("@timestamp", "timestamp"));
10431050
verifyDataRows(result1, rows("2025-08-01 03:47:41"));
10441051
}
1052+
1053+
@Test
1054+
public void testSearchTimeModifierWithSnappedWeek() throws IOException {
1055+
// Test whether alignment to weekday works
1056+
1057+
final int docId = 101;
1058+
Request insertRequest =
1059+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1060+
1061+
// Get the current weekday
1062+
LocalDateTime base = LocalDateTime.now(ZoneOffset.UTC);
1063+
// Truncate to last sunday
1064+
LocalDateTime lastSunday =
1065+
base.minusDays((base.getDayOfWeek().getValue() % 7)).truncatedTo(ChronoUnit.DAYS);
1066+
LocalDateTime lastMonday =
1067+
base.minusDays((base.getDayOfWeek().getValue() + 6) % 7).truncatedTo(ChronoUnit.DAYS);
1068+
LocalDateTime lastFriday =
1069+
base.minusDays((base.getDayOfWeek().getValue() + 2) % 7).truncatedTo(ChronoUnit.DAYS);
1070+
1071+
// Insert data
1072+
LocalDateTime insertedSunday = lastSunday.plusMinutes(10);
1073+
insertRequest.setJsonEntity(
1074+
StringUtils.format(
1075+
"{\"value\":100090,\"category\":\"A\",\"@timestamp\":\"%s\"}\n",
1076+
insertedSunday.format(DateTimeFormatter.ISO_DATE_TIME)));
1077+
client().performRequest(insertRequest);
1078+
1079+
JSONObject result1 =
1080+
executeQuery(String.format("source=%s earliest=@w0 latest='@w+1h'", TEST_INDEX_TIME_DATA));
1081+
verifySchema(
1082+
result1,
1083+
schema("timestamp", "timestamp"),
1084+
schema("value", "int"),
1085+
schema("category", "string"),
1086+
schema("@timestamp", "timestamp"));
1087+
verifyDataRows(
1088+
result1, rows(insertedSunday.format(PPL_TIMESTAMP_FORMATTER), "A", 100090, null));
1089+
1090+
// Test last Monday
1091+
LocalDateTime insertedMonday = lastMonday.plusHours(2).plusMinutes(10);
1092+
Request insertRequest2 =
1093+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1094+
insertRequest2.setJsonEntity(
1095+
StringUtils.format(
1096+
"{\"value\":100091,\"category\":\"B\",\"@timestamp\":\"%s\"}\n",
1097+
insertedMonday.format(DateTimeFormatter.ISO_DATE_TIME)));
1098+
client().performRequest(insertRequest2);
1099+
1100+
JSONObject result2 =
1101+
executeQuery(
1102+
String.format(
1103+
"source=%s earliest='@w1+2h' latest='@w1+2h+30min'", TEST_INDEX_TIME_DATA));
1104+
verifySchema(
1105+
result2,
1106+
schema("timestamp", "timestamp"),
1107+
schema("value", "int"),
1108+
schema("category", "string"),
1109+
schema("@timestamp", "timestamp"));
1110+
verifyDataRows(
1111+
result2, rows(insertedMonday.format(PPL_TIMESTAMP_FORMATTER), "B", 100091, null));
1112+
1113+
// Test last Friday
1114+
LocalDateTime insertedFriday = lastFriday.plusMinutes(10);
1115+
Request insertRequest3 =
1116+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1117+
insertRequest3.setJsonEntity(
1118+
StringUtils.format(
1119+
"{\"value\":100092,\"category\":\"C\",\"@timestamp\":\"%s\"}\n",
1120+
insertedFriday.format(DateTimeFormatter.ISO_DATE_TIME)));
1121+
client().performRequest(insertRequest3);
1122+
1123+
JSONObject result3 =
1124+
executeQuery(
1125+
String.format("source=%s earliest=@w5 latest='@w5+30minutes'", TEST_INDEX_TIME_DATA));
1126+
verifySchema(
1127+
result3,
1128+
schema("timestamp", "timestamp"),
1129+
schema("value", "int"),
1130+
schema("category", "string"),
1131+
schema("@timestamp", "timestamp"));
1132+
verifyDataRows(
1133+
result3, rows(insertedFriday.format(PPL_TIMESTAMP_FORMATTER), "C", 100092, null));
1134+
1135+
Request deleteRequest =
1136+
new Request(
1137+
"DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1138+
client().performRequest(deleteRequest);
1139+
}
1140+
1141+
@Test
1142+
public void testSearchWithRelativeTimeModifiers() throws IOException {
1143+
final int docId = 101;
1144+
1145+
LocalDateTime currentTime = LocalDateTime.now(ZoneOffset.UTC);
1146+
LocalDateTime testTime = currentTime.minusMinutes(30).truncatedTo(ChronoUnit.MINUTES);
1147+
1148+
Request insertRequest =
1149+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1150+
insertRequest.setJsonEntity(
1151+
StringUtils.format(
1152+
"{\"value\":200001,\"category\":\"RELATIVE\",\"@timestamp\":\"%s\"}\n",
1153+
testTime.format(DateTimeFormatter.ISO_DATE_TIME)));
1154+
client().performRequest(insertRequest);
1155+
1156+
// Test -1h (1 hour ago) since it's only 30 minutes old
1157+
JSONObject result1 =
1158+
executeQuery(
1159+
String.format(
1160+
"source=%s earliest=-1h | fields @timestamp, value | head 5",
1161+
TEST_INDEX_TIME_DATA));
1162+
verifySchema(result1, schema("@timestamp", "timestamp"), schema("value", "int"));
1163+
verifyDataRows(result1, rows(testTime.format(PPL_TIMESTAMP_FORMATTER), 200001));
1164+
1165+
// Test -30m (30 minutes ago)
1166+
JSONObject result2 =
1167+
executeQuery(
1168+
String.format(
1169+
"source=%s earliest=-50m latest=now | fields @timestamp, value | head 5",
1170+
TEST_INDEX_TIME_DATA));
1171+
verifySchema(result2, schema("@timestamp", "timestamp"), schema("value", "int"));
1172+
verifyDataRows(result2, rows(testTime.format(PPL_TIMESTAMP_FORMATTER), 200001));
1173+
1174+
// Test -7d (7 days ago) - should return no results since our data is recent
1175+
JSONObject result3 =
1176+
executeQuery(
1177+
String.format(
1178+
"source=%s earliest=-7d latest=-6d | fields @timestamp | head 1",
1179+
TEST_INDEX_TIME_DATA));
1180+
verifySchema(result3, schema("@timestamp", "timestamp"));
1181+
verifyNumOfRows(result3, 0);
1182+
1183+
Request deleteRequest =
1184+
new Request(
1185+
"DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1186+
client().performRequest(deleteRequest);
1187+
}
1188+
1189+
@Test
1190+
public void testSearchWithTimeUnitSnapping() throws IOException {
1191+
final int docId = 101;
1192+
1193+
LocalDateTime currentHour = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.HOURS);
1194+
LocalDateTime testTime = currentHour.plusMinutes(15).truncatedTo(ChronoUnit.MINUTES);
1195+
1196+
Request insertRequest =
1197+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1198+
insertRequest.setJsonEntity(
1199+
StringUtils.format(
1200+
"{\"value\":200002,\"category\":\"SNAP\",\"@timestamp\":\"%s\"}\n",
1201+
testTime.format(DateTimeFormatter.ISO_DATE_TIME)));
1202+
client().performRequest(insertRequest);
1203+
1204+
// Test @h (snap to hour)
1205+
JSONObject result1 =
1206+
executeQuery(
1207+
String.format(
1208+
"source=%s earliest=@h latest='@h+1h' | fields @timestamp, value",
1209+
TEST_INDEX_TIME_DATA));
1210+
verifySchema(result1, schema("@timestamp", "timestamp"), schema("value", "int"));
1211+
verifyDataRows(result1, rows(testTime.format(PPL_TIMESTAMP_FORMATTER), 200002));
1212+
1213+
// Test @d (snap to day)
1214+
JSONObject result2 =
1215+
executeQuery(
1216+
String.format(
1217+
"source=%s earliest=@d latest='@d+1d' | fields @timestamp | head 5",
1218+
TEST_INDEX_TIME_DATA));
1219+
verifySchema(result2, schema("@timestamp", "timestamp"));
1220+
verifyDataRows(result2, rows(testTime.format(PPL_TIMESTAMP_FORMATTER)));
1221+
1222+
// Test @M (snap to month)
1223+
JSONObject result3 =
1224+
executeQuery(
1225+
String.format(
1226+
"source=%s earliest=@mon latest='@mon+1mon' | fields @timestamp | head 5",
1227+
TEST_INDEX_TIME_DATA));
1228+
verifySchema(result3, schema("@timestamp", "timestamp"));
1229+
verifyDataRows(result3, rows(testTime.format(PPL_TIMESTAMP_FORMATTER)));
1230+
1231+
Request deleteRequest =
1232+
new Request(
1233+
"DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1234+
client().performRequest(deleteRequest);
1235+
}
1236+
1237+
@Test
1238+
public void testSearchWithQuarterlyModifiers() throws IOException {
1239+
final int docId = 101;
1240+
1241+
LocalDateTime currentQuarter =
1242+
LocalDateTime.now(ZoneOffset.UTC)
1243+
.plusYears(1)
1244+
.withMonth(((LocalDateTime.now(ZoneOffset.UTC).getMonthValue() - 1) / 3) * 3 + 1)
1245+
.withDayOfMonth(1)
1246+
.truncatedTo(ChronoUnit.DAYS);
1247+
LocalDateTime testTime = currentQuarter.plusDays(15);
1248+
1249+
Request insertRequest =
1250+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1251+
insertRequest.setJsonEntity(
1252+
StringUtils.format(
1253+
"{\"value\":200003,\"category\":\"QUARTER\",\"@timestamp\":\"%s\"}\n",
1254+
testTime.format(DateTimeFormatter.ISO_DATE_TIME)));
1255+
client().performRequest(insertRequest);
1256+
1257+
// Test @q (snap to quarter)
1258+
JSONObject result1 =
1259+
executeQuery(
1260+
String.format(
1261+
"source=%s earliest=+1year@q latest='+1year@q+3M' | fields @timestamp, value",
1262+
TEST_INDEX_TIME_DATA));
1263+
verifySchema(result1, schema("@timestamp", "timestamp"), schema("value", "int"));
1264+
verifyDataRows(result1, rows(testTime.format(PPL_TIMESTAMP_FORMATTER), 200003));
1265+
1266+
// Test -2q (2 quarters ago, equivalent to -6M), should return no data
1267+
JSONObject result2 =
1268+
executeQuery(
1269+
String.format(
1270+
"source=%s earliest='+1year-2q' latest='+1year-1q' | fields @timestamp | head 1",
1271+
TEST_INDEX_TIME_DATA));
1272+
verifySchema(result2, schema("@timestamp", "timestamp"));
1273+
verifyNumOfRows(result2, 0);
1274+
1275+
Request deleteRequest =
1276+
new Request(
1277+
"DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1278+
client().performRequest(deleteRequest);
1279+
}
1280+
1281+
@Test
1282+
public void testSearchWithComplexChainedExpressions() throws IOException {
1283+
final int docId = 101;
1284+
1285+
LocalDateTime lastDay =
1286+
LocalDateTime.now(ZoneOffset.UTC).minusDays(1).truncatedTo(ChronoUnit.DAYS);
1287+
LocalDateTime testTime = lastDay.minusHours(2).plusMinutes(10);
1288+
1289+
Request insertRequest =
1290+
new Request("PUT", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1291+
insertRequest.setJsonEntity(
1292+
StringUtils.format(
1293+
"{\"value\":200004,\"category\":\"COMPLEX\",\"@timestamp\":\"%s\"}\n",
1294+
testTime.format(DateTimeFormatter.ISO_DATE_TIME)));
1295+
client().performRequest(insertRequest);
1296+
1297+
// Test -1d@d-2h+10m (1 day ago at day start, minus 2 hours, plus 10 minutes)
1298+
JSONObject result1 =
1299+
executeQuery(
1300+
String.format(
1301+
"source=%s earliest='-1d@d-2h' latest='-1d@d-2h+20m' | fields @timestamp,"
1302+
+ " value",
1303+
TEST_INDEX_TIME_DATA));
1304+
verifySchema(result1, schema("@timestamp", "timestamp"), schema("value", "int"));
1305+
verifyDataRows(result1, rows(testTime.format(PPL_TIMESTAMP_FORMATTER), 200004));
1306+
1307+
// Test -1mon@mon+7d (1 month ago at month start, plus 7 days) - should return no results since
1308+
// our data is recent
1309+
JSONObject result2 =
1310+
executeQuery(
1311+
String.format(
1312+
"source=%s earliest='-1mon@mon+7d' latest='-1mon@mon+8d' | fields @timestamp | head"
1313+
+ " 1",
1314+
TEST_INDEX_TIME_DATA));
1315+
verifySchema(result2, schema("@timestamp", "timestamp"));
1316+
verifyNumOfRows(result2, 0);
1317+
1318+
Request deleteRequest =
1319+
new Request(
1320+
"DELETE", String.format("/%s/_doc/%d?refresh=true", TEST_INDEX_TIME_DATA, docId));
1321+
client().performRequest(deleteRequest);
1322+
}
10451323
}

‎ppl/src/main/antlr/OpenSearchPPLLexer.g4‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -518,11 +518,12 @@ NUMERIC_ID : DEC_DIGIT+ ID_LITERAL;
518518

519519
// LITERALS AND VALUES
520520
//STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING;
521+
fragment WEEK_SNAP_UNIT: 'W' [0-7];
521522
fragment TIME_SNAP_UNIT: 'S' | 'SEC' | 'SECOND'
522523
| 'M' | 'MIN' | 'MINUTE'
523524
| 'H' | 'HR' | 'HOUR' | 'HOURS'
524525
| 'D' | 'DAY'
525-
| 'W' | 'WEEK'
526+
| 'W' | 'WEEK' | WEEK_SNAP_UNIT
526527
| 'MON' | 'MONTH'
527528
| 'Q' | 'QTR' | 'QUARTER'
528529
| 'Y' | 'YR' | 'YEAR';

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,7 @@ timeModifierValue
772772
| INTEGER_LITERAL
773773
| stringLiteral
774774
| TIME_SNAP
775-
| (PLUS | MINUS) SPANLENGTH (TIME_SNAP)?
775+
| (PLUS | MINUS) SPANLENGTH (TIME_SNAP)? ((PLUS | MINUS) SPANLENGTH)*
776776
;
777777

778778
// tables

0 commit comments

Comments
 (0)