Skip to content

Commit dc05942

Browse files
songkant-awssongkant-aws
authored
test(stats): pin output column order with explicit | fields projection (opensearch-project#5459)
Eventstats and streamstats output schema is [original-table-columns, agg-columns]. The original-table-columns slice depends on the catalog's iteration order — V3 uses HashMap bucket order, analytics-engine uses alphabetical. Tests that hardcode one of those orders break on the other route. Add `| fields ...` to every PPL query so the output schema is fixed by the projection list rather than the catalog. Tests now pass on both routes without any product-code change. Signed-off-by: Songkan Tang <songkant@amazon.com>
1 parent 8113f69 commit dc05942

2 files changed

Lines changed: 174 additions & 84 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEventstatsIT.java‎

Lines changed: 55 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public void testEventstats() throws IOException {
3232
executeQuery(
3333
String.format(
3434
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
35-
+ " as max",
35+
+ " as max | fields name, country, state, month, year, age, cnt, avg, min, max",
3636
TEST_INDEX_STATE_COUNTRY));
3737

3838
verifySchemaInOrder(
@@ -62,7 +62,7 @@ public void testEventstatsWithNull() throws IOException {
6262
executeQuery(
6363
String.format(
6464
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
65-
+ " as max",
65+
+ " as max | fields name, country, state, month, year, age, cnt, avg, min, max",
6666
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
6767

6868
verifySchemaInOrder(
@@ -94,7 +94,8 @@ public void testEventstatsBy() throws IOException {
9494
executeQuery(
9595
String.format(
9696
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
97-
+ " as max by country",
97+
+ " as max by country | fields name, country, state, month, year, age, cnt,"
98+
+ " avg, min, max",
9899
TEST_INDEX_STATE_COUNTRY));
99100

100101
verifySchemaInOrder(
@@ -124,7 +125,8 @@ public void testEventstatsByWithNull() throws IOException {
124125
executeQuery(
125126
String.format(
126127
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
127-
+ " as max by country",
128+
+ " as max by country | fields name, country, state, month, year, age, cnt,"
129+
+ " avg, min, max",
128130
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
129131

130132
verifySchemaInOrder(
@@ -153,7 +155,8 @@ public void testEventstatsByWithNull() throws IOException {
153155
executeQuery(
154156
String.format(
155157
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
156-
+ " as max by state",
158+
+ " as max by state | fields name, country, state, month, year, age, cnt, avg,"
159+
+ " min, max",
157160
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
158161
verifyDataRows(
159162
actual,
@@ -171,7 +174,8 @@ public void testEventstatsByWithNullBucket() throws IOException {
171174
executeQuery(
172175
String.format(
173176
"source=%s | eventstats bucket_nullable=false count() as cnt, avg(age) as avg,"
174-
+ " min(age) as min, max(age) as max by country",
177+
+ " min(age) as min, max(age) as max by country | fields name, country, state,"
178+
+ " month, year, age, cnt, avg, min, max",
175179
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
176180

177181
verifyDataRows(
@@ -187,7 +191,8 @@ public void testEventstatsByWithNullBucket() throws IOException {
187191
executeQuery(
188192
String.format(
189193
"source=%s | eventstats bucket_nullable=false count() as cnt, avg(age) as avg,"
190-
+ " min(age) as min, max(age) as max by state",
194+
+ " min(age) as min, max(age) as max by state | fields name, country, state,"
195+
+ " month, year, age, cnt, avg, min, max",
191196
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
192197
verifyDataRows(
193198
actual,
@@ -205,7 +210,8 @@ public void testEventstatsBySpan() throws IOException {
205210
executeQuery(
206211
String.format(
207212
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
208-
+ " as max by span(age, 10) as age_span",
213+
+ " as max by span(age, 10) as age_span | fields name, country, state, month,"
214+
+ " year, age, cnt, avg, min, max",
209215
TEST_INDEX_STATE_COUNTRY));
210216

211217
verifyDataRows(
@@ -222,7 +228,8 @@ public void testEventstatsBySpanWithNull() throws IOException {
222228
executeQuery(
223229
String.format(
224230
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
225-
+ " as max by span(age, 10) as age_span",
231+
+ " as max by span(age, 10) as age_span | fields name, country, state, month,"
232+
+ " year, age, cnt, avg, min, max",
226233
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
227234

228235
verifyDataRows(
@@ -241,7 +248,8 @@ public void testEventstatsByMultiplePartitions1() throws IOException {
241248
executeQuery(
242249
String.format(
243250
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
244-
+ " as max by span(age, 10) as age_span, country",
251+
+ " as max by span(age, 10) as age_span, country | fields name, country, state,"
252+
+ " month, year, age, cnt, avg, min, max",
245253
TEST_INDEX_STATE_COUNTRY));
246254

247255
verifyDataRows(
@@ -258,7 +266,8 @@ public void testEventstatsByMultiplePartitions2() throws IOException {
258266
executeQuery(
259267
String.format(
260268
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
261-
+ " as max by span(age, 10) as age_span, state",
269+
+ " as max by span(age, 10) as age_span, state | fields name, country, state,"
270+
+ " month, year, age, cnt, avg, min, max",
262271
TEST_INDEX_STATE_COUNTRY));
263272

264273
verifyDataRows(
@@ -275,7 +284,8 @@ public void testEventstatsByMultiplePartitionsWithNull1() throws IOException {
275284
executeQuery(
276285
String.format(
277286
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
278-
+ " as max by span(age, 10) as age_span, country",
287+
+ " as max by span(age, 10) as age_span, country | fields name, country, state,"
288+
+ " month, year, age, cnt, avg, min, max",
279289
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
280290

281291
verifyDataRows(
@@ -294,7 +304,8 @@ public void testEventstatsByMultiplePartitionsWithNull2() throws IOException {
294304
executeQuery(
295305
String.format(
296306
"source=%s | eventstats count() as cnt, avg(age) as avg, min(age) as min, max(age)"
297-
+ " as max by span(age, 10) as age_span, state",
307+
+ " as max by span(age, 10) as age_span, state | fields name, country, state,"
308+
+ " month, year, age, cnt, avg, min, max",
298309
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
299310

300311
verifyDataRows(
@@ -328,7 +339,8 @@ public void testMultipleEventstats() throws IOException {
328339
executeQuery(
329340
String.format(
330341
"source=%s | eventstats avg(age) as avg_age by state, country | eventstats"
331-
+ " avg(avg_age) as avg_state_age by country",
342+
+ " avg(avg_age) as avg_state_age by country | fields name, country, state,"
343+
+ " month, year, age, avg_age, avg_state_age",
332344
TEST_INDEX_STATE_COUNTRY));
333345

334346
verifyDataRows(
@@ -345,7 +357,8 @@ public void testMultipleEventstatsWithNull() throws IOException {
345357
executeQuery(
346358
String.format(
347359
"source=%s | eventstats avg(age) as avg_age by state, country | eventstats"
348-
+ " avg(avg_age) as avg_state_age by country",
360+
+ " avg(avg_age) as avg_state_age by country | fields name, country, state,"
361+
+ " month, year, age, avg_age, avg_state_age",
349362
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
350363

351364
verifyDataRows(
@@ -365,7 +378,8 @@ public void testMultipleEventstatsWithNullBucket() throws IOException {
365378
String.format(
366379
"source=%s | eventstats bucket_nullable=false avg(age) as avg_age by state, country"
367380
+ " | eventstats bucket_nullable=false avg(avg_age) as avg_state_age by"
368-
+ " country",
381+
+ " country | fields name, country, state, month, year, age, avg_age,"
382+
+ " avg_state_age",
369383
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
370384

371385
verifyDataRows(
@@ -386,7 +400,9 @@ public void testMultipleEventstatsWithEval() throws IOException {
386400
"source=%s | eventstats avg(age) as avg_age by country, state, name | eval"
387401
+ " avg_age_divide_20 = avg_age - 20 | eventstats avg(avg_age_divide_20) as"
388402
+ " avg_state_age by country, state | where avg_state_age > 0 | eventstats"
389-
+ " count(avg_state_age) as count_country_age_greater_20 by country",
403+
+ " count(avg_state_age) as count_country_age_greater_20 by country | fields"
404+
+ " name, country, state, month, year, age, avg_age, avg_age_divide_20,"
405+
+ " avg_state_age, count_country_age_greater_20",
390406
TEST_INDEX_STATE_COUNTRY));
391407

392408
verifyDataRows(
@@ -422,7 +438,8 @@ public void testEventstatsVariance() throws IOException {
422438
executeQuery(
423439
String.format(
424440
"source=%s | eventstats stddev_pop(age), stddev_samp(age), var_pop(age),"
425-
+ " var_samp(age)",
441+
+ " var_samp(age) | fields name, country, state, month, year, age,"
442+
+ " `stddev_pop(age)`, `stddev_samp(age)`, `var_pop(age)`, `var_samp(age)`",
426443
TEST_INDEX_STATE_COUNTRY));
427444

428445
verifySchemaInOrder(
@@ -492,7 +509,8 @@ public void testEventstatsVarianceWithNull() throws IOException {
492509
executeQuery(
493510
String.format(
494511
"source=%s | eventstats stddev_pop(age), stddev_samp(age), var_pop(age),"
495-
+ " var_samp(age)",
512+
+ " var_samp(age) | fields name, country, state, month, year, age,"
513+
+ " `stddev_pop(age)`, `stddev_samp(age)`, `var_pop(age)`, `var_samp(age)`",
496514
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
497515

498516
verifySchemaInOrder(
@@ -555,7 +573,8 @@ public void testEventstatsVarianceBy() throws IOException {
555573
executeQuery(
556574
String.format(
557575
"source=%s | eventstats stddev_pop(age), stddev_samp(age), var_pop(age),"
558-
+ " var_samp(age) by country",
576+
+ " var_samp(age) by country | fields name, country, state, month, year, age,"
577+
+ " `stddev_pop(age)`, `stddev_samp(age)`, `var_pop(age)`, `var_samp(age)`",
559578
TEST_INDEX_STATE_COUNTRY));
560579

561580
verifyDataRows(
@@ -571,7 +590,8 @@ public void testEventstatsVarianceBySpan() throws IOException {
571590
JSONObject actual =
572591
executeQuery(
573592
String.format(
574-
"source=%s | where country != 'USA' | eventstats stddev_samp(age) by span(age, 10)",
593+
"source=%s | where country != 'USA' | eventstats stddev_samp(age) by span(age, 10)"
594+
+ " | fields name, country, state, month, year, age, `stddev_samp(age)`",
575595
TEST_INDEX_STATE_COUNTRY));
576596

577597
verifyDataRows(
@@ -586,7 +606,8 @@ public void testEventstatsVarianceWithNullBy() throws IOException {
586606
executeQuery(
587607
String.format(
588608
"source=%s | eventstats stddev_pop(age), stddev_samp(age), var_pop(age),"
589-
+ " var_samp(age) by country",
609+
+ " var_samp(age) by country | fields name, country, state, month, year, age,"
610+
+ " `stddev_pop(age)`, `stddev_samp(age)`, `var_pop(age)`, `var_samp(age)`",
590611
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
591612

592613
verifyDataRows(
@@ -634,7 +655,9 @@ public void testEventstatsDistinctCount() throws IOException {
634655
JSONObject actual =
635656
executeQuery(
636657
String.format(
637-
"source=%s | eventstats dc(state) as dc_state", TEST_INDEX_STATE_COUNTRY));
658+
"source=%s | eventstats dc(state) as dc_state | fields name, country, state, month,"
659+
+ " year, age, dc_state",
660+
TEST_INDEX_STATE_COUNTRY));
638661

639662
verifySchemaInOrder(
640663
actual,
@@ -659,7 +682,8 @@ public void testEventstatsDistinctCountByCountry() throws IOException {
659682
JSONObject actual =
660683
executeQuery(
661684
String.format(
662-
"source=%s | eventstats dc(state) as dc_state by country",
685+
"source=%s | eventstats dc(state) as dc_state by country | fields name, country,"
686+
+ " state, month, year, age, dc_state",
663687
TEST_INDEX_STATE_COUNTRY));
664688

665689
verifySchemaInOrder(
@@ -685,7 +709,8 @@ public void testEventstatsDistinctCountFunction() throws IOException {
685709
JSONObject actual =
686710
executeQuery(
687711
String.format(
688-
"source=%s | eventstats distinct_count(country) as dc_country",
712+
"source=%s | eventstats distinct_count(country) as dc_country | fields name,"
713+
+ " country, state, month, year, age, dc_country",
689714
TEST_INDEX_STATE_COUNTRY));
690715

691716
verifySchemaInOrder(
@@ -711,7 +736,8 @@ public void testEventstatsDistinctCountWithNull() throws IOException {
711736
JSONObject actual =
712737
executeQuery(
713738
String.format(
714-
"source=%s | eventstats dc(state) as dc_state",
739+
"source=%s | eventstats dc(state) as dc_state | fields name, country, state, month,"
740+
+ " year, age, dc_state",
715741
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
716742

717743
verifySchemaInOrder(
@@ -739,7 +765,9 @@ public void testEventstatsEarliestAndLatest() throws IOException {
739765
JSONObject actual =
740766
executeQuery(
741767
String.format(
742-
"source=%s | eventstats earliest(message), latest(message) by server",
768+
"source=%s | eventstats earliest(message), latest(message) by server | fields"
769+
+ " created_at, server, `@timestamp`, message, level, `earliest(message)`,"
770+
+ " `latest(message)`",
743771
TEST_INDEX_LOGS));
744772
verifySchema(
745773
actual,

0 commit comments

Comments
 (0)