Skip to content

Commit d94e4de

Browse files
LantaoJinLantaoJin
authored
Add big5 to IT Suite (opensearch-project#3822)
* Add big5 to IT Suite Signed-off-by: Lantao Jin <ltjin@amazon.com> * ignore client error Signed-off-by: Lantao Jin <ltjin@amazon.com> * add tq1 Signed-off-by: Lantao Jin <ltjin@amazon.com> * refactor and ppl rewriting Signed-off-by: Lantao Jin <ltjin@amazon.com> * update the relevance queries Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent c07ed2a commit d94e4de

49 files changed

Lines changed: 722 additions & 11 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.big5;
7+
8+
import org.junit.FixMethodOrder;
9+
import org.junit.runners.MethodSorters;
10+
11+
@FixMethodOrder(MethodSorters.JVM)
12+
public class CalcitePPLBig5IT extends PPLBig5IT {
13+
private boolean initialized = false;
14+
15+
@Override
16+
public void init() throws Exception {
17+
super.init();
18+
enableCalcite();
19+
disallowCalciteFallback();
20+
// warm-up
21+
if (!initialized) {
22+
executeQuery("source=big5 | join on 1=1 big5"); // trigger non-pushdown
23+
initialized = true;
24+
}
25+
}
26+
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.big5;
7+
8+
import java.io.IOException;
9+
import java.util.LinkedHashMap;
10+
import java.util.Locale;
11+
import java.util.Map;
12+
import org.junit.AfterClass;
13+
import org.junit.FixMethodOrder;
14+
import org.junit.Ignore;
15+
import org.junit.Test;
16+
import org.junit.runners.MethodSorters;
17+
import org.opensearch.sql.ppl.PPLIntegTestCase;
18+
19+
@FixMethodOrder(MethodSorters.JVM)
20+
public class PPLBig5IT extends PPLIntegTestCase {
21+
private boolean initialized = false;
22+
private static final Map<String, Long> summary = new LinkedHashMap<>();
23+
24+
@Override
25+
public void init() throws Exception {
26+
super.init();
27+
loadIndex(Index.BIG5);
28+
disableCalcite();
29+
// warm-up
30+
if (!initialized) {
31+
executeQuery("source=big5 | eval a = 1"); // trigger non-pushdown
32+
initialized = true;
33+
}
34+
}
35+
36+
@AfterClass
37+
public static void reset() throws IOException {
38+
long total = 0;
39+
for (long duration : summary.values()) {
40+
total += duration;
41+
}
42+
System.out.println("Summary:");
43+
for (Map.Entry<String, Long> entry : summary.entrySet()) {
44+
System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue());
45+
}
46+
System.out.printf(
47+
Locale.ENGLISH,
48+
"Total %d queries succeed. Average duration: %d ms%n",
49+
summary.size(),
50+
total / summary.size());
51+
System.out.println();
52+
summary.clear();
53+
}
54+
55+
protected void timing(String query, String ppl) throws IOException {
56+
long start = System.currentTimeMillis();
57+
executeQuery(ppl);
58+
long duration = System.currentTimeMillis() - start;
59+
summary.put(query, duration);
60+
}
61+
62+
@Test
63+
public void asc_sort_timestamp() throws IOException {
64+
String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp.ppl"));
65+
timing("asc_sort_timestamp", ppl);
66+
}
67+
68+
@Test
69+
public void asc_sort_timestamp_can_match_shortcut() throws IOException {
70+
String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_can_match_shortcut.ppl"));
71+
timing("asc_sort_timestamp_can_match_shortcut", ppl);
72+
}
73+
74+
@Test
75+
public void asc_sort_timestamp_no_can_match_shortcut() throws IOException {
76+
String ppl =
77+
sanitize(loadFromFile("big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl"));
78+
timing("asc_sort_timestamp_no_can_match_shortcut", ppl);
79+
}
80+
81+
@Test
82+
public void asc_sort_with_after_timestamp() throws IOException {
83+
String ppl = sanitize(loadFromFile("big5/queries/asc_sort_with_after_timestamp.ppl"));
84+
timing("asc_sort_with_after_timestamp", ppl);
85+
}
86+
87+
@Test
88+
public void composite_date_histogram_daily() throws IOException {
89+
String ppl = sanitize(loadFromFile("big5/queries/composite_date_histogram_daily.ppl"));
90+
timing("composite_date_histogram_daily", ppl);
91+
}
92+
93+
@Test
94+
public void composite_terms_keyword() throws IOException {
95+
String ppl = sanitize(loadFromFile("big5/queries/composite_terms_keyword.ppl"));
96+
timing("composite_terms_keyword", ppl);
97+
}
98+
99+
@Test
100+
public void composite_terms() throws IOException {
101+
String ppl = sanitize(loadFromFile("big5/queries/composite_terms.ppl"));
102+
timing("composite_terms", ppl);
103+
}
104+
105+
@Test
106+
public void date_histogram_hourly_agg() throws IOException {
107+
String ppl = sanitize(loadFromFile("big5/queries/date_histogram_hourly_agg.ppl"));
108+
timing("date_histogram_hourly_agg", ppl);
109+
}
110+
111+
@Test
112+
public void date_histogram_minute_agg() throws IOException {
113+
String ppl = sanitize(loadFromFile("big5/queries/date_histogram_minute_agg.ppl"));
114+
timing("date_histogram_minute_agg", ppl);
115+
}
116+
117+
@Test
118+
public void test_default() throws IOException {
119+
String ppl = sanitize(loadFromFile("big5/queries/default.ppl"));
120+
timing("default", ppl);
121+
}
122+
123+
@Test
124+
public void desc_sort_timestamp() throws IOException {
125+
String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp.ppl"));
126+
timing("desc_sort_timestamp", ppl);
127+
}
128+
129+
@Test
130+
public void desc_sort_timestamp_can_match_shortcut() throws IOException {
131+
String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_can_match_shortcut.ppl"));
132+
timing("desc_sort_timestamp_can_match_shortcut", ppl);
133+
}
134+
135+
@Test
136+
public void desc_sort_timestamp_no_can_match_shortcut() throws IOException {
137+
String ppl =
138+
sanitize(loadFromFile("big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl"));
139+
timing("desc_sort_timestamp_no_can_match_shortcut", ppl);
140+
}
141+
142+
@Test
143+
public void desc_sort_with_after_timestamp() throws IOException {
144+
String ppl = sanitize(loadFromFile("big5/queries/desc_sort_with_after_timestamp.ppl"));
145+
timing("desc_sort_with_after_timestamp", ppl);
146+
}
147+
148+
@Test
149+
public void keyword_in_range() throws IOException {
150+
String ppl = sanitize(loadFromFile("big5/queries/keyword_in_range.ppl"));
151+
timing("keyword_in_range", ppl);
152+
}
153+
154+
@Test
155+
public void keyword_terms() throws IOException {
156+
String ppl = sanitize(loadFromFile("big5/queries/keyword_terms.ppl"));
157+
timing("keyword_terms", ppl);
158+
}
159+
160+
@Test
161+
public void keyword_terms_low_cardinality() throws IOException {
162+
String ppl = sanitize(loadFromFile("big5/queries/keyword_terms_low_cardinality.ppl"));
163+
timing("keyword_terms_low_cardinality", ppl);
164+
}
165+
166+
@Test
167+
public void multi_terms_keyword() throws IOException {
168+
String ppl = sanitize(loadFromFile("big5/queries/multi_terms_keyword.ppl"));
169+
timing("multi_terms_keyword", ppl);
170+
}
171+
172+
@Test
173+
public void query_string_on_message() throws IOException {
174+
String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message.ppl"));
175+
timing("query_string_on_message", ppl);
176+
}
177+
178+
@Test
179+
public void query_string_on_message_filtered() throws IOException {
180+
String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered.ppl"));
181+
timing("query_string_on_message_filtered", ppl);
182+
}
183+
184+
@Test
185+
public void query_string_on_message_filtered_sorted_num() throws IOException {
186+
String ppl =
187+
sanitize(loadFromFile("big5/queries/query_string_on_message_filtered_sorted_num.ppl"));
188+
timing("query_string_on_message_filtered_sorted_num", ppl);
189+
}
190+
191+
@Test
192+
public void range() throws IOException {
193+
String ppl = sanitize(loadFromFile("big5/queries/range.ppl"));
194+
timing("range", ppl);
195+
}
196+
197+
@Ignore("Failed to parse request payload")
198+
public void range_auto_date_histo() throws IOException {
199+
String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo.ppl"));
200+
timing("range_auto_date_histo", ppl);
201+
}
202+
203+
@Ignore("Failed to parse request payload")
204+
public void range_auto_date_histo_with_metrics() throws IOException {
205+
String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo_with_metrics.ppl"));
206+
timing("range_auto_date_histo_with_metrics", ppl);
207+
}
208+
209+
@Test
210+
public void range_numeric() throws IOException {
211+
String ppl = sanitize(loadFromFile("big5/queries/range_numeric.ppl"));
212+
timing("range_numeric", ppl);
213+
}
214+
215+
@Test
216+
public void range_field_conjunction_big_range_big_term_query() throws IOException {
217+
String ppl =
218+
sanitize(loadFromFile("big5/queries/range_field_conjunction_big_range_big_term_query.ppl"));
219+
timing("range_field_conjunction_big_range_big_term_query", ppl);
220+
}
221+
222+
@Test
223+
public void range_field_conjunction_small_range_big_term_query() throws IOException {
224+
String ppl =
225+
sanitize(
226+
loadFromFile("big5/queries/range_field_conjunction_small_range_big_term_query.ppl"));
227+
timing("range_field_conjunction_small_range_big_term_query", ppl);
228+
}
229+
230+
@Test
231+
public void range_field_conjunction_small_range_small_term_query() throws IOException {
232+
String ppl =
233+
sanitize(
234+
loadFromFile("big5/queries/range_field_conjunction_small_range_small_term_query.ppl"));
235+
timing("range_field_conjunction_small_range_small_term_query", ppl);
236+
}
237+
238+
@Test
239+
public void range_field_disjunction_big_range_small_term_query() throws IOException {
240+
String ppl =
241+
sanitize(
242+
loadFromFile("big5/queries/range_field_disjunction_big_range_small_term_query.ppl"));
243+
timing("range_field_disjunction_big_range_small_term_query", ppl);
244+
}
245+
246+
@Test
247+
public void range_with_asc_sort() throws IOException {
248+
String ppl = sanitize(loadFromFile("big5/queries/range_with_asc_sort.ppl"));
249+
timing("range_with_asc_sort", ppl);
250+
}
251+
252+
@Test
253+
public void range_with_desc_sort() throws IOException {
254+
String ppl = sanitize(loadFromFile("big5/queries/range_with_desc_sort.ppl"));
255+
timing("range_with_desc_sort", ppl);
256+
}
257+
258+
@Test
259+
public void scroll() throws IOException {
260+
String ppl = sanitize(loadFromFile("big5/queries/scroll.ppl"));
261+
timing("scroll", ppl);
262+
}
263+
264+
@Test
265+
public void sort_keyword_can_match_shortcut() throws IOException {
266+
String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_can_match_shortcut.ppl"));
267+
timing("sort_keyword_can_match_shortcut", ppl);
268+
}
269+
270+
@Test
271+
public void sort_keyword_no_can_match_shortcut() throws IOException {
272+
String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_no_can_match_shortcut.ppl"));
273+
timing("sort_keyword_no_can_match_shortcut", ppl);
274+
}
275+
276+
@Test
277+
public void sort_numeric_asc() throws IOException {
278+
String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc.ppl"));
279+
timing("sort_numeric_asc", ppl);
280+
}
281+
282+
@Test
283+
public void sort_numeric_asc_with_match() throws IOException {
284+
String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc_with_match.ppl"));
285+
timing("sort_numeric_asc_with_match", ppl);
286+
}
287+
288+
@Test
289+
public void sort_numeric_desc() throws IOException {
290+
String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc.ppl"));
291+
timing("sort_numeric_desc", ppl);
292+
}
293+
294+
@Test
295+
public void sort_numeric_desc_with_match() throws IOException {
296+
String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc_with_match.ppl"));
297+
timing("sort_numeric_desc_with_match", ppl);
298+
}
299+
300+
@Test
301+
public void term() throws IOException {
302+
String ppl = sanitize(loadFromFile("big5/queries/term.ppl"));
303+
timing("term", ppl);
304+
}
305+
306+
@Test
307+
public void terms_significant_1() throws IOException {
308+
String ppl = sanitize(loadFromFile("big5/queries/terms_significant_1.ppl"));
309+
timing("terms_significant_1", ppl);
310+
}
311+
312+
@Test
313+
public void terms_significant_2() throws IOException {
314+
String ppl = sanitize(loadFromFile("big5/queries/terms_significant_2.ppl"));
315+
timing("terms_significant_2", ppl);
316+
}
317+
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -454,15 +454,4 @@ public void testQ22() throws IOException {
454454
rows("30", 1, 7638.57),
455455
rows("31", 1, 9331.13));
456456
}
457-
458-
/**
459-
* Sanitizes the PPL query by removing block comments and replacing new lines with spaces.
460-
*
461-
* @param ppl the PPL query string
462-
* @return the sanitized PPL query string
463-
*/
464-
private static String sanitize(String ppl) {
465-
String withoutComments = ppl.replaceAll("(?s)/\\*.*?\\*/", "");
466-
return withoutComments.replaceAll("\\r\\n", " ").replaceAll("\\n", " ").trim();
467-
}
468457
}

‎integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import static org.opensearch.sql.legacy.TestUtils.getArrayIndexMapping;
1313
import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping;
1414
import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping;
15+
import static org.opensearch.sql.legacy.TestUtils.getBig5MappingFile;
1516
import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping;
1617
import static org.opensearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping;
1718
import static org.opensearch.sql.legacy.TestUtils.getDateIndexMapping;
@@ -901,6 +902,11 @@ public enum Index {
901902
"tpch",
902903
getTpchMappingFile("customer_index_mapping.json"),
903904
"src/test/resources/tpch/data/customer.json"),
905+
BIG5(
906+
"big5",
907+
"big5",
908+
getBig5MappingFile("big5_index_mapping.json"),
909+
"src/test/resources/big5/data/big5.json"),
904910
ARRAY(
905911
TestsConstants.TEST_INDEX_ARRAY,
906912
"array",

‎integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,4 +477,12 @@ public static String getTpchMappingFile(String fileName) {
477477
return null;
478478
}
479479
}
480+
481+
public static String getBig5MappingFile(String fileName) {
482+
try {
483+
return TestUtils.fileToString("src/test/resources/big5/mappings/" + fileName, false);
484+
} catch (IOException e) {
485+
return null;
486+
}
487+
}
480488
}

0 commit comments

Comments
 (0)