Skip to content

Commit 3b30525

Browse files
ahkcsahkcs
authored
spath: parquet-backed test indices for analytics-engine route (#5441)
`CalcitePPLSpathCommandIT.init()` was creating its four test indices by raw `PUT /<idx>/_doc/N` requests, which auto-creates the index via the default Lucene path. The analytics-engine compatibility run (`-Dtests.analytics.parquet_indices=true`) injects the parquet/composite settings *inside* `TestUtils.createIndexByRestClient`, so the raw-PUT indices were Lucene-only and DataFusion failed with `UnsupportedOperationException: acquireReader is not supported in EngineBackedIndexer` for every test on the analytics-engine route. Fix: create the empty index up-front via `createIndexByRestClient(..., null)` so the toggle has a chance to inject parquet settings, then let the subsequent doc PUTs populate it via dynamic mapping. No mapping is declared — DataFusion is fine with dynamic mapping on a parquet-backed composite index. Same pattern as `CalciteEvalCommandIT` and `CalciteFieldFormatCommandIT`. No change for the v2 / Calcite path (the helper is a no-op when the parquet toggle isn't set). ## Pass rate Pairs with opensearch-project/OpenSearch#21664. Both PRs are required to move the analytics-engine route off 0 / 16. | IT | Route | Before | After | |---|---|---|---| | `CalcitePPLSpathCommandIT` | analytics-engine (`-Dtests.analytics.force_routing=true -Dtests.analytics.parquet_indices=true`) | 0 / 16 | **16 / 16** | | `CalcitePPLSpathCommandIT` | default v2 / Calcite (no flags) | 16 / 16 | 16 / 16 (no regression) | Signed-off-by: Kai Huang <ahkcs@amazon.com>
1 parent 743d51f commit 3b30525

1 file changed

Lines changed: 56 additions & 32 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSpathCommandIT.java‎

Lines changed: 56 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,17 @@
1515
import org.json.JSONObject;
1616
import org.junit.jupiter.api.Test;
1717
import org.opensearch.client.Request;
18+
import org.opensearch.sql.legacy.TestUtils;
1819
import org.opensearch.sql.ppl.PPLIntegTestCase;
1920

2021
public class CalcitePPLSpathCommandIT extends PPLIntegTestCase {
22+
// Pre-create each test index through TestUtils.createIndexByRestClient so the
23+
// analytics-engine compatibility run (tests.analytics.parquet_indices=true)
24+
// provisions them as parquet-backed composite stores. Raw `PUT /<idx>/_doc/N`
25+
// bypasses the helper, yielding a Lucene-only index that DataFusion cannot
26+
// acquireReader on (`UnsupportedOperationException: acquireReader is not
27+
// supported in EngineBackedIndexer`). Mapping passed as null — dynamic
28+
// mapping infers the doc fields from the subsequent PUTs.
2129
@Override
2230
public void init() throws Exception {
2331
super.init();
@@ -26,48 +34,64 @@ public void init() throws Exception {
2634
loadIndex(Index.BANK);
2735

2836
// Simple JSON docs for path-based extraction
29-
Request request1 = new Request("PUT", "/test_spath/_doc/1?refresh=true");
30-
request1.setJsonEntity("{\"doc\": \"{\\\"n\\\": 1}\"}");
31-
client().performRequest(request1);
37+
if (!TestUtils.isIndexExist(client(), "test_spath")) {
38+
TestUtils.createIndexByRestClient(client(), "test_spath", null);
3239

33-
Request request2 = new Request("PUT", "/test_spath/_doc/2?refresh=true");
34-
request2.setJsonEntity("{\"doc\": \"{\\\"n\\\": 2}\"}");
35-
client().performRequest(request2);
40+
Request request1 = new Request("PUT", "/test_spath/_doc/1?refresh=true");
41+
request1.setJsonEntity("{\"doc\": \"{\\\"n\\\": 1}\"}");
42+
client().performRequest(request1);
3643

37-
Request request3 = new Request("PUT", "/test_spath/_doc/3?refresh=true");
38-
request3.setJsonEntity("{\"doc\": \"{\\\"n\\\": 3}\"}");
39-
client().performRequest(request3);
44+
Request request2 = new Request("PUT", "/test_spath/_doc/2?refresh=true");
45+
request2.setJsonEntity("{\"doc\": \"{\\\"n\\\": 2}\"}");
46+
client().performRequest(request2);
47+
48+
Request request3 = new Request("PUT", "/test_spath/_doc/3?refresh=true");
49+
request3.setJsonEntity("{\"doc\": \"{\\\"n\\\": 3}\"}");
50+
client().performRequest(request3);
51+
}
4052

4153
// Auto-extract mode: flatten rules and edge cases (empty, malformed)
42-
Request autoExtractDoc = new Request("PUT", "/test_spath_auto/_doc/1?refresh=true");
43-
autoExtractDoc.setJsonEntity(
44-
"{\"nested_doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"John\\\"}}\","
45-
+ " \"array_doc\": \"{\\\"tags\\\":[\\\"java\\\",\\\"sql\\\"]}\","
46-
+ " \"merge_doc\": \"{\\\"a\\\":{\\\"b\\\":1},\\\"a.b\\\":2}\","
47-
+ " \"stringify_doc\": \"{\\\"n\\\":30,\\\"b\\\":true,\\\"x\\\":null}\","
48-
+ " \"empty_doc\": \"{}\","
49-
+ " \"malformed_doc\": \"{\\\"user\\\":{\\\"name\\\":\"}");
50-
client().performRequest(autoExtractDoc);
54+
if (!TestUtils.isIndexExist(client(), "test_spath_auto")) {
55+
TestUtils.createIndexByRestClient(client(), "test_spath_auto", null);
56+
57+
Request autoExtractDoc = new Request("PUT", "/test_spath_auto/_doc/1?refresh=true");
58+
autoExtractDoc.setJsonEntity(
59+
"{\"nested_doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"John\\\"}}\","
60+
+ " \"array_doc\": \"{\\\"tags\\\":[\\\"java\\\",\\\"sql\\\"]}\","
61+
+ " \"merge_doc\": \"{\\\"a\\\":{\\\"b\\\":1},\\\"a.b\\\":2}\","
62+
+ " \"stringify_doc\": \"{\\\"n\\\":30,\\\"b\\\":true,\\\"x\\\":null}\","
63+
+ " \"empty_doc\": \"{}\","
64+
+ " \"malformed_doc\": \"{\\\"user\\\":{\\\"name\\\":\"}");
65+
client().performRequest(autoExtractDoc);
66+
}
5167

5268
// Auto-extract mode: 2-doc index for spath + command (eval/where/stats/sort) tests
53-
Request cmdDoc1 = new Request("PUT", "/test_spath_cmd/_doc/1?refresh=true");
54-
cmdDoc1.setJsonEntity(
55-
"{\"doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"John\\\",\\\"age\\\":30}}\"}");
56-
client().performRequest(cmdDoc1);
69+
if (!TestUtils.isIndexExist(client(), "test_spath_cmd")) {
70+
TestUtils.createIndexByRestClient(client(), "test_spath_cmd", null);
71+
72+
Request cmdDoc1 = new Request("PUT", "/test_spath_cmd/_doc/1?refresh=true");
73+
cmdDoc1.setJsonEntity(
74+
"{\"doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"John\\\",\\\"age\\\":30}}\"}");
75+
client().performRequest(cmdDoc1);
5776

58-
Request cmdDoc2 = new Request("PUT", "/test_spath_cmd/_doc/2?refresh=true");
59-
cmdDoc2.setJsonEntity(
60-
"{\"doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"Alice\\\",\\\"age\\\":25}}\"}");
61-
client().performRequest(cmdDoc2);
77+
Request cmdDoc2 = new Request("PUT", "/test_spath_cmd/_doc/2?refresh=true");
78+
cmdDoc2.setJsonEntity(
79+
"{\"doc\": \"{\\\"user\\\":{\\\"name\\\":\\\"Alice\\\",\\\"age\\\":25}}\"}");
80+
client().performRequest(cmdDoc2);
81+
}
6282

6383
// Auto-extract mode: null input handling (doc 1 establishes mapping, doc 2 has null)
64-
Request nullDoc1 = new Request("PUT", "/test_spath_null/_doc/1?refresh=true");
65-
nullDoc1.setJsonEntity("{\"doc\": \"{\\\"n\\\": 1}\"}");
66-
client().performRequest(nullDoc1);
84+
if (!TestUtils.isIndexExist(client(), "test_spath_null")) {
85+
TestUtils.createIndexByRestClient(client(), "test_spath_null", null);
86+
87+
Request nullDoc1 = new Request("PUT", "/test_spath_null/_doc/1?refresh=true");
88+
nullDoc1.setJsonEntity("{\"doc\": \"{\\\"n\\\": 1}\"}");
89+
client().performRequest(nullDoc1);
6790

68-
Request nullDoc2 = new Request("PUT", "/test_spath_null/_doc/2?refresh=true");
69-
nullDoc2.setJsonEntity("{\"doc\": null}");
70-
client().performRequest(nullDoc2);
91+
Request nullDoc2 = new Request("PUT", "/test_spath_null/_doc/2?refresh=true");
92+
nullDoc2.setJsonEntity("{\"doc\": null}");
93+
client().performRequest(nullDoc2);
94+
}
7195
}
7296

7397
@Test

0 commit comments

Comments
 (0)