|
| 1 | +/* |
| 2 | + * Copyright OpenSearch Contributors |
| 3 | + * SPDX-License-Identifier: Apache-2.0 |
| 4 | + */ |
| 5 | + |
| 6 | +package org.opensearch.sql.legacy; |
| 7 | + |
| 8 | +import static org.hamcrest.Matchers.equalTo; |
| 9 | +import static org.hamcrest.Matchers.greaterThan; |
| 10 | + |
| 11 | +import java.io.IOException; |
| 12 | +import org.json.JSONArray; |
| 13 | +import org.json.JSONObject; |
| 14 | +import org.junit.Before; |
| 15 | +import org.junit.Test; |
| 16 | +import org.opensearch.client.Request; |
| 17 | +import org.opensearch.client.Response; |
| 18 | +import org.opensearch.client.ResponseException; |
| 19 | +import org.opensearch.sql.legacy.utils.StringUtils; |
| 20 | + |
| 21 | +/** |
| 22 | + * Integration test verifying PIT contexts are created only when needed and properly cleaned up. |
| 23 | + * |
| 24 | + * @see <a href="https://github.com/opensearch-project/sql/issues/5002">Issue #5002</a> |
| 25 | + */ |
| 26 | +public class PointInTimeLeakIT extends SQLIntegTestCase { |
| 27 | + |
| 28 | + private static final String TEST_INDEX = "test-logs-2025.01.01"; |
| 29 | + private static final String PIT_STATS_ENDPOINT = |
| 30 | + "/_nodes/stats/indices/search?filter_path=nodes.*.indices.search.point_in_time_current"; |
| 31 | + |
| 32 | + @Before |
| 33 | + public void setUpTestIndex() throws IOException { |
| 34 | + try { |
| 35 | + executeRequest(new Request("DELETE", "/" + TEST_INDEX)); |
| 36 | + } catch (ResponseException e) { |
| 37 | + if (e.getResponse().getStatusLine().getStatusCode() != 404) { |
| 38 | + throw e; |
| 39 | + } |
| 40 | + } |
| 41 | + |
| 42 | + Request createIndex = new Request("PUT", "/" + TEST_INDEX); |
| 43 | + createIndex.setJsonEntity( |
| 44 | + "{ \"mappings\": { \"properties\": { \"action\": {\"type\": \"text\", \"fields\":" |
| 45 | + + " {\"keyword\": {\"type\": \"keyword\"}}}, \"timestamp\": {\"type\": \"date\"} " |
| 46 | + + " } }}"); |
| 47 | + executeRequest(createIndex); |
| 48 | + |
| 49 | + Request bulkRequest = new Request("POST", "/" + TEST_INDEX + "/_bulk"); |
| 50 | + bulkRequest.addParameter("refresh", "true"); |
| 51 | + bulkRequest.setJsonEntity( |
| 52 | + "{\"index\":{}}\n" |
| 53 | + + "{\"action\":\"login_success\",\"timestamp\":\"2025-01-01T10:00:00Z\"}\n" |
| 54 | + + "{\"index\":{}}\n" |
| 55 | + + "{\"action\":\"login_success\",\"timestamp\":\"2025-01-01T10:01:00Z\"}\n" |
| 56 | + + "{\"index\":{}}\n" |
| 57 | + + "{\"action\":\"login_failed\",\"timestamp\":\"2025-01-01T10:02:00Z\"}\n"); |
| 58 | + executeRequest(bulkRequest); |
| 59 | + } |
| 60 | + |
| 61 | + @Test |
| 62 | + public void testNoPitLeakWithoutFetchSize() throws IOException, InterruptedException { |
| 63 | + int baselinePitCount = getCurrentPitCount(); |
| 64 | + |
| 65 | + int numQueries = 10; |
| 66 | + |
| 67 | + for (int i = 0; i < numQueries; i++) { |
| 68 | + String query = |
| 69 | + StringUtils.format( |
| 70 | + "SELECT * FROM %s WHERE action LIKE 'login%%' ORDER BY timestamp ASC", TEST_INDEX); |
| 71 | + |
| 72 | + JSONObject response = executeQueryWithoutFetchSize(query); |
| 73 | + |
| 74 | + assertTrue("Query should succeed", response.has("datarows")); |
| 75 | + JSONArray dataRows = response.getJSONArray("datarows"); |
| 76 | + assertThat("Should return results", dataRows.length(), greaterThan(0)); |
| 77 | + assertFalse("Should not have cursor for non-paginated query", response.has("cursor")); |
| 78 | + } |
| 79 | + |
| 80 | + int currentPitCount = getCurrentPitCount(); |
| 81 | + int leakedPits = currentPitCount - baselinePitCount; |
| 82 | + |
| 83 | + assertThat("No PITs should leak after fix", leakedPits, equalTo(0)); |
| 84 | + } |
| 85 | + |
| 86 | + @Test |
| 87 | + public void testPitManagedProperlyWithFetchSize() throws IOException { |
| 88 | + int baselinePitCount = getCurrentPitCount(); |
| 89 | + |
| 90 | + String query = |
| 91 | + StringUtils.format( |
| 92 | + "SELECT * FROM %s WHERE action LIKE 'login%%' ORDER BY timestamp ASC", TEST_INDEX); |
| 93 | + |
| 94 | + JSONObject response = executeQueryWithFetchSize(query, 2); |
| 95 | + |
| 96 | + assertTrue("Should have cursor with fetch_size", response.has("cursor")); |
| 97 | + String cursor = response.getString("cursor"); |
| 98 | + |
| 99 | + JSONObject closeResponse = executeCursorCloseQuery(cursor); |
| 100 | + assertTrue("Cursor close should succeed", closeResponse.getBoolean("succeeded")); |
| 101 | + |
| 102 | + int finalPitCount = getCurrentPitCount(); |
| 103 | + |
| 104 | + assertThat( |
| 105 | + "PIT should be cleaned up after cursor close", finalPitCount, equalTo(baselinePitCount)); |
| 106 | + } |
| 107 | + |
| 108 | + @Test |
| 109 | + public void testCompareV1AndV2EnginePitBehavior() throws IOException { |
| 110 | + int baselinePitCount = getCurrentPitCount(); |
| 111 | + |
| 112 | + String v1Query = |
| 113 | + StringUtils.format( |
| 114 | + "SELECT * FROM %s WHERE action LIKE 'login%%' ORDER BY timestamp ASC", TEST_INDEX); |
| 115 | + |
| 116 | + JSONObject v1Response = executeQueryWithoutFetchSize(v1Query); |
| 117 | + int afterV1PitCount = getCurrentPitCount(); |
| 118 | + int v1Leaked = afterV1PitCount - baselinePitCount; |
| 119 | + |
| 120 | + String v2Query = |
| 121 | + StringUtils.format( |
| 122 | + "SELECT * FROM `%s` WHERE action LIKE 'login%%' ORDER BY timestamp ASC", TEST_INDEX); |
| 123 | + |
| 124 | + JSONObject v2Response = executeQueryWithoutFetchSize(v2Query); |
| 125 | + int afterV2PitCount = getCurrentPitCount(); |
| 126 | + int v2Leaked = afterV2PitCount - afterV1PitCount; |
| 127 | + |
| 128 | + assertTrue("V1 should return results", v1Response.has("datarows")); |
| 129 | + assertTrue("V2 should return results", v2Response.has("datarows")); |
| 130 | + |
| 131 | + assertThat("V1 Legacy SQL should not leak PITs", v1Leaked, equalTo(0)); |
| 132 | + assertThat("V2 SQL should not leak PITs", v2Leaked, equalTo(0)); |
| 133 | + } |
| 134 | + |
| 135 | + private JSONObject executeQueryWithoutFetchSize(String query) throws IOException { |
| 136 | + Request sqlRequest = new Request("POST", "/_plugins/_sql?format=jdbc"); |
| 137 | + sqlRequest.setJsonEntity(String.format("{\"query\": \"%s\"}", query)); |
| 138 | + |
| 139 | + Response response = client().performRequest(sqlRequest); |
| 140 | + return new JSONObject(TestUtils.getResponseBody(response)); |
| 141 | + } |
| 142 | + |
| 143 | + private JSONObject executeQueryWithFetchSize(String query, int fetchSize) throws IOException { |
| 144 | + Request sqlRequest = new Request("POST", "/_plugins/_sql?format=jdbc"); |
| 145 | + sqlRequest.setJsonEntity( |
| 146 | + String.format("{\"query\": \"%s\", \"fetch_size\": %d}", query, fetchSize)); |
| 147 | + |
| 148 | + Response response = client().performRequest(sqlRequest); |
| 149 | + return new JSONObject(TestUtils.getResponseBody(response)); |
| 150 | + } |
| 151 | + |
| 152 | + private int getCurrentPitCount() throws IOException { |
| 153 | + Request statsRequest = new Request("GET", PIT_STATS_ENDPOINT); |
| 154 | + Response response = client().performRequest(statsRequest); |
| 155 | + JSONObject stats = new JSONObject(TestUtils.getResponseBody(response)); |
| 156 | + |
| 157 | + if (!stats.has("nodes")) { |
| 158 | + return 0; |
| 159 | + } |
| 160 | + |
| 161 | + int totalPits = 0; |
| 162 | + JSONObject nodes = stats.getJSONObject("nodes"); |
| 163 | + for (String nodeId : nodes.keySet()) { |
| 164 | + Object pitValue = |
| 165 | + stats.optQuery("/nodes/" + nodeId + "/indices/search/point_in_time_current"); |
| 166 | + if (pitValue instanceof Number) { |
| 167 | + totalPits += ((Number) pitValue).intValue(); |
| 168 | + } |
| 169 | + } |
| 170 | + |
| 171 | + return totalPits; |
| 172 | + } |
| 173 | +} |
0 commit comments